diff --git a/.gitignore b/.gitignore index 2d3023e2b..f30c0fc8f 100644 --- a/.gitignore +++ b/.gitignore @@ -57,5 +57,7 @@ docs/blob-report/ docs/.playwright/ actionlint -# cargo-mutants run output (never commit) +# cargo-mutants run output (never commit). The tool rotates the previous +# run's directory to `.old` on every re-run, so both forms need ignoring. mutants.out/ +mutants.out.old/ diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 616af1e95..f8a099548 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -155,6 +155,16 @@ The `dirsql` Rust crate handles all heavy lifting: Wraps `rusqlite` with an anonymous disk-backed temp database (#402) — ephemeral like `:memory:`, but index pages spill to disk so resident memory does not scale with the corpus. Handles DDL execution (run verbatim -- no injected columns, epic #358), row insertion with per-file ownership recorded in the internal `_dirsql_internal_rows` table, querying, and row deletion by file path. The internal bookkeeping tables (`_dirsql_internal_rows`, `_dirsql_files`, `_dirsql_meta`) are a private surface: a SQLite authorizer installed on the `query()` path denies any read (or schema `PRAGMA`) targeting the reserved `_dirsql_*` namespace, so they are unreachable through the public query surface (issue #378) while the engine still writes them in the same transaction as the user rows. +**A table's name is declared, never derived.** Both `[[table]]` and the SDK +`Table` take a required `name`; nothing tokenizes the `CREATE TABLE` head of +`ddl` to work it out. After the DDL runs, SQLite's own catalog +(`pragma_table_list`) settles whether a table by that name exists — a load-time +error when it does not. That keeps quoted, schema-qualified and +`IF NOT EXISTS` DDL working without dirsql owning a SQL tokenizer, per *never +reinvent what SQLite expresses natively*. `validate_identifier` remains: `name` +is spliced into `format!()`-built INSERT/DELETE SQL, so it is still the +injection guard. + **Named tables are real; path-tables are virtual.** A declared `[[table]]` (or programmatic `Table`) is a real SQLite table whose rows are inserted on build and maintained by the watcher — the `db` module above. A [path-table](../docs/reference/path-tables.md) (`SELECT * FROM './'`, epic path-as-table) is a `dirsql_path` **virtual table** (`vtab.rs` / `path_table.rs`, rusqlite's `vtab` feature): no rows are stored, no reconcile or watcher runs, and the filesystem is walked live at query time (`xFilter`/`xNext` enumerate matched files; `xColumn` supplies stat values and lazily reads `content`). SQLite stays the entire query engine either way; the vtab only enumerates rows and supplies column values. Path-tables are registered on demand — see *Query execution* below. ### `scanner` -- Directory traversal diff --git a/PARITY.md b/PARITY.md index 8f27a49b8..4cc89f906 100644 --- a/PARITY.md +++ b/PARITY.md @@ -6,7 +6,7 @@ API surface comparison across the three language SDKs. | Concept | Python | Rust | TypeScript | |-------------|-------------------------|--------------------------|--------------------------| -| Table def | `Table(ddl, glob, on_file, strict)` | `Table::new(ddl, glob, on_file)` / `Table::strict(...)` / `Table::try_new(...)` | `new Table({...})` or `{ ddl, glob, onFile, strict? }` (plain object) | +| Table def | `Table(name, ddl, glob, on_file, strict)` | `Table::new(name, ddl, glob, on_file)` / `Table::strict(...)` / `Table::try_new(...)` | `new Table({...})` or `{ name, ddl, glob, onFile, strict? }` (plain object) | | on-file callback | `(path) -> list[dict]` | `Fn(&str) -> Vec` | `(path) => Record[]` | | Row event | `RowEvent` (class, frozen attrs; `file_path` on all variants) | `RowEvent` (enum: Insert/Update/Delete/Error; `file_path` on all variants) | `RowEvent` (plain object with action string; `filePath` on all variants) | | Row type | `dict[str, Any]` | `HashMap` | `Record` | @@ -339,7 +339,7 @@ public-API binding, so no drift. ### TypeScript - Uses `camelCase` for method names. - `RowEvent` field names use `camelCase` (`oldRow`, `filePath`), not `snake_case`. -- Table definitions may be written as `new Table({...})` or as a plain object literal (`{ ddl, glob, onFile, strict? }`); `Table` is a thin identity wrapper that exists for parity with the Python/Rust `Table` constructors. Both forms are interchangeable at every call site that takes `TableDef[]`. +- Table definitions may be written as `new Table({...})` or as a plain object literal (`{ name, ddl, glob, onFile, strict? }`); `Table` is a thin identity wrapper that exists for parity with the Python/Rust `Table` constructors. Both forms are interchangeable at every call site that takes `TableDef[]`. - The constructor is overloaded: `new DirSQL(configPath: string)` or `new DirSQL(options: { root?, tables?, ignore?, config? })`. There is no separate `fromConfig` factory. - No separate `AsyncDirSQL` — JS is async by default, so `DirSQL` has `ready: Promise`, `query(): Promise`, and `watch(): AsyncIterable` built in. - `query()`, `startWatcher()`, and `pollEvents()` all return `Promise`s and run on the libuv threadpool so the JS event loop stays responsive (even for long poll timeouts). @@ -371,7 +371,8 @@ Real-core file map: | Config file | `from_config_test.py` | `from_config.rs`, `config.rs` | `from-config.test.ts` | | Persistence | `persist_test.py` | `persist.rs`, `persist_parsed_path_table.rs` | `persist.test.ts` | | Extensions | `extensions_test.py`, `extension_package_test.py`, `config_extension_package_test.py` | `extensions.rs` | `extensions.test.ts`, `extension-package.test.ts`, `config-extension-package.test.ts` | -| Table-name resolution (#204) | `table_name_resolution_test.py` | `table_name_resolution.rs` | `table-name-resolution.test.ts` | +| Quoted-identifier DDL (#204) | `table_name_resolution_test.py` | `table_name_resolution.rs` | (covered by `declared-table-name.test.ts`) | +| Declared table `name` (#962) | `declared_table_name_test.py` | `declared_table_name.rs`, `declared_table_name_e2e.rs` | `declared-table-name.test.ts` | | Docs examples | `docs_examples_test.py` | `docs_examples.rs` | `docs-examples.test.ts` | | Docs gap-fills | `docs_gaps_test.py` | `docs_gaps.rs` | `docs-gaps.test.ts` | | Path-tables (#627/#628/#629) | `path_table_test.py` | `path_table_query.rs`, `path_table_globs.rs`, `vtab.rs` | `path-table.test.ts` | @@ -412,7 +413,7 @@ incl. #313). | Strict mode (extra keys) | Y | Y | Y | | Strict mode (missing keys) | Y | Y | Y | | Strict mode (exact match) | Y | Y | Y | -| `Table` construction + `ddl`/`glob` attributes | Y | Y | Y (`Table` class + plain-object interchangeability) | +| `Table` construction + `name`/`ddl`/`glob` attributes | Y | Y | Y (`Table` class + plain-object interchangeability) | | Quoted-identifier DDL registers/queries by bare name (#204) | Y | Y | Y | | Path-table query (`FROM './'`, glob scoping, zero-match rows, live re-scan, JOIN vs named) | Y (#629) | Y | Y (#629) | | Path-table hidden `content` (excluded from `*`, selectable by name, non-UTF-8 → NULL) | Y (#629) | Y | Y (#629) | @@ -461,6 +462,8 @@ incl. #313). | Missing config file errors | Y | Y | Y | | Invalid TOML errors | Y | core | Y | | `[[table]]` missing `ddl` errors | Y | core | Y | +| `[[table]]` missing `name` errors | Y | Y | Y | +| `[[table]]` `name` absent from the catalog after `ddl` errors | Y | Y | Y | | Config `persist` / `persist_path` resolution | core | Y (`from_config.rs`) | core | ### Persistence diff --git a/README.md b/README.md index 286ac4fa5..4078fdfda 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ async def main(): "./my-blog", tables=[ Table( + name="posts", ddl="CREATE TABLE posts (title TEXT, author TEXT)", glob="posts/*.json", on_file=lambda path: [json.loads(open(path, encoding="utf-8").read())], diff --git a/ROADMAP.md b/ROADMAP.md index 7eee13373..05f673f9b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -35,6 +35,7 @@ Format is **inferrable from glob extension**: `.json` -> `json`, `.jsonl`/`.ndjs ```toml [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT, price REAL)" glob = "catalog/*.json" each = "data.items" @@ -45,6 +46,7 @@ each = "data.items" ```toml [[table]] +name = "comments" ddl = "CREATE TABLE comments (thread_id TEXT, body TEXT, author TEXT)" glob = "comments/{thread_id}/index.jsonl" # thread_id captured from glob path, body and author from JSONL content @@ -67,6 +69,7 @@ columns.display_name = "metadata.author.name" # dot-path into nested object ```toml [[table]] +name = "posts" ddl = "CREATE TABLE posts (title TEXT, author TEXT)" glob = "posts/*.json" # format inferred from .json, columns match JSON keys 1:1 @@ -79,19 +82,23 @@ glob = "posts/*.json" ignore = ["node_modules/**", ".git/**"] [[table]] +name = "comments" ddl = "CREATE TABLE comments (thread_id TEXT, body TEXT, author TEXT, resolved INTEGER)" glob = "_comments/{thread_id}/index.jsonl" [[table]] +name = "documents" ddl = "CREATE TABLE documents (title TEXT, draft INTEGER, body TEXT)" glob = "**/index.md" [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT, price REAL)" glob = "catalog/*.json" each = "data.items" [[table]] +name = "metrics" ddl = "CREATE TABLE metrics (date TEXT, requests INTEGER, errors INTEGER)" glob = "logs/*.csv" ``` diff --git a/docs/getting-started.md b/docs/getting-started.md index efd8eec64..cf56c2540 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -164,6 +164,7 @@ Now create a `.dirsql.toml` that points a table at it: ```bash cat > .dirsql.toml <<'EOF' [[table]] +name = "notes" ddl = "CREATE TABLE notes (title TEXT, author TEXT)" glob = "notes/**/*.md" on-file = "sh note.sh {path}" diff --git a/docs/howto/columns-from-paths.md b/docs/howto/columns-from-paths.md index 9228eb098..2ccc7b05e 100644 --- a/docs/howto/columns-from-paths.md +++ b/docs/howto/columns-from-paths.md @@ -33,6 +33,7 @@ Point a table at it in `.dirsql.toml`: ```toml [[table]] +name = "photos" ddl = "CREATE TABLE photos (year TEXT, month TEXT, basename TEXT)" glob = "photos/*/*/*.jpg" on-file = "python3 pathcols.py {path}" diff --git a/docs/howto/define-tables.md b/docs/howto/define-tables.md index 70467a627..603d038ee 100644 --- a/docs/howto/define-tables.md +++ b/docs/howto/define-tables.md @@ -26,6 +26,7 @@ Then declare the table in `.dirsql.toml`: ```toml [[table]] +name = "posts" ddl = "CREATE TABLE posts (title TEXT, slug TEXT)" glob = "posts/**/*.md" on-file = "python3 extract.py {path}" diff --git a/docs/howto/embed.md b/docs/howto/embed.md index 0a93525ea..649bfe2d5 100644 --- a/docs/howto/embed.md +++ b/docs/howto/embed.md @@ -60,6 +60,7 @@ async def main() -> None: "./comments-root", tables=[ Table( + name="comments", ddl="CREATE TABLE comments (thread TEXT, author TEXT, body TEXT)", glob="comments/*/*.json", on_file=on_file, @@ -83,6 +84,7 @@ const db = new DirSQL({ root: "./comments-root", tables: [ { + name: "comments", ddl: "CREATE TABLE comments (thread TEXT, author TEXT, body TEXT)", glob: "comments/*/*.json", onFile: (path) => [ @@ -129,6 +131,7 @@ fn main() -> Result<(), Box> { let db = DirSQL::builder() .root("./comments-root") .table(Table::new( + "comments", "CREATE TABLE comments (thread TEXT, author TEXT, body TEXT)", "comments/*/*.json", on_file, diff --git a/docs/howto/extract-from-contents.md b/docs/howto/extract-from-contents.md index 8ea2766b4..f9138da8e 100644 --- a/docs/howto/extract-from-contents.md +++ b/docs/howto/extract-from-contents.md @@ -18,6 +18,7 @@ stdout works. With [`jq`](https://jqlang.org/): ```toml [[table]] +name = "books" ddl = "CREATE TABLE books (title TEXT, author TEXT, year INTEGER)" glob = "books/*.json" on-file = "jq -c '[{title, author, year}]' {path}" @@ -53,6 +54,7 @@ row per line, slurp it: ```toml [[table]] +name = "events" ddl = "CREATE TABLE events (event TEXT, user TEXT)" glob = "logs/*.jsonl" on-file = "jq -c -s '.' {path}" diff --git a/docs/howto/parse-files-into-columns.md b/docs/howto/parse-files-into-columns.md index d587836c5..5ac093e98 100644 --- a/docs/howto/parse-files-into-columns.md +++ b/docs/howto/parse-files-into-columns.md @@ -84,6 +84,7 @@ command in verbatim: ```toml [[table]] +name = "posts" ddl = "CREATE TABLE posts (title TEXT, author TEXT)" glob = "posts/*.md" on-file = "python3 extract.py {path}" diff --git a/docs/howto/react-to-changes.md b/docs/howto/react-to-changes.md index 420aa4a35..311915c52 100644 --- a/docs/howto/react-to-changes.md +++ b/docs/howto/react-to-changes.md @@ -14,6 +14,7 @@ prints each file's basename: ```toml # .dirsql.toml [[table]] +name = "files" ddl = "CREATE TABLE files (basename TEXT)" glob = "**/*" on-file = '''sh -c 'printf "[{\"basename\":\"%s\"}]" "${1##*/}"' sh {path}''' diff --git a/docs/howto/skip-files.md b/docs/howto/skip-files.md index c79d2bf44..a6297c18d 100644 --- a/docs/howto/skip-files.md +++ b/docs/howto/skip-files.md @@ -22,6 +22,7 @@ Exclude the noise in `.dirsql.toml`: ignore = ["notes/drafts/**", "**/*.tmp"] [[table]] +name = "notes" ddl = "CREATE TABLE notes (basename TEXT)" glob = "notes/**/*" on-file = '''sh -c 'printf "[{\"basename\":\"%s\"}]" "${1##*/}"' sh {path}''' diff --git a/docs/howto/write-a-plugin.md b/docs/howto/write-a-plugin.md index 768c1d3bf..eb081aa33 100644 --- a/docs/howto/write-a-plugin.md +++ b/docs/howto/write-a-plugin.md @@ -108,6 +108,7 @@ path = "sqlite_vec" entrypoint = "sqlite3_vec_init" [[table]] +name = "notes" ddl = "CREATE TABLE notes (path TEXT, text TEXT, embedding TEXT)" glob = "notes/*.md" on-file = "uv run --with model2vec python embed.py {path} {root}" diff --git a/docs/index.md b/docs/index.md index 3481bc5d7..4117104b6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -34,6 +34,7 @@ db = DirSQL( "./my-project", tables=[ Table( + name="records", ddl="CREATE TABLE records (name TEXT, size INTEGER, type TEXT)", glob="data/*.json", on_file=lambda path: [json.loads(open(path, encoding="utf-8").read())], @@ -52,6 +53,7 @@ let db = DirSQL::new( "./my-project", vec![ Table::new( + "records", "CREATE TABLE records (name TEXT, size INTEGER, type TEXT)", "data/*.json", |path| vec![serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap()], @@ -70,6 +72,7 @@ const db = new DirSQL({ root: './my-project', tables: [ new Table({ + name: 'records', ddl: 'CREATE TABLE records (name TEXT, size INTEGER, type TEXT)', glob: 'data/*.json', onFile: (path) => [JSON.parse(readFileSync(path, 'utf8'))], diff --git a/docs/reference/config.md b/docs/reference/config.md index 79e0ca32c..407be9efa 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -166,7 +166,8 @@ what its required `on-file` command emits — dirsql injects nothing (see | Key | Required | Description | |---|---|---| -| `ddl` | yes | A SQLite `CREATE TABLE` statement. The table name is parsed from it. Only the columns declared here are kept; keys the `on-file` command emits that are not declared are dropped. | +| `name` | yes | The table's SQL name — the name you query it by. Declared, never derived from `ddl`: dirsql does not read the DDL text. The `ddl` must create a table by this name; if it doesn't, loading fails. | +| `ddl` | yes | A SQLite `CREATE TABLE` statement, run verbatim. Only the columns declared here are kept; keys the `on-file` command emits that are not declared are dropped. | | `glob` | yes | Glob pattern matched against root-relative paths. Every table whose glob matches a file receives that file's rows — a file can populate multiple tables. A `{name}` segment is rewritten to `*` (it matches one path segment but captures nothing). | | `on-file` | **yes** | A command run once per matched file; its stdout (a JSON array of row objects) becomes the file's rows. Must be non-empty. A `[[table]]` with no `on-file` is a load error (see [parse errors](#parse-errors)). See [Command hooks](./hooks.md#on-file). | | `strict` | no (default `false`) | When `true`, rows whose keys do not exactly match the declared columns are rejected with an error: extra keys error, and every declared column must be supplied by the `on-file` output. When `false`, extra keys are dropped and missing columns become `NULL`. | @@ -180,11 +181,13 @@ instead of declaring a table. ```toml [[table]] +name = "comments" ddl = "CREATE TABLE comments (path TEXT, author TEXT, body TEXT)" glob = "_comments/*/*.jsonl" on-file = "jq -c -s '.' {path}" [[table]] +name = "papers" ddl = "CREATE TABLE papers (paper_id TEXT, title TEXT)" glob = "**/meta.json" on-file = "uv run python extract_papers.py {path}" @@ -241,7 +244,16 @@ SDKs raise/reject) when: - The TOML is malformed. - Any table contains an unknown key (top level, `[dirsql]`, `[[table]]`, or `[[dirsql.extension]]`). The error names the offending key. -- A `[[table]]` entry omits `ddl` or `glob`. +- A `[[table]]` entry omits `name`, `ddl`, or `glob` (or `name` is + empty/whitespace). +- A `[[table]]` entry's `ddl` runs but creates no table by its `name`. The + error carries the entry's name and points at the fix: + + > `table 'messages': its `ddl` ran but created no table called 'messages'. Set `name` to the table the `ddl` creates.` + + dirsql asks SQLite's catalog rather than interpreting the DDL, so quoted + (`CREATE TABLE "messages"`), schema-qualified (`main.messages`) and + `IF NOT EXISTS` forms all match a plain `name = "messages"`. - A `[[table]]` entry omits `on-file` (or it is empty/whitespace). The error names the offending glob and points at the fix: @@ -275,11 +287,13 @@ deterministic = true timeout = "600s" [[table]] +name = "comments" ddl = "CREATE TABLE comments (author TEXT, body TEXT)" glob = "_comments/*/*.jsonl" on-file = "jq -c -s '.' {path}" [[table]] +name = "documents" ddl = "CREATE TABLE documents (title TEXT, summary TEXT)" glob = "**/index.md" on-file = "uv run python extract_doc.py {path}" diff --git a/docs/reference/sdk.md b/docs/reference/sdk.md index 191a312be..21cf19821 100644 --- a/docs/reference/sdk.md +++ b/docs/reference/sdk.md @@ -317,27 +317,30 @@ rather than waiting — an intentional, language-idiomatic difference. ::: code-group ```python [Python] -Table(*, ddl: str, glob: str, on_file: Callable[[str], list[dict]], strict: bool = False) +Table(*, name: str, ddl: str, glob: str, on_file: Callable[[str], list[dict]], strict: bool = False) ``` ```typescript [TypeScript] -new Table({ ddl, glob, onFile, strict? }) +new Table({ name, ddl, glob, onFile, strict? }) // or a plain object — TableDef and Table are interchangeable: -{ ddl: string, glob: string, onFile: (path: string) => Record[], strict?: boolean } +{ name: string, ddl: string, glob: string, onFile: (path: string) => Record[], strict?: boolean } ``` ```rust [Rust] -Table::new(ddl, glob, on_file) // on_file: Fn(&str) -> Vec, infallible -Table::try_new(ddl, glob, on_file) // on_file: Fn(&str) -> Result, _> -Table::strict(ddl, glob, on_file) // Table::new with strict = true +Table::new(name, ddl, glob, on_file) // on_file: Fn(&str) -> Vec, infallible +Table::try_new(name, ddl, glob, on_file) // on_file: Fn(&str) -> Result, _> +Table::strict(name, ddl, glob, on_file) // Table::new with strict = true ``` ::: Maps files to table rows. -- `ddl` — A SQLite `CREATE TABLE` statement; the table name is parsed from - it. Table names must be unique across all tables. +- `name` — The table's SQL name, declared rather than derived from `ddl`. + Must be unique across all tables. Building fails if the `ddl` creates no + table by this name (checked against SQLite's catalog, before any file is + indexed). +- `ddl` — A SQLite `CREATE TABLE` statement, run verbatim. - `glob` — Glob pattern matched against root-relative paths. Every table whose glob matches a file receives that file's rows — a file can populate multiple tables. A `{name}` segment is rewritten to `*` (it matches one path segment diff --git a/packages/python/changelog.d/2026-08-20-declared-table-name.md b/packages/python/changelog.d/2026-08-20-declared-table-name.md new file mode 100644 index 000000000..1f715a098 --- /dev/null +++ b/packages/python/changelog.d/2026-08-20-declared-table-name.md @@ -0,0 +1,3 @@ +**Changed** + +- **A table's name is now declared, not derived from `ddl`.** `Table` takes a required keyword-only `name` (`Table(name=..., ddl=..., glob=..., on_file=...)`), and a `[[table]]` entry in a `.dirsql.toml` requires a `name` key. The core no longer parses the DDL text to find the name; a `name` its `ddl` does not create fails at load with an error naming the entry. `Table.name` is now the declared `str` rather than an `Optional[str]` parsed from the DDL. (#962) diff --git a/packages/python/dirsql/_dirsql.pyi b/packages/python/dirsql/_dirsql.pyi index 76f3310a2..41af49ca8 100644 --- a/packages/python/dirsql/_dirsql.pyi +++ b/packages/python/dirsql/_dirsql.pyi @@ -30,6 +30,7 @@ class ExtensionSpec(TypedDict): class Table: """A table definition. Construct via keyword arguments only.""" + name: str ddl: str glob: str strict: bool @@ -37,6 +38,7 @@ class Table: def __init__( self, *, + name: str, ddl: str, glob: str, on_file: Callable[[str], list[Row]], diff --git a/packages/python/dirsql/resolve_config_extensions_test.py b/packages/python/dirsql/resolve_config_extensions_test.py index 0481da449..382d8d399 100644 --- a/packages/python/dirsql/resolve_config_extensions_test.py +++ b/packages/python/dirsql/resolve_config_extensions_test.py @@ -70,7 +70,9 @@ def it_returns_none_on_malformed_toml(cfg_dir): def it_returns_none_when_there_is_no_dirsql_section(cfg_dir): path = os.path.join(cfg_dir, ".dirsql.toml") - _write(path, '[[table]]\nddl = "CREATE TABLE t (x TEXT)"\nglob = "*"\n') + _write( + path, '[[table]]\nname = "t"\nddl = "CREATE TABLE t (x TEXT)"\nglob = "*"\n' + ) assert rce.resolve_config_extension_specs(path) is None def it_returns_none_when_no_extensions_are_declared(cfg_dir): @@ -146,7 +148,9 @@ def it_returns_none_when_no_config_uses_a_bare_name(cfg_dir): a = os.path.join(cfg_dir, "a.toml") b = os.path.join(cfg_dir, "b.toml") _write(a, '[[dirsql.extension]]\npath = "ext/a.so"\n') - _write(b, '[[table]]\nddl = "CREATE TABLE t (x TEXT)"\nglob = "*"\n') + _write( + b, '[[table]]\nname = "t"\nddl = "CREATE TABLE t (x TEXT)"\nglob = "*"\n' + ) with _patch() as resolver: assert rce.resolve_configs_extension_specs([a, b]) is None resolver.assert_not_called() diff --git a/packages/python/e2e-attestations/claude-issue-962-0jjrfv.json b/packages/python/e2e-attestations/claude-issue-962-0jjrfv.json new file mode 100644 index 000000000..dccaec853 --- /dev/null +++ b/packages/python/e2e-attestations/claude-issue-962-0jjrfv.json @@ -0,0 +1,7 @@ +{ + "command": "uv run python -m pytest tests/e2e/ -x -q", + "ran_at": 1787251467, + "exit_code": 0, + "commit": "706ed098a4e0bfabb3d498b8669fe194051940dd", + "branch": "claude/issue-962-0jjrfv" +} diff --git a/packages/python/migrations.d/2026-08-20-declared-table-name.md b/packages/python/migrations.d/2026-08-20-declared-table-name.md new file mode 100644 index 000000000..cd1131d83 --- /dev/null +++ b/packages/python/migrations.d/2026-08-20-declared-table-name.md @@ -0,0 +1,52 @@ +### `Table` and `[[table]]` require an explicit `name` (#962) + +#### Summary + +`Table` previously derived its SQL name by parsing the `CREATE TABLE` head of +`ddl`, exposing the result as an `Optional[str]` attribute. The name is now +**declared**: `Table` takes a required keyword-only `name`, and a `[[table]]` +entry in a `.dirsql.toml` requires a `name` key. Whether the `ddl` creates a +table by that name is checked against SQLite's catalog at load time. + +#### Required changes + +| Before | After | Fix | +| ------ | ----- | --- | +| `Table(ddl=..., glob=..., on_file=...)` | `TypeError: Table.__new__() missing 1 required keyword argument: 'name'` | Pass `name=""`, matching the table the `ddl` creates. | +| `[[table]]` with `ddl` + `glob` + `on-file` | `await db.ready()` raises: `Missing required field 'name' in [[table]] entry` | Add `name = "
"` to the entry. | +| `table.name` may be `None` | `table.name` is the declared `str` | Drop any `if table.name is None` handling. | + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +- A `[[table]]` whose `ddl` creates a table under a different name than its + `name` now raises at `ready()` with an error naming the entry + (`table 'messages': …`), before any file is ingested. +- Quoted (`CREATE TABLE "messages"`), schema-qualified and `IF NOT EXISTS` DDL + keep working unchanged: SQLite records the bare identifier, so a plain + `name="messages"` matches. + +#### Verification + +```python +import asyncio, json +from dirsql import DirSQL, Table + +db = DirSQL( + "./data-root", + tables=[ + Table( + name="records", + ddl="CREATE TABLE records (id TEXT)", + glob="data/*.json", + on_file=lambda path: json.load(open(path, encoding="utf-8")), + ) + ], +) +asyncio.run(db.ready()) +# -> builds; `SELECT id FROM records` returns the file's rows. +# Omitting `name=` raises TypeError; a `name` the ddl never creates raises at ready(). +``` diff --git a/packages/python/src/lib.rs b/packages/python/src/lib.rs index 686ea8408..840eb90f6 100644 --- a/packages/python/src/lib.rs +++ b/packages/python/src/lib.rs @@ -12,7 +12,7 @@ #[cfg(feature = "extension-module")] mod python { - use ::dirsql::{DirSQL, Extension, Row, RowEvent, Table, Value, db::parse_table_name}; + use ::dirsql::{DirSQL, Extension, Row, RowEvent, Table, Value}; use pyo3::exceptions::{PyOverflowError, PyRuntimeError}; use pyo3::prelude::*; use pyo3::types::{PyBool, PyByteArray, PyBytes, PyDict, PyInt, PyList}; @@ -24,6 +24,9 @@ mod python { /// callable for `on_file`. #[pyclass(name = "Table", frozen)] struct PyTable { + /// The table's SQL name, declared rather than derived from `ddl`. + #[pyo3(get)] + name: String, #[pyo3(get)] ddl: String, #[pyo3(get)] @@ -32,27 +35,19 @@ mod python { on_file: Py, #[pyo3(get)] strict: bool, - /// Parsed table name (from `ddl`) via `dirsql::db::parse_table_name`, - /// or `None` if the DDL doesn't match `CREATE TABLE (...)`. - /// `None` rather than a construction error so `DirSQL.ready()` keeps - /// surfacing malformed DDLs as the loud failure path (the core's - /// `DirSqlError::Ddl`). - #[pyo3(get)] - name: Option, } #[pymethods] impl PyTable { #[new] - #[pyo3(signature = (*, ddl, glob, on_file, strict=false))] - fn new(ddl: String, glob: String, on_file: Py, strict: bool) -> Self { - let name = parse_table_name(&ddl); + #[pyo3(signature = (*, name, ddl, glob, on_file, strict=false))] + fn new(name: String, ddl: String, glob: String, on_file: Py, strict: bool) -> Self { PyTable { + name, ddl, glob, on_file, strict, - name, } } } @@ -235,6 +230,7 @@ mod python { fn build_table(py: Python<'_>, t: &PyTable) -> Table { let on_file_ref = t.on_file.clone_ref(py); let mut table = Table::try_new( + t.name.clone(), t.ddl.clone(), t.glob.clone(), make_on_file_closure(on_file_ref), diff --git a/packages/python/tests/e2e/capture_collision_test.py b/packages/python/tests/e2e/capture_collision_test.py index c65403495..c61e88891 100644 --- a/packages/python/tests/e2e/capture_collision_test.py +++ b/packages/python/tests/e2e/capture_collision_test.py @@ -42,6 +42,7 @@ def it_exits_nonzero_and_names_the_collision(tmp_path): cfg = root / ".dirsql.toml" cfg.write_text( "[[table]]\n" + 'name = "comments"\n' 'ddl = "CREATE TABLE comments (thread_id TEXT, basename TEXT)"\n' 'glob = "_comments/{thread_id}/*.txt"\n' 'on-file = "cat {path}"\n' diff --git a/packages/python/tests/e2e/declared_table_name_test.py b/packages/python/tests/e2e/declared_table_name_test.py new file mode 100644 index 000000000..7660bbd96 --- /dev/null +++ b/packages/python/tests/e2e/declared_table_name_test.py @@ -0,0 +1,93 @@ +"""CLI e2e: the declared `[[table]] name` key through the real launcher. + +A table's name is declared, never derived from `ddl`. The launcher must query +a config table under its declared `name`, exit non-zero when a `[[table]]` +entry omits `name`, and exit non-zero when the entry's `ddl` never creates +that name. No mocks: real console script, real process, real filesystem, real +SQLite, real `on-file` command spawn. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess + + +def _cli() -> str: + """Resolve the `dirsql` console script for this test env.""" + dirsql = shutil.which("dirsql") + assert dirsql is not None, ( + "`dirsql` console script not on PATH -- run `uv run maturin develop`" + ) + return dirsql + + +def _fixture(tmp_path, config: str): + root = tmp_path / "data-root" + (root / "data").mkdir(parents=True) + (root / "data" / "a.json").write_text('[{"id": "one"}, {"id": "two"}]') + cfg = root / ".dirsql.toml" + cfg.write_text(config) + return root, cfg + + +def _query(root, cfg, sql: str): + return subprocess.run( + [_cli(), "query", sql, "--config", str(cfg)], + cwd=str(root), + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=30, + ) + + +def describe_declared_table_name(): + def it_queries_a_table_under_its_declared_name(tmp_path): + root, cfg = _fixture( + tmp_path, + "[[table]]\n" + 'name = "records"\n' + 'ddl = "CREATE TABLE records (id TEXT)"\n' + 'glob = "data/*.json"\n' + 'on-file = "cat {path}"\n', + ) + + proc = _query(root, cfg, "SELECT id FROM records ORDER BY id") + + assert proc.returncode == 0, f"expected success, got {proc.stderr!r}" + assert [row["id"] for row in json.loads(proc.stdout)] == ["one", "two"] + + def it_exits_nonzero_when_a_table_entry_has_no_name(tmp_path): + root, cfg = _fixture( + tmp_path, + "[[table]]\n" + 'ddl = "CREATE TABLE records (id TEXT)"\n' + 'glob = "data/*.json"\n' + 'on-file = "cat {path}"\n', + ) + + proc = _query(root, cfg, "SELECT id FROM records") + + assert proc.returncode != 0, f"expected failure, got {proc!r}" + assert "name" in proc.stderr and "[[table]]" in proc.stderr, ( + f"stderr must name the missing `name` key, got {proc.stderr!r}" + ) + + def it_exits_nonzero_when_the_ddl_never_creates_the_declared_name(tmp_path): + root, cfg = _fixture( + tmp_path, + "[[table]]\n" + 'name = "messages"\n' + 'ddl = "CREATE TABLE records (id TEXT)"\n' + 'glob = "data/*.json"\n' + 'on-file = "cat {path}"\n', + ) + + proc = _query(root, cfg, "SELECT id FROM messages") + + assert proc.returncode != 0, f"expected failure, got {proc!r}" + assert "table 'messages'" in proc.stderr, ( + f"stderr must carry the config-entry prefix, got {proc.stderr!r}" + ) diff --git a/packages/python/tests/e2e/extension_package_test.py b/packages/python/tests/e2e/extension_package_test.py index 5ac8bc0ac..1ffc97705 100644 --- a/packages/python/tests/e2e/extension_package_test.py +++ b/packages/python/tests/e2e/extension_package_test.py @@ -145,6 +145,7 @@ def it_resolves_loads_and_queries_a_package_name_extension(staged, tmp_path): 'path = "dirsql_testext_pkg"\n' 'entrypoint = "sqlite3_extension_init"\n\n' "[[table]]\n" + 'name = "files"\n' 'ddl = "CREATE TABLE files (path TEXT)"\n' 'glob = "*.txt"\n' f"{_HOOK_PATH}\n" diff --git a/packages/python/tests/e2e/fanout_test.py b/packages/python/tests/e2e/fanout_test.py index 53721f0cb..931bc087f 100644 --- a/packages/python/tests/e2e/fanout_test.py +++ b/packages/python/tests/e2e/fanout_test.py @@ -86,11 +86,13 @@ def server(tmp_path): cfg = root / ".dirsql.toml" cfg.write_text( "[[table]]\n" + 'name = "ta"\n' 'ddl = "CREATE TABLE ta (path TEXT)"\n' 'glob = "data/*/metadata.json"\n' f"{_HOOK_PATH}\n" "\n" "[[table]]\n" + 'name = "tb"\n' 'ddl = "CREATE TABLE tb (path TEXT)"\n' 'glob = "data/**/metadata.json"\n' f"{_HOOK_PATH}\n" diff --git a/packages/python/tests/e2e/fixtures/dirsql_plugin_fixture/dirsql.toml b/packages/python/tests/e2e/fixtures/dirsql_plugin_fixture/dirsql.toml index 88a68d4c2..965f1af19 100644 --- a/packages/python/tests/e2e/fixtures/dirsql_plugin_fixture/dirsql.toml +++ b/packages/python/tests/e2e/fixtures/dirsql_plugin_fixture/dirsql.toml @@ -3,6 +3,7 @@ # injected by the launcher as an ordinary `-c` flag. Adds one table so an # installed plugin is observable ("installed = active") with zero config edits. [[table]] +name = "notes" ddl = "CREATE TABLE notes (path TEXT, basename TEXT)" glob = "*.md" on-file = '''sh -c 'rel=${1#"$2"/}; base=${1##*/}; printf "[{\"path\":\"%s\",\"basename\":\"%s\"}]" "$rel" "$base"' sh {path} {root}''' diff --git a/packages/python/tests/e2e/plugin_discovery_test.py b/packages/python/tests/e2e/plugin_discovery_test.py index 74319db47..021220e1a 100644 --- a/packages/python/tests/e2e/plugin_discovery_test.py +++ b/packages/python/tests/e2e/plugin_discovery_test.py @@ -147,6 +147,7 @@ def it_composes_a_user_config_with_the_plugin(staged): cfg = data / "user.toml" cfg.write_text( "[[table]]\n" + 'name = "posts"\n' 'ddl = "CREATE TABLE posts (path TEXT, basename TEXT)"\n' 'glob = "*.md"\n' f"{_HOOK_PATH_BASENAME}\n" @@ -166,7 +167,7 @@ def it_errors_when_a_plugin_table_collides_with_a_user_config(staged): site_dir, data = staged cfg = data / "dup.toml" cfg.write_text( - '[[table]]\nddl = "CREATE TABLE notes (path TEXT)"\nglob = "*.md"\n' + '[[table]]\nname = "notes"\nddl = "CREATE TABLE notes (path TEXT)"\nglob = "*.md"\n' 'on-file = "cat {path}"\n' ) out = _run(site_dir, ["query", "SELECT 1", "-c", "dup.toml"], data) diff --git a/packages/python/tests/e2e/plugin_extension_discovery_test.py b/packages/python/tests/e2e/plugin_extension_discovery_test.py index 64dcfd952..def03affb 100644 --- a/packages/python/tests/e2e/plugin_extension_discovery_test.py +++ b/packages/python/tests/e2e/plugin_extension_discovery_test.py @@ -163,6 +163,7 @@ def it_resolves_the_fragment_alongside_a_user_config(staged): cfg = data / "user.toml" cfg.write_text( "[[table]]\n" + 'name = "posts"\n' 'ddl = "CREATE TABLE posts (path TEXT)"\n' 'glob = "*.md"\n' 'on-file = "cat {path}"\n' diff --git a/packages/python/tests/e2e/query_contract_test.py b/packages/python/tests/e2e/query_contract_test.py index 60cd830b1..43d3f8f34 100644 --- a/packages/python/tests/e2e/query_contract_test.py +++ b/packages/python/tests/e2e/query_contract_test.py @@ -99,7 +99,7 @@ def server(tmp_path): (root / "a.txt").write_text("hello") cfg = root / ".dirsql.toml" cfg.write_text( - '[[table]]\nddl = "CREATE TABLE files (path TEXT)"\nglob = "*.txt"\n' + '[[table]]\nname = "files"\nddl = "CREATE TABLE files (path TEXT)"\nglob = "*.txt"\n' f"{_HOOK_PATH}\n" ) diff --git a/packages/python/tests/integration/binding/async_dirsql_test.py b/packages/python/tests/integration/binding/async_dirsql_test.py index 7f37adda8..d81b3e88c 100644 --- a/packages/python/tests/integration/binding/async_dirsql_test.py +++ b/packages/python/tests/integration/binding/async_dirsql_test.py @@ -17,6 +17,7 @@ async def it_creates_instance_synchronously(jsonl_dir): jsonl_dir, tables=[ Table( + name="comments", ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)", glob="comments/**/index.jsonl", on_file=lambda path: [ @@ -39,6 +40,7 @@ async def it_indexes_files_after_ready(jsonl_dir): jsonl_dir, tables=[ Table( + name="comments", ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)", glob="comments/**/index.jsonl", on_file=lambda path: [ @@ -67,6 +69,7 @@ async def it_skips_the_file_on_extract_error_during_ready(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT)", glob="data/*.json", on_file=lambda path: [ @@ -87,6 +90,7 @@ async def it_allows_multiple_ready_calls(jsonl_dir): jsonl_dir, tables=[ Table( + name="comments", ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)", glob="comments/**/index.jsonl", on_file=lambda path: [ @@ -113,6 +117,7 @@ async def it_returns_results_as_list_of_dicts(jsonl_dir): jsonl_dir, tables=[ Table( + name="comments", ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)", glob="comments/**/index.jsonl", on_file=lambda path: [ @@ -140,6 +145,7 @@ async def it_awaits_ready_transparently_for_eager_queries(jsonl_dir): jsonl_dir, tables=[ Table( + name="comments", ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)", glob="comments/**/index.jsonl", on_file=lambda path: [ @@ -165,6 +171,7 @@ async def it_raises_on_invalid_sql(jsonl_dir): jsonl_dir, tables=[ Table( + name="comments", ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)", glob="comments/**/index.jsonl", on_file=lambda path: [ @@ -190,6 +197,7 @@ async def it_emits_insert_events_for_new_files(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT)", glob="**/*.json", on_file=lambda path: [ @@ -241,6 +249,7 @@ async def it_sets_file_path_relative_to_root_on_events(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT)", glob="**/*.json", on_file=lambda path: [ @@ -295,6 +304,7 @@ async def it_emits_a_delete_when_a_multi_row_file_shrinks(tmp_dir): tmp_dir, tables=[ Table( + name="rows", ddl="CREATE TABLE rows (idx INTEGER, name TEXT)", glob="*.jsonl", on_file=lambda path: [ @@ -357,6 +367,7 @@ async def it_emits_delete_events_for_removed_files(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT)", glob="**/*.json", on_file=lambda path: [ @@ -405,6 +416,7 @@ async def it_emits_update_events_for_modified_files(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT)", glob="**/*.json", on_file=lambda path: [ @@ -450,6 +462,7 @@ async def it_emits_error_events_for_bad_extract(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT)", glob="**/*.json", on_file=lambda path: [ @@ -495,6 +508,7 @@ async def it_updates_db_on_file_changes(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT)", glob="**/*.json", on_file=lambda path: [ @@ -545,6 +559,7 @@ async def it_watches_without_awaiting_ready_first(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT)", glob="**/*.json", on_file=lambda path: [ @@ -594,6 +609,7 @@ async def it_surfaces_the_real_init_error_on_watch_without_ready(tmp_dir): tmp_dir, tables=[ Table( + name="t", ddl="NOT VALID SQL", glob="data/*.json", on_file=lambda path: [ diff --git a/packages/python/tests/integration/binding/declared_table_name_test.py b/packages/python/tests/integration/binding/declared_table_name_test.py new file mode 100644 index 000000000..8062b0960 --- /dev/null +++ b/packages/python/tests/integration/binding/declared_table_name_test.py @@ -0,0 +1,98 @@ +"""Binding-tier tests (real core, real fs) for the declared table `name`. + +A table's name is *declared*, never derived from ``ddl``. ``Table`` takes it +as a required keyword argument, a ``[[table]]`` entry without ``name`` fails +to load, and a ``name`` the entry's ``ddl`` does not actually create fails at +load time -- checked against SQLite's own catalog, before any ingestion. +""" + +import os +import tempfile + +import pytest + +from dirsql import DirSQL, Table + +# An `on-file` hook emitting the file's root-relative `path`. +_HOOK_PATH = r"""on-file = '''sh -c 'rel=${1#"$2"/}; printf "[{\"path\":\"%s\"}]" "$rel"' sh {path} {root}'''""" + + +@pytest.fixture +def config_dir(): + with tempfile.TemporaryDirectory() as d: + yield d + + +def _fixture(config_dir, config): + os.makedirs(os.path.join(config_dir, "data"), exist_ok=True) + with open(os.path.join(config_dir, "data", "a.csv"), "w") as f: + f.write("anything") + path = os.path.join(config_dir, ".dirsql.toml") + with open(path, "w") as f: + f.write(config) + return path + + +def describe_declared_table_name(): + def it_exposes_the_declared_name_on_the_table(): + table = Table( + name="notes", + ddl='CREATE TABLE "notes" (path TEXT)', + glob="data/*.csv", + on_file=lambda path: [], + ) + assert table.name == "notes" + + @pytest.mark.asyncio + async def it_registers_a_config_table_under_its_declared_name(config_dir): + cfg = _fixture( + config_dir, + """\ +[[table]] +name = "notes" +ddl = "CREATE TABLE notes (path TEXT)" +glob = "data/*.csv" +""" + + _HOOK_PATH + + "\n", + ) + + db = DirSQL(root=config_dir, config=cfg) + await db.ready() + results = await db.query("SELECT path FROM notes") + assert [r["path"] for r in results] == ["data/a.csv"] + + @pytest.mark.asyncio + async def it_errors_when_a_table_entry_has_no_name(config_dir): + cfg = _fixture( + config_dir, + """\ +[[table]] +ddl = "CREATE TABLE notes (path TEXT)" +glob = "data/*.csv" +""" + + _HOOK_PATH + + "\n", + ) + + db = DirSQL(root=config_dir, config=cfg) + with pytest.raises(Exception, match="name"): + await db.ready() + + @pytest.mark.asyncio + async def it_errors_when_the_ddl_never_creates_the_declared_name(config_dir): + cfg = _fixture( + config_dir, + """\ +[[table]] +name = "messages" +ddl = "CREATE TABLE notes (path TEXT)" +glob = "data/*.csv" +""" + + _HOOK_PATH + + "\n", + ) + + db = DirSQL(root=config_dir, config=cfg) + with pytest.raises(Exception, match="table 'messages'"): + await db.ready() diff --git a/packages/python/tests/integration/binding/dirsql_test.py b/packages/python/tests/integration/binding/dirsql_test.py index 4cc1de5f2..9a7a0e45f 100644 --- a/packages/python/tests/integration/binding/dirsql_test.py +++ b/packages/python/tests/integration/binding/dirsql_test.py @@ -16,6 +16,7 @@ async def it_creates_instance_with_tables(jsonl_dir): jsonl_dir, tables=[ Table( + name="comments", ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)", glob="comments/**/index.jsonl", on_file=lambda path: [ @@ -39,6 +40,7 @@ async def it_accepts_ignore_patterns(jsonl_dir): ignore=["**/def/**"], tables=[ Table( + name="comments", ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)", glob="comments/**/index.jsonl", on_file=lambda path: [ @@ -86,6 +88,7 @@ async def it_returns_all_rows(jsonl_dir): jsonl_dir, tables=[ Table( + name="comments", ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)", glob="comments/**/index.jsonl", on_file=lambda path: [ @@ -110,6 +113,7 @@ async def it_returns_dicts_with_column_names(jsonl_dir): jsonl_dir, tables=[ Table( + name="comments", ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)", glob="comments/**/index.jsonl", on_file=lambda path: [ @@ -137,6 +141,7 @@ async def it_filters_with_where_clause(jsonl_dir): jsonl_dir, tables=[ Table( + name="comments", ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)", glob="comments/**/index.jsonl", on_file=lambda path: [ @@ -162,6 +167,7 @@ async def it_excludes_internal_tracking_columns(jsonl_dir): jsonl_dir, tables=[ Table( + name="comments", ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)", glob="comments/**/index.jsonl", on_file=lambda path: [ @@ -193,6 +199,7 @@ async def it_handles_integer_values(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT, count INTEGER)", glob="data/*.json", on_file=lambda path: [ @@ -223,6 +230,7 @@ async def it_supports_multiple_table_definitions(tmp_dir): tmp_dir, tables=[ Table( + name="posts", ddl="CREATE TABLE posts (title TEXT, author_id TEXT)", glob="posts/*.json", on_file=lambda path: [ @@ -230,6 +238,7 @@ async def it_supports_multiple_table_definitions(tmp_dir): ], ), Table( + name="authors", ddl="CREATE TABLE authors (id TEXT, name TEXT)", glob="authors/*.json", on_file=lambda path: [ @@ -261,6 +270,7 @@ async def it_supports_joins_across_tables(tmp_dir): tmp_dir, tables=[ Table( + name="posts", ddl="CREATE TABLE posts (title TEXT, author_id TEXT)", glob="posts/*.json", on_file=lambda path: [ @@ -268,6 +278,7 @@ async def it_supports_joins_across_tables(tmp_dir): ], ), Table( + name="authors", ddl="CREATE TABLE authors (id TEXT, name TEXT)", glob="authors/*.json", on_file=lambda path: [ @@ -292,6 +303,7 @@ async def it_raises_on_invalid_sql(jsonl_dir): jsonl_dir, tables=[ Table( + name="comments", ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)", glob="comments/**/index.jsonl", on_file=lambda path: [ @@ -319,6 +331,7 @@ async def it_rejects_write_statements_via_query(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT)", glob="*.json", on_file=lambda path: [ @@ -355,6 +368,7 @@ async def it_rejects_attach_and_creates_no_file(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT)", glob="*.json", on_file=lambda path: [ @@ -396,6 +410,7 @@ async def it_cannot_read_external_db_via_attach(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT)", glob="data/*.json", on_file=lambda path: [ @@ -417,6 +432,7 @@ async def it_raises_on_invalid_ddl(tmp_dir): tmp_dir, tables=[ Table( + name="t", ddl="NOT A CREATE TABLE", glob="*.json", on_file=lambda path: [], @@ -432,6 +448,7 @@ async def it_handles_empty_directory(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT)", glob="**/*.json", on_file=lambda path: [ @@ -453,6 +470,7 @@ async def it_handles_extract_returning_empty_list(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT)", glob="**/*.json", on_file=lambda path: [], @@ -480,6 +498,7 @@ def on_file(path): tmp_dir, tables=[ Table( + name="t", ddl="CREATE TABLE t (val INTEGER)", glob="*.json", on_file=on_file, @@ -502,6 +521,7 @@ async def it_round_trips_a_reserved_word_column(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl='CREATE TABLE items (name TEXT, "order" INTEGER)', glob="*.json", on_file=lambda path: [ @@ -526,6 +546,7 @@ async def it_ignores_extra_keys_by_default(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT)", glob="*.json", on_file=lambda path: [ @@ -550,6 +571,7 @@ async def it_fills_missing_keys_with_null(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT, color TEXT, count INTEGER)", glob="*.json", on_file=lambda path: [ @@ -574,6 +596,7 @@ async def it_skips_the_file_on_extra_keys_in_strict_mode(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT)", glob="*.json", on_file=lambda path: [ @@ -598,6 +621,7 @@ async def it_skips_the_file_on_missing_keys_in_strict_mode(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT, color TEXT)", glob="*.json", on_file=lambda path: [ @@ -619,6 +643,7 @@ async def it_allows_exact_match_in_strict_mode(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT, color TEXT)", glob="*.json", on_file=lambda path: [ diff --git a/packages/python/tests/integration/binding/extensions_test.py b/packages/python/tests/integration/binding/extensions_test.py index fb62816ad..2aa8e832f 100644 --- a/packages/python/tests/integration/binding/extensions_test.py +++ b/packages/python/tests/integration/binding/extensions_test.py @@ -25,6 +25,7 @@ async def it_raises_when_a_constructor_extension_is_missing(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT)", glob="*.json", on_file=_noop_extract, @@ -56,6 +57,7 @@ async def it_builds_normally_when_no_extensions_are_given(tmp_dir): tmp_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT)", glob="*.json", on_file=lambda _path: [{"name": "x"}], diff --git a/packages/python/tests/integration/binding/fanout_test.py b/packages/python/tests/integration/binding/fanout_test.py index 1665ce919..e5ebd65b9 100644 --- a/packages/python/tests/integration/binding/fanout_test.py +++ b/packages/python/tests/integration/binding/fanout_test.py @@ -21,6 +21,7 @@ def _write_fanout_file(tmp_dir): def _table(name, glob, col, val): return Table( + name=name, ddl=f"CREATE TABLE {name} ({col} TEXT)", glob=glob, on_file=lambda _path, col=col, val=val: [{col: val}], @@ -72,6 +73,7 @@ async def it_errors_when_a_placeholder_collides_with_a_column(tmp_dir): tmp_dir, tables=[ Table( + name="a", ddl="CREATE TABLE a (id TEXT, col_a TEXT)", glob="data/{id}/metadata.json", on_file=lambda _path: [{"col_a": "A"}], diff --git a/packages/python/tests/integration/binding/from_config_test.py b/packages/python/tests/integration/binding/from_config_test.py index c9293ddc3..496d7c9da 100644 --- a/packages/python/tests/integration/binding/from_config_test.py +++ b/packages/python/tests/integration/binding/from_config_test.py @@ -45,6 +45,7 @@ async def it_produces_one_row_per_matched_file(config_dir): os.path.join(config_dir, ".dirsql.toml"), """\ [[table]] +name = "items" ddl = "CREATE TABLE items (path TEXT, basename TEXT)" glob = "items/*.csv" """ @@ -74,6 +75,7 @@ async def it_errors_when_a_placeholder_collides_with_a_column(config_dir): os.path.join(config_dir, ".dirsql.toml"), """\ [[table]] +name = "comments" ddl = "CREATE TABLE comments (thread_id TEXT, basename TEXT)" glob = "comments/{thread_id}/*.txt" """ @@ -101,6 +103,7 @@ async def it_treats_a_non_colliding_placeholder_as_a_wildcard(config_dir): os.path.join(config_dir, ".dirsql.toml"), """\ [[table]] +name = "comments" ddl = "CREATE TABLE comments (path TEXT, basename TEXT)" glob = "comments/{thread_id}/*.txt" """ @@ -127,6 +130,7 @@ async def it_exposes_stat_virtuals(config_dir): os.path.join(config_dir, ".dirsql.toml"), """\ [[table]] +name = "files" ddl = "CREATE TABLE files (path TEXT, basename TEXT, dir TEXT, ext TEXT, size INTEGER, mtime INTEGER)" glob = "docs/*.md" """ @@ -166,6 +170,7 @@ async def it_respects_ignore_patterns(config_dir): ignore = ["**/node_modules/**"] [[table]] +name = "items" ddl = "CREATE TABLE items (path TEXT)" glob = "data/**/*.json" """ @@ -190,6 +195,7 @@ async def it_loads_multiple_tables(config_dir): os.path.join(config_dir, ".dirsql.toml"), """\ [[table]] +name = "posts" ddl = "CREATE TABLE posts (basename TEXT)" glob = "posts/*.txt" """ @@ -197,6 +203,7 @@ async def it_loads_multiple_tables(config_dir): + """ [[table]] +name = "authors" ddl = "CREATE TABLE authors (basename TEXT)" glob = "authors/*.txt" """ @@ -238,6 +245,7 @@ async def it_raises_on_a_hookless_table(config_dir): os.path.join(config_dir, ".dirsql.toml"), """\ [[table]] +name = "files" ddl = "CREATE TABLE files (path TEXT, size INTEGER)" glob = "**/*.md" """, @@ -252,6 +260,7 @@ async def it_raises_on_missing_ddl(config_dir): os.path.join(config_dir, ".dirsql.toml"), """\ [[table]] +name = "t" glob = "*.json" """, ) @@ -267,6 +276,7 @@ async def it_supports_sql_queries_after_config_init(config_dir): os.path.join(config_dir, ".dirsql.toml"), """\ [[table]] +name = "items" ddl = "CREATE TABLE items (basename TEXT, size INTEGER)" glob = "items/*.json" """ diff --git a/packages/python/tests/integration/binding/on_file_rename_test.py b/packages/python/tests/integration/binding/on_file_rename_test.py index d8adb796e..5b9ce7036 100644 --- a/packages/python/tests/integration/binding/on_file_rename_test.py +++ b/packages/python/tests/integration/binding/on_file_rename_test.py @@ -31,6 +31,7 @@ async def it_builds_tables_via_the_on_file_keyword(jsonl_dir): jsonl_dir, tables=[ Table( + name="comments", ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)", glob="comments/**/index.jsonl", on_file=_rows, @@ -44,6 +45,7 @@ async def it_builds_tables_via_the_on_file_keyword(jsonl_dir): def it_rejects_the_old_extract_keyword(): with pytest.raises(TypeError): Table( + name="t", ddl="CREATE TABLE t (n INTEGER)", glob="**/*.json", extract=lambda path: [], diff --git a/packages/python/tests/integration/binding/path_table_test.py b/packages/python/tests/integration/binding/path_table_test.py index 8279e1356..3095a25fd 100644 --- a/packages/python/tests/integration/binding/path_table_test.py +++ b/packages/python/tests/integration/binding/path_table_test.py @@ -27,6 +27,7 @@ def _open(root): root, tables=[ Table( + name="rows_csv", ddl="CREATE TABLE rows_csv (path TEXT)", glob="docs/*.csv", on_file=lambda path: [{"path": "docs/c.csv"}], diff --git a/packages/python/tests/integration/binding/persist_test.py b/packages/python/tests/integration/binding/persist_test.py index 87835a90e..a27ee38c0 100644 --- a/packages/python/tests/integration/binding/persist_test.py +++ b/packages/python/tests/integration/binding/persist_test.py @@ -24,6 +24,7 @@ def on_file(path): return [json.loads(open(path, encoding="utf-8").read())] return Table( + name="items", ddl="CREATE TABLE items (name TEXT, price REAL)", glob="items/*.json", on_file=on_file, @@ -189,6 +190,7 @@ async def it_forces_full_rebuild_on_config_change(persist_dir): persist_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT, price REAL, sku TEXT)", glob="items/*.json", on_file=lambda path: ( @@ -227,6 +229,7 @@ async def it_excludes_dotdirsql_from_walk(persist_dir): persist_dir, tables=[ Table( + name="items", ddl="CREATE TABLE items (name TEXT, price REAL)", glob="**/*.json", on_file=lambda path: [ diff --git a/packages/python/tests/integration/binding/repeatable_config_test.py b/packages/python/tests/integration/binding/repeatable_config_test.py index 869347bc4..7837a81cd 100644 --- a/packages/python/tests/integration/binding/repeatable_config_test.py +++ b/packages/python/tests/integration/binding/repeatable_config_test.py @@ -31,7 +31,7 @@ def _write(path, content): def _table_config(name, glob): return ( - f'[[table]]\nddl = "CREATE TABLE {name} (basename TEXT)"\nglob = "{glob}"\n' + f'[[table]]\nname = "{name}"\nddl = "CREATE TABLE {name} (basename TEXT)"\nglob = "{glob}"\n' f"{_HOOK_BASENAME}\n" ) diff --git a/packages/python/tests/integration/binding/scan_failures_test.py b/packages/python/tests/integration/binding/scan_failures_test.py index 5d2931e73..08cb75764 100644 --- a/packages/python/tests/integration/binding/scan_failures_test.py +++ b/packages/python/tests/integration/binding/scan_failures_test.py @@ -13,10 +13,10 @@ from dirsql import DirSQL, Table -def _db(tmp_dir, on_file, ddl="CREATE TABLE items (name TEXT)", **kw): +def _db(tmp_dir, on_file, name="items", ddl="CREATE TABLE items (name TEXT)", **kw): return DirSQL( tmp_dir, - tables=[Table(ddl=ddl, glob="*.json", on_file=on_file, **kw)], + tables=[Table(name=name, ddl=ddl, glob="*.json", on_file=on_file, **kw)], ) diff --git a/packages/python/tests/integration/binding/table_name_resolution_test.py b/packages/python/tests/integration/binding/table_name_resolution_test.py index 1d60eb163..503beb700 100644 --- a/packages/python/tests/integration/binding/table_name_resolution_test.py +++ b/packages/python/tests/integration/binding/table_name_resolution_test.py @@ -1,9 +1,9 @@ -"""Binding-tier tests (real core) for table-name resolution from DDL. +"""Binding-tier tests (real core) for a quoted-identifier DDL. -The only thing dirsql needs from a (bring-your-own) DDL is the table *name*, -which must also resolve for a **quoted identifier** -- the canonical shape -emitted by ORMs / schema tools. ``Table.name`` must be the bare identifier, -and a quoted-DDL table fully usable end to end. +A table's name is declared, but its ``ddl`` is bring-your-own -- including the +**quoted identifier** ORMs and schema tools emit. Quotes are SQL *delimiters*, +not part of the name, so SQLite records the bare identifier and the declared +``name`` matches it without dirsql reading the DDL text. """ import json @@ -15,20 +15,13 @@ def describe_quoted_identifier_ddl(): - def it_resolves_table_name_to_the_bare_identifier(): - table = Table( - ddl='CREATE TABLE "comments" (id TEXT, body TEXT, author TEXT)', - glob="comments/**/index.jsonl", - on_file=lambda path: [], - ) - assert table.name == "comments" - @pytest.mark.asyncio async def it_registers_and_queries_a_quoted_ddl_table(jsonl_dir): db = DirSQL( jsonl_dir, tables=[ Table( + name="comments", ddl='CREATE TABLE "comments" (id TEXT, body TEXT, author TEXT)', glob="comments/**/index.jsonl", on_file=lambda path: [ diff --git a/packages/python/tests/integration/binding/value_fidelity_test.py b/packages/python/tests/integration/binding/value_fidelity_test.py index 0fdf2a25c..e37fdc0f1 100644 --- a/packages/python/tests/integration/binding/value_fidelity_test.py +++ b/packages/python/tests/integration/binding/value_fidelity_test.py @@ -11,10 +11,12 @@ from dirsql import DirSQL, Table -def _db(tmp_dir, on_file, ddl="CREATE TABLE t (v)"): +def _db(tmp_dir, on_file, name="t", ddl="CREATE TABLE t (v)"): with open(os.path.join(tmp_dir, "marker.json"), "w") as f: f.write("{}") - return DirSQL(tmp_dir, tables=[Table(ddl=ddl, glob="*.json", on_file=on_file)]) + return DirSQL( + tmp_dir, tables=[Table(name=name, ddl=ddl, glob="*.json", on_file=on_file)] + ) def describe_integer_range(): diff --git a/packages/rust/benches/db_bench.rs b/packages/rust/benches/db_bench.rs index e0168aba0..8bc995ef7 100644 --- a/packages/rust/benches/db_bench.rs +++ b/packages/rust/benches/db_bench.rs @@ -20,8 +20,11 @@ fn bench_create_table(c: &mut Criterion) { c.bench_function("db/create_table", |b| { b.iter(|| { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE items (id INTEGER, name TEXT, score REAL)") - .unwrap(); + db.create_table( + "items", + "CREATE TABLE items (id INTEGER, name TEXT, score REAL)", + ) + .unwrap(); }); }); } @@ -32,8 +35,11 @@ fn bench_insert_rows(c: &mut Criterion) { group.bench_with_input(BenchmarkId::from_parameter(count), &count, |b, &n| { b.iter(|| { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE items (id INTEGER, name TEXT, score REAL)") - .unwrap(); + db.create_table( + "items", + "CREATE TABLE items (id INTEGER, name TEXT, score REAL)", + ) + .unwrap(); for i in 0..n { db.insert_row("items", &make_row(i), "data.jsonl", i) .unwrap(); @@ -49,8 +55,11 @@ fn bench_query(c: &mut Criterion) { for count in [100, 1000] { group.bench_with_input(BenchmarkId::from_parameter(count), &count, |b, &n| { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE items (id INTEGER, name TEXT, score REAL)") - .unwrap(); + db.create_table( + "items", + "CREATE TABLE items (id INTEGER, name TEXT, score REAL)", + ) + .unwrap(); for i in 0..n { db.insert_row("items", &make_row(i), "data.jsonl", i) .unwrap(); @@ -69,8 +78,11 @@ fn bench_delete_by_file(c: &mut Criterion) { b.iter_with_setup( || { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE items (id INTEGER, name TEXT, score REAL)") - .unwrap(); + db.create_table( + "items", + "CREATE TABLE items (id INTEGER, name TEXT, score REAL)", + ) + .unwrap(); for i in 0..500 { let file = if i % 2 == 0 { "a.jsonl" } else { "b.jsonl" }; db.insert_row("items", &make_row(i), file, i).unwrap(); diff --git a/packages/rust/changelog.d/2026-08-20-declared-table-name.md b/packages/rust/changelog.d/2026-08-20-declared-table-name.md new file mode 100644 index 000000000..f7ac99d74 --- /dev/null +++ b/packages/rust/changelog.d/2026-08-20-declared-table-name.md @@ -0,0 +1,3 @@ +**Changed** + +- **A table's name is now declared, not derived from `ddl`.** `[[table]]` gains a required `name` key, and `Table::new` / `Table::try_new` / `Table::strict` take it as their first argument. dirsql no longer tokenizes the `CREATE TABLE` head of `ddl` to work the name out — `db::parse_table_name` is gone, along with the bespoke SQL-as-text parsing it needed for quoted, schema-qualified and `IF NOT EXISTS` forms. Whether the `ddl` actually creates a table by that name is settled after it runs, against SQLite's own catalog (`pragma_table_list`): no row means a load-time error carrying the config entry's name, raised before any file is ingested. `Db::create_table` now takes `(name, ddl)`; `validate_identifier` stays as the injection guard, since `name` is spliced into `format!()`-built INSERT/DELETE SQL. Duplicate-table detection across composed configs compares `name` keys. (#962) diff --git a/packages/rust/migrations.d/2026-08-20-declared-table-name.md b/packages/rust/migrations.d/2026-08-20-declared-table-name.md new file mode 100644 index 000000000..1a1f0972f --- /dev/null +++ b/packages/rust/migrations.d/2026-08-20-declared-table-name.md @@ -0,0 +1,65 @@ +### `[[table]]` and `Table` require an explicit `name` (#962) + +#### Summary + +A table's SQL name was derived by tokenizing the `CREATE TABLE` head of `ddl` +(`db::parse_table_name`) — bespoke SQL-as-text parsing that had to grow cases +for quoted identifiers, `schema.table`, `IF NOT EXISTS`, and leading comments. +The name is now **declared**: `[[table]]` requires a `name` key, and +`Table::new` / `Table::try_new` / `Table::strict` take it as their first +argument. After the DDL runs, SQLite's own catalog (`pragma_table_list`) +settles whether a table by that name exists; if not, the build fails before any +file is ingested. `Db::create_table` takes `(name, ddl)`, and +`db::parse_table_name` is deleted. + +#### Required changes + +| Before | After | Fix | +| ------ | ----- | --- | +| `[[table]]` with `ddl` + `glob` + `on-file` | Load fails: `Missing required field 'name' in [[table]] entry` | Add `name = "
"`, matching the table the `ddl` creates. | +| `Table::new(ddl, glob, on_file)` | Does not compile | `Table::new(name, ddl, glob, on_file)`; same for `try_new` / `strict`. | +| `Db::create_table(ddl)` | Does not compile | `Db::create_table(name, ddl)`. | +| `dirsql::db::parse_table_name(&ddl)` | Does not compile | Use the name you declared. The catalog (`pragma_table_list` / `sqlite_master`) answers "does this table exist". | +| `DirSqlError::Ddl` returned for a DDL whose name could not be parsed | No longer produced for that case | Match `DirSqlError::Core(_)` for a DDL SQLite rejects, and the new `DirSqlError::TableNotCreated { name }` for a `name` the DDL never creates. | + +#### Deprecations removed + +- `dirsql::db::parse_table_name` — removed outright, with the + `strip_keyword_ci` / `parse_identifier` / `skip_ws_comments` tokenizer + helpers behind it. + +#### Behavior changes without code changes + +- A `[[table]]` whose `ddl` creates a table under a different name than its + `name` is now a hard load error (`table 'messages': its `ddl` ran but created + no table called 'messages'. Set `name` to the table the `ddl` creates.`). + Previously the parsed name always agreed with the DDL by construction, so + this mismatch could not arise. +- Duplicate-table detection across composed configs compares the declared + `name` keys rather than parsed DDL names. A config whose DDL was previously + unparseable took part in no collision check; every entry now does. +- Quoted (`CREATE TABLE "messages"`), schema-qualified (`main.messages`) and + `IF NOT EXISTS` DDL keep working unchanged — SQLite records the bare + identifier in its catalog, so a plain `name = "messages"` matches. + +#### Verification + +```bash +# .dirsql.toml +# [[table]] +# name = "records" +# ddl = "CREATE TABLE records (id TEXT)" +# glob = "data/*.json" +# on-file = "cat {path}" + +dirsql query "SELECT id FROM records" -c ./.dirsql.toml +# -> [{"id":"one"},{"id":"two"}] + +# Drop the `name` key: +dirsql query "SELECT id FROM records" -c ./.dirsql.toml +# -> error: Missing required field 'name' in [[table]] entry + +# Set name = "messages" while the ddl still creates `records`: +dirsql query "SELECT id FROM messages" -c ./.dirsql.toml +# -> error: table 'messages': its `ddl` ran but created no table called 'messages'. +``` diff --git a/packages/rust/src/cli/run.rs b/packages/rust/src/cli/run.rs index 7ad898b8e..38d23c58e 100644 --- a/packages/rust/src/cli/run.rs +++ b/packages/rust/src/cli/run.rs @@ -556,6 +556,7 @@ fn default_records_table() -> Table { .expect("DEFAULT_CONFIG_TOML must be valid dirsql config TOML"); let table_config = &config.tables[0]; Table::new( + table_config.name.clone(), table_config.ddl.clone(), table_config.glob.clone(), |_path| vec![Row::new()], diff --git a/packages/rust/src/config.rs b/packages/rust/src/config.rs index 9c0b1356e..3d650c6a2 100644 --- a/packages/rust/src/config.rs +++ b/packages/rust/src/config.rs @@ -109,9 +109,6 @@ impl std::fmt::Display for Source { /// `[[dirsql.function]]`, `ignore`) concatenates in input order. A table-name /// or function-name collision anywhere in the combined set errors, naming both /// sources. -/// -/// Tables whose DDL yields no parseable table name are concatenated without a -/// collision check; `Db::create_table` rejects them downstream. pub fn combine_configs(configs: &[(Source, Config)]) -> Result { let (first, rest) = configs.split_first().ok_or(ConfigError::NoConfigs)?; if rest.is_empty() { @@ -129,11 +126,9 @@ pub fn combine_configs(configs: &[(Source, Config)]) -> Result { for (source, config) in configs { for table in &config.tables { - if let Some(name) = crate::db::parse_table_name(&table.ddl) - && let Some(prior) = table_sources.insert(name.clone(), source) - { + if let Some(prior) = table_sources.insert(table.name.clone(), source) { return Err(ConfigError::DuplicateTable { - name, + name: table.name.clone(), first: prior.clone(), second: source.clone(), }); @@ -230,6 +225,10 @@ pub struct FunctionSpec { /// [`crate::Table`] with your own on-file callback. #[derive(Debug, Clone)] pub struct TableConfig { + /// The table's SQL name, declared rather than derived. `ddl` must create a + /// table by this name; the mismatch is caught against SQLite's catalog at + /// load time. + pub name: String, pub ddl: String, pub glob: String, pub strict: Option, @@ -289,6 +288,7 @@ struct RawExtension { #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct RawTable { + name: Option, ddl: Option, glob: Option, strict: Option, @@ -336,6 +336,13 @@ pub fn load_config_str(content: &str) -> Result { let mut tables = Vec::with_capacity(raw_tables.len()); for raw_table in raw_tables { + let name = match raw_table.name { + Some(name) if name.trim().is_empty() => { + return Err(ConfigError::EmptyField("name")); + } + Some(name) => name, + None => return Err(ConfigError::MissingField("name")), + }; let ddl = raw_table.ddl.ok_or(ConfigError::MissingField("ddl"))?; let glob = raw_table.glob.ok_or(ConfigError::MissingField("glob"))?; @@ -348,6 +355,7 @@ pub fn load_config_str(content: &str) -> Result { }; tables.push(TableConfig { + name, ddl, glob, strict: raw_table.strict, @@ -470,11 +478,13 @@ mod tests { ignore = ["node_modules/**", ".git/**"] [[table]] +name = "comments" ddl = "CREATE TABLE comments (thread_id TEXT, path TEXT)" glob = "_comments/{thread_id}/index.jsonl" on-file = "cat {path}" [[table]] +name = "items" ddl = "CREATE TABLE items (path TEXT, size INTEGER)" glob = "catalog/*.json" on-file = "cat {path}" @@ -493,10 +503,58 @@ strict = true assert_eq!(t1.strict, Some(true)); } + #[test] + fn missing_name_returns_error() { + let toml = r#" +[[table]] +ddl = "CREATE TABLE t (x TEXT)" +glob = "*.json" +on-file = "cat {path}" +"#; + let err = load_config_str(toml).unwrap_err(); + assert!( + matches!(err, ConfigError::MissingField("name")), + "got: {err:?}" + ); + assert!(err.to_string().contains("name"), "got: {err}"); + } + + #[test] + fn blank_name_returns_error() { + let toml = r#" +[[table]] +name = " " +ddl = "CREATE TABLE t (x TEXT)" +glob = "*.json" +on-file = "cat {path}" +"#; + let err = load_config_str(toml).unwrap_err(); + assert!( + matches!(err, ConfigError::EmptyField("name")), + "got: {err:?}" + ); + } + + #[test] + fn name_is_kept_verbatim_and_does_not_come_from_the_ddl() { + // The two are independent inputs at parse time; whether they agree is + // settled later against SQLite's catalog, not here. + let toml = r#" +[[table]] +name = "messages" +ddl = "CREATE TABLE notes (x TEXT)" +glob = "*.json" +on-file = "cat {path}" +"#; + let config = load_config_str(toml).unwrap(); + assert_eq!(config.tables[0].name, "messages"); + } + #[test] fn missing_ddl_returns_error() { let toml = r#" [[table]] +name = "t" glob = "*.json" on-file = "cat {path}" "#; @@ -508,6 +566,7 @@ on-file = "cat {path}" fn missing_glob_returns_error() { let toml = r#" [[table]] +name = "t" ddl = "CREATE TABLE t (x TEXT)" "#; let err = load_config_str(toml).unwrap_err(); @@ -544,6 +603,7 @@ ignore = ["*.tmp"] fn no_dirsql_section_defaults_to_empty_ignore() { let toml = r#" [[table]] +name = "t" ddl = "CREATE TABLE t (x TEXT)" glob = "*.json" on-file = "cat {path}" @@ -585,16 +645,19 @@ persist_path = "/var/cache/dirsql.db" fn multiple_tables_preserve_order() { let toml = r#" [[table]] +name = "a" ddl = "CREATE TABLE a (path TEXT)" glob = "a/*.json" on-file = "cat {path}" [[table]] +name = "b" ddl = "CREATE TABLE b (path TEXT)" glob = "b/*.csv" on-file = "cat {path}" [[table]] +name = "c" ddl = "CREATE TABLE c (path TEXT)" glob = "c/*.yaml" on-file = "cat {path}" @@ -612,6 +675,7 @@ on-file = "cat {path}" // is a hard parse error naming the key, never silently dropped. let toml = r#" [[table]] +name = "t" ddl = "CREATE TABLE t (path TEXT)" glob = "*.json" on-file = "cat {path}" @@ -679,6 +743,7 @@ path = "./ext/vec0.dylib" entrypoint = "sqlite3_vec_init" [[table]] +name = "t" ddl = "CREATE TABLE t (path TEXT)" glob = "*.json" on-file = "cat {path}" @@ -720,6 +785,7 @@ entrypoint = "sqlite3_x_init" fn extensions_default_empty_when_absent() { let toml = r#" [[table]] +name = "t" ddl = "CREATE TABLE t (path TEXT)" glob = "*.json" on-file = "cat {path}" @@ -747,6 +813,7 @@ path = "b.so" fn on_file_parses_when_present() { let toml = r#" [[table]] +name = "papers" ddl = "CREATE TABLE papers (paper_id TEXT, title TEXT)" glob = "**/meta.json" on-file = "uv run python extract_papers.py {path}" @@ -763,6 +830,7 @@ on-file = "uv run python extract_papers.py {path}" fn on_file_absent_is_a_hookless_load_error() { let toml = r#" [[table]] +name = "t" ddl = "CREATE TABLE t (path TEXT)" glob = "*.json" "#; @@ -780,6 +848,7 @@ glob = "*.json" fn on_file_empty_errors() { let toml = r#" [[table]] +name = "t" ddl = "CREATE TABLE t (path TEXT)" glob = "*.json" on-file = " " @@ -841,6 +910,7 @@ path = "vec0.so" entrypoint = "sqlite3_vec_init" [[table]] +name = "t" ddl = "CREATE TABLE t (path TEXT)" glob = "*.json" on-file = "cat {path}" @@ -857,17 +927,20 @@ on-file = "cat {path}" fn combine_concatenates_tables_in_input_order() { let a = cfg(r#" [[table]] +name = "a" ddl = "CREATE TABLE a (path TEXT)" glob = "a/*.json" on-file = "cat {path}" [[table]] +name = "b" ddl = "CREATE TABLE b (path TEXT)" glob = "b/*.json" on-file = "cat {path}" "#); let b = cfg(r#" [[table]] +name = "c" ddl = "CREATE TABLE c (path TEXT)" glob = "c/*.json" on-file = "cat {path}" @@ -916,10 +989,10 @@ on-file = "cat {path}" #[test] fn combine_duplicate_table_name_errors_naming_both_sources() { let a = cfg( - "[[table]]\nddl = \"CREATE TABLE t (x TEXT)\"\nglob = \"a/*.json\"\non-file = \"cat {path}\"\n", + "[[table]]\nname = \"t\"\nddl = \"CREATE TABLE t (x TEXT)\"\nglob = \"a/*.json\"\non-file = \"cat {path}\"\n", ); let b = cfg( - "[[table]]\nddl = \"CREATE TABLE t (y TEXT)\"\nglob = \"b/*.json\"\non-file = \"cat {path}\"\n", + "[[table]]\nname = \"t\"\nddl = \"CREATE TABLE t (y TEXT)\"\nglob = \"b/*.json\"\non-file = \"cat {path}\"\n", ); let err = combine_configs(&[ (src("/proj/.dirsql.toml"), a), @@ -949,8 +1022,8 @@ on-file = "cat {path}" // The single-entry identity path runs no collision check, exactly like // a plain single-config load. let config = cfg(concat!( - "[[table]]\nddl = \"CREATE TABLE t (x TEXT)\"\nglob = \"a/*.json\"\non-file = \"cat {path}\"\n", - "[[table]]\nddl = \"CREATE TABLE t (y TEXT)\"\nglob = \"b/*.json\"\non-file = \"cat {path}\"\n", + "[[table]]\nname = \"t\"\nddl = \"CREATE TABLE t (x TEXT)\"\nglob = \"a/*.json\"\non-file = \"cat {path}\"\n", + "[[table]]\nname = \"t\"\nddl = \"CREATE TABLE t (y TEXT)\"\nglob = \"b/*.json\"\non-file = \"cat {path}\"\n", )); let merged = combine_configs(&[(src("/a"), config)]).unwrap(); assert_eq!(merged.tables.len(), 2); @@ -959,8 +1032,8 @@ on-file = "cat {path}" #[test] fn combine_intra_config_duplicate_in_multi_merge_errors() { let a = cfg(concat!( - "[[table]]\nddl = \"CREATE TABLE t (x TEXT)\"\nglob = \"a/*.json\"\non-file = \"cat {path}\"\n", - "[[table]]\nddl = \"CREATE TABLE t (y TEXT)\"\nglob = \"b/*.json\"\non-file = \"cat {path}\"\n", + "[[table]]\nname = \"t\"\nddl = \"CREATE TABLE t (x TEXT)\"\nglob = \"a/*.json\"\non-file = \"cat {path}\"\n", + "[[table]]\nname = \"t\"\nddl = \"CREATE TABLE t (y TEXT)\"\nglob = \"b/*.json\"\non-file = \"cat {path}\"\n", )); let b = cfg("[dirsql]\nignore = [\"c/**\"]\n"); let err = combine_configs(&[(src("/a"), a), (src("/b"), b)]).unwrap_err(); @@ -981,10 +1054,10 @@ on-file = "cat {path}" #[test] fn combine_duplicate_table_name_detected_through_quoting() { let a = cfg( - "[[table]]\nddl = 'CREATE TABLE \"t\" (x TEXT)'\nglob = \"a/*.json\"\non-file = \"cat {path}\"\n", + "[[table]]\nname = 't'\nddl = 'CREATE TABLE \"t\" (x TEXT)'\nglob = \"a/*.json\"\non-file = \"cat {path}\"\n", ); let b = cfg( - "[[table]]\nddl = \"CREATE TABLE t (y TEXT)\"\nglob = \"b/*.json\"\non-file = \"cat {path}\"\n", + "[[table]]\nname = \"t\"\nddl = \"CREATE TABLE t (y TEXT)\"\nglob = \"b/*.json\"\non-file = \"cat {path}\"\n", ); let err = combine_configs(&[(src("/a"), a), (src("/b"), b)]).unwrap_err(); assert!( @@ -993,25 +1066,13 @@ on-file = "cat {path}" ); } - #[test] - fn combine_unparseable_ddl_concatenates_without_collision_check() { - let a = cfg( - "[[table]]\nddl = \"not a create table\"\nglob = \"a/*.json\"\non-file = \"cat {path}\"\n", - ); - let b = cfg( - "[[table]]\nddl = \"also not a create table\"\nglob = \"b/*.json\"\non-file = \"cat {path}\"\n", - ); - let merged = combine_configs(&[(src("/a"), a), (src("/b"), b)]).unwrap(); - assert_eq!(merged.tables.len(), 2); - } - #[test] fn combine_error_display_includes_package_source_verbatim() { let a = cfg( - "[[table]]\nddl = \"CREATE TABLE t (x TEXT)\"\nglob = \"a/*.json\"\non-file = \"cat {path}\"\n", + "[[table]]\nname = \"t\"\nddl = \"CREATE TABLE t (x TEXT)\"\nglob = \"a/*.json\"\non-file = \"cat {path}\"\n", ); let b = cfg( - "[[table]]\nddl = \"CREATE TABLE t (y TEXT)\"\nglob = \"b/*.json\"\non-file = \"cat {path}\"\n", + "[[table]]\nname = \"t\"\nddl = \"CREATE TABLE t (y TEXT)\"\nglob = \"b/*.json\"\non-file = \"cat {path}\"\n", ); let err = combine_configs(&[ (src("/proj/.dirsql.toml"), a), @@ -1026,13 +1087,13 @@ on-file = "cat {path}" #[test] fn combine_three_configs_concatenates_across_all() { let a = cfg( - "[dirsql]\nignore = [\"a/**\"]\n\n[[table]]\nddl = \"CREATE TABLE a (x TEXT)\"\nglob = \"a/*\"\non-file = \"cat {path}\"\n", + "[dirsql]\nignore = [\"a/**\"]\n\n[[table]]\nname = \"a\"\nddl = \"CREATE TABLE a (x TEXT)\"\nglob = \"a/*\"\non-file = \"cat {path}\"\n", ); let b = cfg( - "[[table]]\nddl = \"CREATE TABLE b (x TEXT)\"\nglob = \"b/*\"\non-file = \"cat {path}\"\n", + "[[table]]\nname = \"b\"\nddl = \"CREATE TABLE b (x TEXT)\"\nglob = \"b/*\"\non-file = \"cat {path}\"\n", ); let c = cfg( - "[dirsql]\nignore = [\"c/**\"]\n\n[[table]]\nddl = \"CREATE TABLE c (x TEXT)\"\nglob = \"c/*\"\non-file = \"cat {path}\"\n", + "[dirsql]\nignore = [\"c/**\"]\n\n[[table]]\nname = \"c\"\nddl = \"CREATE TABLE c (x TEXT)\"\nglob = \"c/*\"\non-file = \"cat {path}\"\n", ); let merged = combine_configs(&[(src("/a"), a), (src("/b"), b), (src("/c"), c)]).unwrap(); assert_eq!(merged.ignore, vec!["a/**", "c/**"]); diff --git a/packages/rust/src/db.rs b/packages/rust/src/db.rs index ed1dbd302..9a11d68a9 100644 --- a/packages/rust/src/db.rs +++ b/packages/rust/src/db.rs @@ -368,11 +368,13 @@ impl Db { /// entirely in `_dirsql_internal_rows`, so a table's schema is exactly /// the DDL the user wrote. /// - /// Validates that the parsed table name is a safe unquoted SQL identifier - /// before handing the DDL to SQLite, so a DDL like - /// `CREATE TABLE foo;DROP_TABLE_bar--(id TEXT)` can't yield a poisoned - /// table name that breaks downstream `format!()`-built SQL. - pub fn create_table(&self, ddl: &str) -> Result<()> { + /// `name` is the table's declared name, validated as a safe unquoted SQL + /// identifier before the DDL reaches SQLite: it is spliced into + /// `format!()`-built INSERT/DELETE SQL downstream, so a poisoned name must + /// never get that far. Whether the DDL actually creates a table by that + /// name is settled afterwards against SQLite's catalog, not by reading the + /// DDL text. + pub fn create_table(&self, name: &str, ddl: &str) -> Result<()> { // A dirsql table is a per-file row table; an extension-backed virtual // table is not one, so reject `CREATE VIRTUAL TABLE` with a clear // message. Load the extension and use its functions in queries instead. @@ -384,14 +386,13 @@ impl Db { .to_string(), )); } - let table = parse_table_name(ddl).ok_or_else(|| DbError::DdlParse(ddl.to_string()))?; - validate_identifier(&table)?; + validate_identifier(name)?; // A `WITHOUT ROWID` table has no rowid, so `last_insert_rowid()` cannot // identify the inserted row and the `_dirsql_internal_rows.rowid_ref` // mapping would be meaningless. Warn; the table is still created. if is_without_rowid_ddl(ddl) { eprintln!( - "dirsql: table `{table}` is declared WITHOUT ROWID; internal row \ + "dirsql: table `{name}` is declared WITHOUT ROWID; internal row \ bookkeeping relies on rowid and WITHOUT ROWID tables will be \ rejected in a future release" ); @@ -839,136 +840,6 @@ impl rusqlite::types::ToSql for Value { } } -/// Extract the table name from a `CREATE TABLE` DDL statement. -/// -/// The name token in `CREATE TABLE [IF NOT EXISTS] (...)` may be: -/// - a bare identifier: `comments` -/// - a quoted identifier: `"comments"`, `` `comments` ``, `[comments]` -/// - schema-qualified: `main.comments`, `"main"."comments"` -/// -/// Surrounding quotes are SQL *delimiters*, not part of the name, so they are -/// stripped: a tool that emits `CREATE TABLE "comments" (...)` (as ORMs and -/// schema generators routinely do) names the table `comments`. Schema-qualified -/// names resolve to the table segment. Returns `None` when the input isn't a -/// `CREATE TABLE` or carries no name token. -/// -/// Deliberately a small, pure tokenizer rather than a full SQL parser: -/// dirsql constrains table names to safe unquoted identifiers via -/// [`validate_identifier`], so the handful of forms above are the only ones -/// that can actually resolve to a usable table. -/// Strip leading whitespace and SQL comments (`-- ...` to end-of-line and -/// `/* ... */` blocks, repeated) from `s`. An unterminated block comment -/// consumes the rest of the input. Every returned slice starts on a char -/// boundary, so callers can index it safely. -fn skip_ws_comments(s: &str) -> &str { - let mut s = s; - loop { - let trimmed = s.trim_start(); - if let Some(after) = trimmed.strip_prefix("--") { - match after.find('\n') { - Some(i) => s = &after[i + 1..], - None => return "", - } - } else if let Some(after) = trimmed.strip_prefix("/*") { - match after.find("*/") { - Some(i) => s = &after[i + 2..], - None => return "", - } - } else { - return trimmed; - } - } -} - -/// If `s` begins with the ASCII keyword `kw` (case-insensitive) followed by a -/// non-identifier boundary, return the remainder after it; otherwise `None`. -/// The boundary check keeps a longer identifier (`TABLES`, `iffy`) from -/// matching a keyword prefix. -fn strip_keyword_ci<'a>(s: &'a str, kw: &str) -> Option<&'a str> { - let bytes = s.as_bytes(); - if bytes.len() < kw.len() || !bytes[..kw.len()].eq_ignore_ascii_case(kw.as_bytes()) { - return None; - } - let rest = &s[kw.len()..]; - match rest.chars().next() { - Some(c) if c.is_alphanumeric() || c == '_' => None, - _ => Some(rest), - } -} - -pub fn parse_table_name(ddl: &str) -> Option { - // Match `CREATE TABLE` only as the *statement head*: skip leading - // whitespace and line/block comments and scan the original `ddl` by byte - // offset (never a `to_uppercase()` copy, whose length can differ from the - // source and mis-slice on Unicode). A `-- CREATE TABLE ...` comment must - // not hijack the name. - let rest = skip_ws_comments(ddl); - let rest = strip_keyword_ci(rest, "CREATE")?; - let rest = strip_keyword_ci(skip_ws_comments(rest), "TABLE")?; - let mut rest = skip_ws_comments(rest); - - // Skip optional "IF NOT EXISTS". The keyword boundary check in - // `strip_keyword_ci` keeps a table literally named e.g. `iffy` from being - // mistaken for `IF`. - if let Some(after_if) = strip_keyword_ci(rest, "IF") { - let after_not = strip_keyword_ci(skip_ws_comments(after_if), "NOT")?; - let after_exists = strip_keyword_ci(skip_ws_comments(after_not), "EXISTS")?; - rest = skip_ws_comments(after_exists); - } - - // `schema.table`: keep the last dot-separated segment. Each segment may - // be quoted independently. - let mut chars = rest.chars().peekable(); - let mut table = parse_identifier(&mut chars)?; - while chars.peek() == Some(&'.') { - chars.next(); - table = parse_identifier(&mut chars)?; - } - - if table.is_empty() { None } else { Some(table) } -} - -/// Read one identifier segment from `chars`: a quoted identifier -/// (`"..."`, `` `...` ``, or `[...]`, with the doubled-delimiter escape for the -/// matching close on the SQL-standard quote forms) or a bare run up to the -/// first whitespace, `(`, or `.`. Returns the *unquoted* text. Returns `None` -/// only when the cursor is already exhausted. -fn parse_identifier(chars: &mut std::iter::Peekable>) -> Option { - let (open, close) = match *chars.peek()? { - '"' => ('"', '"'), - '`' => ('`', '`'), - '[' => ('[', ']'), - _ => { - let mut bare = String::new(); - while let Some(&c) = chars.peek() { - if c.is_whitespace() || c == '(' || c == '.' { - break; - } - bare.push(c); - chars.next(); - } - return Some(bare); - } - }; - - chars.next(); - let mut name = String::new(); - while let Some(c) = chars.next() { - if c == close { - // On `"` / `` ` ``, a doubled close delimiter is an escaped literal - // (`""` -> `"`); SQLite's `[...]` form has no such escape. - if open != '[' && chars.peek() == Some(&close) { - chars.next(); - name.push(close); - continue; - } - break; - } - name.push(c); - } - Some(name) -} - /// True if `ddl` is a `CREATE VIRTUAL TABLE` statement. dirsql tables are /// per-file row tables, structurally incompatible with an extension-backed /// virtual table, so those are rejected with a clear error. @@ -1005,8 +876,11 @@ mod tests { #[test] fn create_table_from_ddl() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE comments (id TEXT PRIMARY KEY, body TEXT, resolved INTEGER)") - .unwrap(); + db.create_table( + "comments", + "CREATE TABLE comments (id TEXT PRIMARY KEY, body TEXT, resolved INTEGER)", + ) + .unwrap(); let rows = db.query("SELECT * FROM comments").unwrap(); assert_eq!(rows.len(), 0); @@ -1015,14 +889,14 @@ mod tests { #[test] fn create_table_invalid_ddl_returns_error() { let db = Db::new().unwrap(); - let result = db.create_table("NOT VALID SQL"); + let result = db.create_table("t", "NOT VALID SQL"); assert!(result.is_err()); } #[test] fn create_table_runs_ddl_verbatim_no_injected_columns() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT)").unwrap(); db.insert_row( "t", &HashMap::from([("id".into(), Value::Text("1".into()))]), @@ -1043,7 +917,7 @@ mod tests { #[test] fn pragma_table_info_matches_user_ddl_exactly() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE posts (title TEXT, draft INTEGER)") + db.create_table("posts", "CREATE TABLE posts (title TEXT, draft INTEGER)") .unwrap(); assert_eq!( db.get_table_columns("posts").unwrap(), @@ -1054,7 +928,7 @@ mod tests { #[test] fn insert_and_query_rows() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE docs (title TEXT, draft INTEGER)") + db.create_table("docs", "CREATE TABLE docs (title TEXT, draft INTEGER)") .unwrap(); let row = HashMap::from([ @@ -1072,7 +946,7 @@ mod tests { #[test] fn insert_multiple_rows_from_same_file() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE events (action TEXT, ts INTEGER)") + db.create_table("events", "CREATE TABLE events (action TEXT, ts INTEGER)") .unwrap(); for (i, action) in ["created", "resolved", "reopened"].iter().enumerate() { @@ -1092,7 +966,7 @@ mod tests { #[test] fn delete_rows_by_file_path() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE comments (id TEXT, body TEXT)") + db.create_table("comments", "CREATE TABLE comments (id TEXT, body TEXT)") .unwrap(); for (i, (id, file)) in [("1", "a.jsonl"), ("2", "a.jsonl"), ("3", "b.jsonl")] @@ -1117,7 +991,7 @@ mod tests { #[test] fn query_with_where_clause() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE items (name TEXT, count INTEGER)") + db.create_table("items", "CREATE TABLE items (name TEXT, count INTEGER)") .unwrap(); for (i, (name, count)) in [("apple", 5), ("banana", 0), ("cherry", 3)] @@ -1139,175 +1013,10 @@ mod tests { assert_eq!(results[1]["name"], Value::Text("cherry".into())); } - #[test] - fn parse_table_name_simple() { - assert_eq!( - parse_table_name("CREATE TABLE comments (id TEXT)"), - Some("comments".to_string()) - ); - } - - #[test] - fn parse_table_name_if_not_exists() { - assert_eq!( - parse_table_name("CREATE TABLE IF NOT EXISTS comments (id TEXT)"), - Some("comments".to_string()) - ); - } - - #[test] - fn parse_table_name_no_space_before_paren() { - assert_eq!( - parse_table_name("CREATE TABLE t(id TEXT)"), - Some("t".to_string()) - ); - } - - #[test] - fn parse_table_name_invalid() { - assert_eq!(parse_table_name("NOT A DDL"), None); - } - - #[test] - fn parse_table_name_empty_after_create_table() { - assert_eq!(parse_table_name("CREATE TABLE "), None); - } - - #[test] - fn parse_table_name_double_quoted() { - assert_eq!( - parse_table_name(r#"CREATE TABLE "comments" (id TEXT)"#), - Some("comments".to_string()) - ); - } - - #[test] - fn parse_table_name_double_quoted_no_space_before_paren() { - assert_eq!( - parse_table_name(r#"CREATE TABLE "comments"(id TEXT)"#), - Some("comments".to_string()) - ); - } - - #[test] - fn parse_table_name_backtick_quoted() { - assert_eq!( - parse_table_name("CREATE TABLE `comments` (id TEXT)"), - Some("comments".to_string()) - ); - } - - #[test] - fn parse_table_name_bracket_quoted() { - assert_eq!( - parse_table_name("CREATE TABLE [comments] (id TEXT)"), - Some("comments".to_string()) - ); - } - - #[test] - fn parse_table_name_double_quote_escape() { - // `""` inside a double-quoted identifier is one literal `"`. - assert_eq!( - parse_table_name(r#"CREATE TABLE "we""ird" (id TEXT)"#), - Some("we\"ird".to_string()) - ); - } - - #[test] - fn parse_table_name_if_not_exists_quoted_mixed_case() { - assert_eq!( - parse_table_name(r#"CREATE TABLE if not exists "comments" (id TEXT)"#), - Some("comments".to_string()) - ); - } - - #[test] - fn parse_table_name_schema_qualified() { - assert_eq!( - parse_table_name("CREATE TABLE main.comments (id TEXT)"), - Some("comments".to_string()) - ); - } - - #[test] - fn parse_table_name_schema_qualified_quoted() { - assert_eq!( - parse_table_name(r#"CREATE TABLE "main"."comments" (id TEXT)"#), - Some("comments".to_string()) - ); - } - - #[test] - fn parse_table_name_runs_to_end_of_input() { - assert_eq!( - parse_table_name("CREATE TABLE comments"), - Some("comments".to_string()) - ); - } - - #[test] - fn parse_table_name_missing_name_is_none() { - assert_eq!(parse_table_name("CREATE TABLE (id TEXT)"), None); - } - - #[test] - fn parse_table_name_ignores_leading_line_comment() { - assert_eq!( - parse_table_name("-- create table old\nCREATE TABLE t (x TEXT)"), - Some("t".to_string()) - ); - } - - #[test] - fn parse_table_name_ignores_leading_block_comment() { - assert_eq!( - parse_table_name("/* CREATE TABLE old */ CREATE TABLE t (x TEXT)"), - Some("t".to_string()) - ); - } - - #[test] - fn parse_table_name_unterminated_comment_is_none() { - // Unterminated `--` and `/* */` comments consume the rest of the input, - // leaving no statement head. - assert_eq!(parse_table_name("-- no newline, no statement"), None); - assert_eq!(parse_table_name("/* unterminated CREATE TABLE t"), None); - } - - #[test] - fn parse_table_name_rejects_keyword_prefix() { - // `CREATE TABLES` must not match the `TABLE` keyword prefix. - assert_eq!(parse_table_name("CREATE TABLES foo (id TEXT)"), None); - } - - #[test] - fn parse_table_name_if_prefixed_name_is_not_if_not_exists() { - // A table whose name merely starts with `if` is not `IF NOT EXISTS`. - assert_eq!( - parse_table_name("CREATE TABLE ifx (id TEXT)"), - Some("ifx".to_string()) - ); - } - - #[test] - fn parse_table_name_unicode_in_comment_does_not_panic() { - // A case-length-changing char (`fi`) in a comment must neither hijack - // the name nor panic on a byte-index slice. - assert_eq!( - parse_table_name("-- fi\nCREATE TABLE t (x TEXT)"), - Some("t".to_string()) - ); - assert_eq!( - parse_table_name("/* fififi */ CREATE TABLE t (x TEXT)"), - Some("t".to_string()) - ); - } - #[test] fn get_table_columns_returns_user_columns_only() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (name TEXT, count INTEGER)") + db.create_table("t", "CREATE TABLE t (name TEXT, count INTEGER)") .unwrap(); let cols = db.get_table_columns("t").unwrap(); assert!(cols.contains(&"name".to_string())); @@ -1318,7 +1027,7 @@ mod tests { #[test] fn normalize_row_relaxed_drops_extra_keys() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (name TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (name TEXT)").unwrap(); let row = HashMap::from([ ("name".into(), Value::Text("apple".into())), ("color".into(), Value::Text("red".into())), @@ -1332,7 +1041,7 @@ mod tests { #[test] fn normalize_row_relaxed_fills_missing_with_null() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (name TEXT, color TEXT)") + db.create_table("t", "CREATE TABLE t (name TEXT, color TEXT)") .unwrap(); let row = HashMap::from([("name".into(), Value::Text("apple".into()))]); let normalized = db.normalize_row("t", &row, false).unwrap(); @@ -1344,7 +1053,7 @@ mod tests { #[test] fn normalize_row_strict_errors_on_extra_keys() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (name TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (name TEXT)").unwrap(); let row = HashMap::from([ ("name".into(), Value::Text("apple".into())), ("color".into(), Value::Text("red".into())), @@ -1358,7 +1067,7 @@ mod tests { #[test] fn normalize_row_strict_errors_on_missing_keys() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (name TEXT, color TEXT)") + db.create_table("t", "CREATE TABLE t (name TEXT, color TEXT)") .unwrap(); let row = HashMap::from([("name".into(), Value::Text("apple".into()))]); let result = db.normalize_row("t", &row, true); @@ -1370,7 +1079,7 @@ mod tests { #[test] fn normalize_row_strict_accepts_exact_match() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (name TEXT, color TEXT)") + db.create_table("t", "CREATE TABLE t (name TEXT, color TEXT)") .unwrap(); let row = HashMap::from([ ("name".into(), Value::Text("apple".into())), @@ -1383,7 +1092,7 @@ mod tests { #[test] fn insert_and_query_real_value() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (price REAL)").unwrap(); + db.create_table("t", "CREATE TABLE t (price REAL)").unwrap(); let row = HashMap::from([("price".into(), Value::Real(9.99))]); db.insert_row("t", &row, "test.json", 0).unwrap(); let results = db.query("SELECT price FROM t").unwrap(); @@ -1394,7 +1103,7 @@ mod tests { #[test] fn insert_and_query_null_value() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (name TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (name TEXT)").unwrap(); let row = HashMap::from([("name".into(), Value::Null)]); db.insert_row("t", &row, "test.json", 0).unwrap(); let results = db.query("SELECT name FROM t").unwrap(); @@ -1405,7 +1114,7 @@ mod tests { #[test] fn insert_and_query_blob_value() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (data BLOB)").unwrap(); + db.create_table("t", "CREATE TABLE t (data BLOB)").unwrap(); let row = HashMap::from([("data".into(), Value::Blob(vec![0xFF, 0x00]))]); db.insert_row("t", &row, "test.json", 0).unwrap(); let results = db.query("SELECT data FROM t").unwrap(); @@ -1416,7 +1125,7 @@ mod tests { #[test] fn select_star_returns_only_user_columns() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT)").unwrap(); let row = HashMap::from([("id".into(), Value::Text("1".into()))]); db.insert_row("t", &row, "file.json", 0).unwrap(); let results = db.query("SELECT * FROM t").unwrap(); @@ -1427,7 +1136,7 @@ mod tests { #[test] fn dirsql_columns_no_longer_exist_on_user_tables() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT)").unwrap(); let row = HashMap::from([("id".into(), Value::Text("1".into()))]); db.insert_row("t", &row, "file.json", 0).unwrap(); @@ -1476,7 +1185,7 @@ mod tests { #[test] fn query_allows_pragma_on_a_user_table() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT)").unwrap(); let rows = db.query("PRAGMA table_info(t)").unwrap(); assert_eq!(rows.len(), 1, "expected one column row, got {rows:?}"); assert_eq!(rows[0]["name"], Value::Text("id".into())); @@ -1487,7 +1196,7 @@ mod tests { // `delete_rows_by_file` reads `_dirsql_internal_rows` without routing // through query(), so a leaked authorizer would make it fail. let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT)").unwrap(); let row = HashMap::from([("id".into(), Value::Text("1".into()))]); db.insert_row("t", &row, "file.json", 0).unwrap(); @@ -1542,7 +1251,7 @@ mod tests { // The authorizer is cleared after each query, so a denied ATTACH must // not poison the next normal read. let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT)").unwrap(); let _ = db.query("ATTACH 'x.db' AS ext").unwrap_err(); let rows = db.query("SELECT * FROM t").unwrap(); assert_eq!(rows.len(), 0); @@ -1589,7 +1298,7 @@ mod tests { #[test] fn delete_rows_by_file_returns_zero_for_no_matching_rows() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT)").unwrap(); let row = HashMap::from([("id".into(), Value::Text("1".into()))]); db.insert_row("t", &row, "a.json", 0).unwrap(); let deleted = db.delete_rows_by_file("t", "nonexistent.json").unwrap(); @@ -1612,7 +1321,7 @@ mod tests { #[test] fn get_rows_by_file_returns_rows_in_row_index_order() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT)").unwrap(); // Row indices are deliberately out of insertion order. let second = HashMap::from([("id".into(), Value::Text("second".into()))]); let first = HashMap::from([("id".into(), Value::Text("first".into()))]); @@ -1628,8 +1337,8 @@ mod tests { #[test] fn get_rows_by_file_scopes_by_file_and_table() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT)").unwrap(); - db.create_table("CREATE TABLE u (id TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT)").unwrap(); + db.create_table("u", "CREATE TABLE u (id TEXT)").unwrap(); let t_row = HashMap::from([("id".into(), Value::Text("t-row".into()))]); let u_row = HashMap::from([("id".into(), Value::Text("u-row".into()))]); let other = HashMap::from([("id".into(), Value::Text("other-file".into()))]); @@ -1648,8 +1357,11 @@ mod tests { #[test] fn get_rows_by_file_round_trips_all_value_variants() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (i INTEGER, r REAL, s TEXT, b BLOB, n TEXT)") - .unwrap(); + db.create_table( + "t", + "CREATE TABLE t (i INTEGER, r REAL, s TEXT, b BLOB, n TEXT)", + ) + .unwrap(); let row = HashMap::from([ ("i".to_string(), Value::Integer(42)), ("r".to_string(), Value::Real(1.5)), @@ -1682,12 +1394,12 @@ mod tests { } #[test] - fn create_table_without_parseable_name_fails_at_parse() { + fn create_table_rejects_an_unsafe_declared_name() { let db = Db::new().unwrap(); let err = db - .create_table("CREATE TABLE (this is not valid)") + .create_table("t; DROP TABLE u; --", "CREATE TABLE t (id TEXT)") .unwrap_err(); - assert!(matches!(err, DbError::DdlParse(_)), "got: {err}"); + assert!(matches!(err, DbError::InvalidIdentifier(_)), "got: {err}"); } #[test] @@ -1737,7 +1449,10 @@ mod tests { fn create_table_rejects_ddl_with_sql_syntax_in_name_slot() { let db = Db::new().unwrap(); let err = db - .create_table("CREATE TABLE evil;DROP_TABLE--(id TEXT)") + .create_table( + "evil;DROP_TABLE--", + "CREATE TABLE evil;DROP_TABLE--(id TEXT)", + ) .unwrap_err(); assert!(matches!(err, DbError::InvalidIdentifier(_)), "got: {err:?}"); } @@ -1745,7 +1460,7 @@ mod tests { #[test] fn insert_row_rejects_column_name_with_sql_syntax() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT)").unwrap(); let row = HashMap::from([("id); DROP TABLE t; --".into(), Value::Text("x".into()))]); let err = db.insert_row("t", &row, "f.json", 0).unwrap_err(); assert!(matches!(err, DbError::InvalidIdentifier(_)), "got: {err:?}"); @@ -1754,7 +1469,7 @@ mod tests { #[test] fn insert_row_round_trips_reserved_word_column() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (path TEXT, \"order\" INTEGER)") + db.create_table("t", "CREATE TABLE t (path TEXT, \"order\" INTEGER)") .unwrap(); let row = HashMap::from([ ("path".into(), Value::Text("a".into())), @@ -1780,7 +1495,10 @@ mod tests { fn create_table_rejects_virtual_table_with_clear_error() { let db = Db::new().unwrap(); let err = db - .create_table("CREATE VIRTUAL TABLE vss USING vec0(embedding float[4])") + .create_table( + "vss", + "CREATE VIRTUAL TABLE vss USING vec0(embedding float[4])", + ) .unwrap_err(); // Must be a clear "not supported" message, NOT the generic `DdlParse` // echo (which trivially contains "virtual table" because it echoes @@ -1871,7 +1589,7 @@ mod tests { #[test] fn insert_row_records_mapping() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT)").unwrap(); db.insert_row( "t", &HashMap::from([("id".into(), Value::Text("a".into()))]), @@ -1896,7 +1614,7 @@ mod tests { // A user-declared `INTEGER PRIMARY KEY` is a rowid alias: the // inserted value becomes the rowid. let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + db.create_table("t", "CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") .unwrap(); db.insert_row( "t", @@ -1917,7 +1635,7 @@ mod tests { #[test] fn delete_rows_by_file_removes_mapping() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT)").unwrap(); for (i, file) in ["a.jsonl", "a.jsonl", "b.jsonl"].iter().enumerate() { db.insert_row( "t", @@ -1937,7 +1655,7 @@ mod tests { #[test] fn delete_rows_by_file_resolves_ownership_through_mapping() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT)").unwrap(); db.insert_row( "t", &HashMap::from([("id".into(), Value::Text("a".into()))]), @@ -1960,8 +1678,8 @@ mod tests { // Rows in different tables can share a rowid; the delete must not // cross tables for the same file path. let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t1 (id TEXT)").unwrap(); - db.create_table("CREATE TABLE t2 (id TEXT)").unwrap(); + db.create_table("t1", "CREATE TABLE t1 (id TEXT)").unwrap(); + db.create_table("t2", "CREATE TABLE t2 (id TEXT)").unwrap(); db.insert_row( "t1", &HashMap::from([("id".into(), Value::Text("x".into()))]), @@ -1988,7 +1706,8 @@ mod tests { #[test] fn failed_row_insert_leaves_no_mapping_row() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT UNIQUE)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT UNIQUE)") + .unwrap(); db.insert_row( "t", &HashMap::from([("id".into(), Value::Text("dup".into()))]), @@ -2018,7 +1737,7 @@ mod tests { #[test] fn failed_mapping_insert_rolls_back_row_insert() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT)").unwrap(); // Remove the mapping table so the second statement in the transaction // errors with "no such table". db.conn @@ -2044,7 +1763,7 @@ mod tests { #[test] fn create_table_allows_without_rowid_and_warns() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT PRIMARY KEY) WITHOUT ROWID") + db.create_table("t", "CREATE TABLE t (id TEXT PRIMARY KEY) WITHOUT ROWID") .unwrap(); let rows = db.query("SELECT * FROM t").unwrap(); assert_eq!(rows.len(), 0); @@ -2064,7 +1783,7 @@ mod tests { #[test] fn insert_row_joins_callers_open_transaction() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT)").unwrap(); let tx = db.conn.unchecked_transaction().unwrap(); db.insert_row( "t", @@ -2085,7 +1804,7 @@ mod tests { #[test] fn insert_row_inside_committed_transaction_persists() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT)").unwrap(); let tx = db.conn.unchecked_transaction().unwrap(); db.insert_row( "t", @@ -2104,7 +1823,7 @@ mod tests { #[test] fn delete_rows_by_file_joins_callers_open_transaction() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT)").unwrap(); db.insert_row( "t", &HashMap::from([("id".into(), Value::Text("x".into()))]), diff --git a/packages/rust/src/default_config.toml b/packages/rust/src/default_config.toml index 5dfcdb033..c1dc3f702 100644 --- a/packages/rust/src/default_config.toml +++ b/packages/rust/src/default_config.toml @@ -16,6 +16,10 @@ # dirsql query "SELECT * FROM records" -c .dirsql.toml [[table]] +# The table's SQL name. Declared, not guessed from the DDL below -- the two +# must agree, and dirsql checks that against SQLite's catalog at startup. +name = "records" + # The files this table is built from (glob, relative to the index root). glob = "**/*.json" diff --git a/packages/rust/src/lib.rs b/packages/rust/src/lib.rs index f01efc736..97de62302 100644 --- a/packages/rust/src/lib.rs +++ b/packages/rust/src/lib.rs @@ -43,7 +43,7 @@ pub mod cli; use crate::command::Placeholder; use crate::config::Source; -use crate::db::{Db, parse_table_name}; +use crate::db::Db; use crate::functions::ResolvedFunction; use crate::matcher::TableMatcher; use crate::persist::{ @@ -127,6 +127,12 @@ pub enum DirSqlError { #[error("table DDL could not be parsed: {0}")] Ddl(String), + #[error( + "table '{name}': its `ddl` ran but created no table called '{name}'. \ + Set `name` to the table the `ddl` creates." + )] + TableNotCreated { name: String }, + #[error("failed to load extension '{}': {source}", .path.display())] Extension { path: PathBuf, @@ -211,8 +217,13 @@ pub type Result = std::result::Result; /// callback can itself fail (bad file content, IO errors inside the callback, /// etc.). [`Table::strict`] rejects rows that don't match the DDL columns /// exactly. +/// +/// `name` is the table's SQL name, declared rather than derived from `ddl`. +/// The `ddl` must create a table by that name; the mismatch is caught against +/// SQLite's catalog at load time. #[derive(Clone)] pub struct Table { + pub name: String, pub ddl: String, pub glob: String, pub strict: bool, @@ -220,29 +231,45 @@ pub struct Table { } impl Table { - pub fn new(ddl: impl Into, glob: impl Into, on_file: F) -> Self + pub fn new( + name: impl Into, + ddl: impl Into, + glob: impl Into, + on_file: F, + ) -> Self where F: Fn(&str) -> Vec + Send + Sync + 'static, { - Self::try_new(ddl, glob, move |path| { + Self::try_new(name, ddl, glob, move |path| { Ok::, BoxError>(on_file(path)) }) } - pub fn strict(ddl: impl Into, glob: impl Into, on_file: F) -> Self + pub fn strict( + name: impl Into, + ddl: impl Into, + glob: impl Into, + on_file: F, + ) -> Self where F: Fn(&str) -> Vec + Send + Sync + 'static, { - let mut table = Self::new(ddl, glob, on_file); + let mut table = Self::new(name, ddl, glob, on_file); table.strict = true; table } - pub fn try_new(ddl: impl Into, glob: impl Into, on_file: F) -> Self + pub fn try_new( + name: impl Into, + ddl: impl Into, + glob: impl Into, + on_file: F, + ) -> Self where F: Fn(&str) -> std::result::Result, BoxError> + Send + Sync + 'static, { Self { + name: name.into(), ddl: ddl.into(), glob: glob.into(), on_file: Arc::new(on_file), @@ -820,13 +847,19 @@ impl DirSQL { let mut ddl_map: HashMap = HashMap::new(); for table in tables { - let table_name = - parse_table_name(&table.ddl).ok_or_else(|| DirSqlError::Ddl(table.ddl.clone()))?; + let table_name = table.name.clone(); // When the cache already holds this table from a prior run, // skip CREATE TABLE: the schema is preserved verbatim across // runs (the glob_config_hash captures the DDL). if !table_exists(&db, &table_name)? { - db.create_table(&table.ddl)?; + db.create_table(&table_name, &table.ddl)?; + // Ask SQLite whether the DDL actually produced this name + // rather than reading the DDL text. No row means the declared + // `name` names nothing -- a load-time failure, before any file + // is ingested. + if !table_exists(&db, &table_name)? { + return Err(DirSqlError::TableNotCreated { name: table_name }); + } } // Reject a `{name}` glob placeholder whose name is also a declared // column: captures no longer populate columns, so it would read @@ -996,7 +1029,7 @@ impl DirSQL { /// use dirsql::{DirSQL, Table}; /// let db = DirSQL::builder() /// .root("./data") -/// .table(Table::new("CREATE TABLE t (x TEXT)", "*.json", |_, _| vec![])) +/// .table(Table::new("t", "CREATE TABLE t (x TEXT)", "*.json", |_, _| vec![])) /// .ignore(["target/**"]) /// .build()?; /// ``` @@ -1379,10 +1412,9 @@ fn compile_matcher( let mut mappings: Vec<(String, String)> = Vec::with_capacity(tables.len()); let mut names = Vec::with_capacity(tables.len()); for table in tables { - let table_name = - parse_table_name(&table.ddl).ok_or_else(|| DirSqlError::Ddl(table.ddl.clone()))?; + let table_name = table.name.clone(); // Validate up front so a poisoned name from a stored cache or a - // would-be-injection DDL can't propagate into `on_file_map`, + // declared `name` carrying SQL can't propagate into `on_file_map`, // `strict_map`, or any format!()-built SQL down the line. crate::db::validate_identifier(&table_name).map_err(map_db_error)?; if seen.insert(table_name.clone(), ()).is_some() { @@ -1532,11 +1564,15 @@ fn reconcile_scan( Ok((to_parse, trusted, deleted)) } +/// Whether `main` holds a table called `name`, asked of SQLite's own catalog. +/// `pragma_table_list` is the catalog API for exactly this question, so no +/// part of dirsql has to interpret DDL text to answer it. fn table_exists(db: &Db, name: &str) -> Result { let count: i64 = db .conn() .query_row( - "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1", + "SELECT COUNT(*) FROM pragma_table_list \ + WHERE schema = 'main' AND type = 'table' AND name = ?1", rusqlite::params![name], |row| row.get(0), ) @@ -1656,6 +1692,7 @@ fn build_tables_from_config( // `Table::try_new`: a hook failure is the scan's to record, not this // closure's to hide. The scan skips the file and reports it. let mut table = Table::try_new( + table_cfg.name.clone(), table_cfg.ddl.clone(), table_cfg.glob.clone(), move |abs_path: &str| run_on_file(&command, abs_path, &config_dir, &root), @@ -2216,6 +2253,7 @@ mod internal_tests { let db = DirSQL::with_ignore_and_fs( dir.path(), vec![Table::new( + "items", "CREATE TABLE items (name TEXT)", "**/*.txt", |_| { @@ -2246,7 +2284,12 @@ mod internal_tests { let fake = FakeFs::default().with_canonical_root(".", "/ws/canonical"); let db = DirSQL::with_ignore_and_fs( ".", - vec![Table::new("CREATE TABLE t (x TEXT)", "*.txt", |_| vec![])], + vec![Table::new( + "t", + "CREATE TABLE t (x TEXT)", + "*.txt", + |_| vec![], + )], Vec::::new(), Arc::new(fake), ) @@ -2271,6 +2314,7 @@ mod internal_tests { let db = DirSQL::with_ignore_and_fs( &root, vec![Table::new( + "items", "CREATE TABLE items (name TEXT)", "**/*.txt", |_| { @@ -2308,6 +2352,7 @@ mod internal_tests { let mut db = DirSQL::with_ignore_and_fs( &root, vec![Table::new( + "items", "CREATE TABLE items (name TEXT)", "**/*.txt", |_| { @@ -2347,6 +2392,7 @@ mod internal_tests { let db = DirSQL::with_ignore_and_fs( &root, vec![Table::new( + "items", "CREATE TABLE items (name TEXT)", "*.txt", |_| { @@ -2521,6 +2567,7 @@ mod internal_tests { let db = DirSQL::with_ignore_and_fs( dir.path(), vec![Table::new( + "items", "CREATE TABLE items (name TEXT)", "**/*.txt", |_| { @@ -2657,12 +2704,17 @@ mod internal_tests { let fake = FakeFs::default().with_dir(subdir.clone()); let db = DirSQL::with_ignore_and_fs( dir.path(), - vec![Table::new("CREATE TABLE files (name TEXT)", "**/*", |_| { - vec![Row::from_iter([( - "name".to_string(), - Value::Text("x".into()), - )])] - })], + vec![Table::new( + "files", + "CREATE TABLE files (name TEXT)", + "**/*", + |_| { + vec![Row::from_iter([( + "name".to_string(), + Value::Text("x".into()), + )])] + }, + )], Vec::::new(), Arc::new(fake), ) @@ -2701,6 +2753,7 @@ mod internal_tests { let db = DirSQL::with_ignore_and_fs( dir.path(), vec![Table::strict( + "items", "CREATE TABLE items (name TEXT)", "**/*.txt", |_| { @@ -2746,7 +2799,7 @@ mod internal_tests { fn is_configless_only_when_config_and_tables_are_both_empty() { // Pure truth table (no I/O): the missing-`files` hint is armed only // when neither a config path nor a programmatic table was supplied. - let table = Table::new("CREATE TABLE x (a TEXT)", "*", |_| vec![Row::new()]); + let table = Table::new("x", "CREATE TABLE x (a TEXT)", "*", |_| vec![Row::new()]); assert!(is_configless(&[], &[])); assert!(!is_configless(&[PathBuf::from("c.toml")], &[])); assert!(!is_configless(&[], std::slice::from_ref(&table))); @@ -2793,7 +2846,7 @@ mod internal_tests { fn resolve_with_a_programmatic_table_disarms_the_hint() { let with_table = DirSQL::builder() .root("/tmp/x") - .table(Table::new("CREATE TABLE t (a TEXT)", "*.t", |_| { + .table(Table::new("t", "CREATE TABLE t (a TEXT)", "*.t", |_| { vec![Row::new()] })) .resolve() @@ -2840,8 +2893,8 @@ mod internal_tests { let err = match DirSQL::new( dir.path(), vec![ - Table::new("CREATE TABLE t (a TEXT)", "*.a", |_| vec![]), - Table::new("CREATE TABLE t (b TEXT)", "*.b", |_| vec![]), + Table::new("t", "CREATE TABLE t (a TEXT)", "*.a", |_| vec![]), + Table::new("t", "CREATE TABLE t (b TEXT)", "*.b", |_| vec![]), ], ) { Ok(_) => panic!("expected a duplicate-table error"), @@ -2940,6 +2993,7 @@ mod internal_tests { let db = DirSQL::with_ignore_and_fs( dir.path(), vec![Table::try_new( + "items", "CREATE TABLE items (name TEXT)", "**/*.txt", |_| Err("boom".into()), @@ -2962,11 +3016,17 @@ mod internal_tests { let db = DirSQL::builder() .root(dir.path()) .tables(vec![Table::new( + "b", "CREATE TABLE b (y TEXT)", "*.b", |_| vec![], )]) - .table(Table::new("CREATE TABLE a (x TEXT)", "*.a", |_| vec![])) + .table(Table::new( + "a", + "CREATE TABLE a (x TEXT)", + "*.a", + |_| vec![], + )) .ignore(["skip/**"]) .extensions(Vec::::new()) .suppress_config_extensions(true) @@ -3072,6 +3132,7 @@ mod internal_tests { let first = DirSQL::builder() .root(dir.path()) .tables(vec![Table::new( + "t", "CREATE TABLE t (x TEXT)", "*.txt", |_| vec![], @@ -3083,6 +3144,7 @@ mod internal_tests { let second = DirSQL::builder() .root(dir.path()) .tables(vec![Table::new( + "t", "CREATE TABLE t (x TEXT)", "*.txt", |_| vec![], @@ -3096,7 +3158,12 @@ mod internal_tests { #[test] fn prepare_persist_cold_start_reports_rebuild() { let dir = TempDir::new().unwrap(); - let tables = vec![Table::new("CREATE TABLE t (x TEXT)", "*.txt", |_| vec![])]; + let tables = vec![Table::new( + "t", + "CREATE TABLE t (x TEXT)", + "*.txt", + |_| vec![], + )]; let ctx = prepare_persist(dir.path(), &tables, &[], None).unwrap(); assert!(ctx.cached.is_empty()); assert!(!ctx.expected_meta.is_empty()); @@ -3265,10 +3332,12 @@ mod internal_tests { fn build_tables_from_config_creates_on_file_tables() { let cfg = config::load_config_str(concat!( "[[table]]\n", + "name = \"a\"\n", "ddl = \"CREATE TABLE a (x TEXT)\"\n", "glob = \"*.a\"\n", "on-file = \"printf '[{\\\"x\\\":1}]'\"\n\n", "[[table]]\n", + "name = \"b\"\n", "ddl = \"CREATE TABLE b (y TEXT)\"\n", "glob = \"*.b\"\n", "on-file = \"echo hi\"\n", diff --git a/packages/rust/src/persist.rs b/packages/rust/src/persist.rs index 568cfaa97..737a99fd1 100644 --- a/packages/rust/src/persist.rs +++ b/packages/rust/src/persist.rs @@ -152,8 +152,10 @@ pub fn is_trusted( pub fn compute_glob_config_hash(tables: &[Table], ignore: &[String]) -> String { let mut entries: BTreeMap = BTreeMap::new(); for table in tables { - let name = crate::db::parse_table_name(&table.ddl).unwrap_or_default(); - entries.insert(name, (table.ddl.clone(), table.glob.clone(), table.strict)); + entries.insert( + table.name.clone(), + (table.ddl.clone(), table.glob.clone(), table.strict), + ); } let mut hasher = blake3::Hasher::new(); hasher.update(b"v1\n"); @@ -441,8 +443,8 @@ mod tests { #[test] fn glob_config_hash_is_deterministic() { - let t1 = Table::new("CREATE TABLE a (x TEXT)", "*.json", |_| vec![]); - let t2 = Table::new("CREATE TABLE b (y TEXT)", "*.csv", |_| vec![]); + let t1 = Table::new("a", "CREATE TABLE a (x TEXT)", "*.json", |_| vec![]); + let t2 = Table::new("b", "CREATE TABLE b (y TEXT)", "*.csv", |_| vec![]); let h1 = compute_glob_config_hash( &[t1.clone(), t2.clone()], &["foo".to_string(), "bar".to_string()], @@ -453,8 +455,8 @@ mod tests { #[test] fn glob_config_hash_changes_when_glob_changes() { - let t1 = Table::new("CREATE TABLE a (x TEXT)", "*.json", |_| vec![]); - let t2 = Table::new("CREATE TABLE a (x TEXT)", "*.csv", |_| vec![]); + let t1 = Table::new("a", "CREATE TABLE a (x TEXT)", "*.json", |_| vec![]); + let t2 = Table::new("a", "CREATE TABLE a (x TEXT)", "*.csv", |_| vec![]); let h1 = compute_glob_config_hash(&[t1], &[]); let h2 = compute_glob_config_hash(&[t2], &[]); assert_ne!(h1, h2); @@ -462,8 +464,8 @@ mod tests { #[test] fn glob_config_hash_changes_when_strict_changes() { - let mut t1 = Table::new("CREATE TABLE a (x TEXT)", "*.json", |_| vec![]); - let mut t2 = Table::new("CREATE TABLE a (x TEXT)", "*.json", |_| vec![]); + let mut t1 = Table::new("a", "CREATE TABLE a (x TEXT)", "*.json", |_| vec![]); + let mut t2 = Table::new("a", "CREATE TABLE a (x TEXT)", "*.json", |_| vec![]); t2.strict = true; let h1 = compute_glob_config_hash(&[t1.clone()], &[]); let h2 = compute_glob_config_hash(&[t2], &[]); diff --git a/packages/rust/tests/async_sdk.rs b/packages/rust/tests/async_sdk.rs index b860cb390..4d925034b 100644 --- a/packages/rust/tests/async_sdk.rs +++ b/packages/rust/tests/async_sdk.rs @@ -7,6 +7,7 @@ use tempfile::TempDir; fn comments_table() -> Table { Table::new( + "comments", "CREATE TABLE comments (id TEXT, body TEXT, author TEXT)", "comments/**/index.txt", |path| { @@ -36,13 +37,18 @@ fn comments_table() -> Table { } fn items_table() -> Table { - Table::new("CREATE TABLE items (name TEXT)", "**/*.txt", |path| { - let content = std::fs::read_to_string(path).unwrap(); - vec![HashMap::from([( - "name".into(), - Value::Text(content.trim().to_string()), - )])] - }) + Table::new( + "items", + "CREATE TABLE items (name TEXT)", + "**/*.txt", + |path| { + let content = std::fs::read_to_string(path).unwrap(); + vec![HashMap::from([( + "name".into(), + Value::Text(content.trim().to_string()), + )])] + }, + ) } #[tokio::test] @@ -156,6 +162,7 @@ async fn from_config_path_loads_config_with_explicit_root() { &cfg_path, r#" [[table]] +name = "files" ddl = "CREATE TABLE files (path TEXT)" glob = "*.csv" on-file = "printf '[{}]'" @@ -206,13 +213,18 @@ async fn sync_backed_methods_before_ready_error() { // complete during the assertion window. let gate = std::sync::Arc::new(std::sync::Barrier::new(2)); let gate_in_extract = gate.clone(); - let gated_table = Table::new("CREATE TABLE items (name TEXT)", "**/*.txt", move |_| { - gate_in_extract.wait(); - vec![HashMap::from([( - "name".into(), - Value::Text("apple".into()), - )])] - }); + let gated_table = Table::new( + "items", + "CREATE TABLE items (name TEXT)", + "**/*.txt", + move |_| { + gate_in_extract.wait(); + vec![HashMap::from([( + "name".into(), + Value::Text("apple".into()), + )])] + }, + ); let db = AsyncDirSQL::new(root.path(), vec![gated_table]).unwrap(); // Every sync-backed method threads `self.sync()?` first, so all surface @@ -257,8 +269,8 @@ async fn sync_backed_methods_before_ready_error() { #[tokio::test] async fn init_failure_surfaces_through_ready_and_sync() { let root = TempDir::new().unwrap(); - let t1 = Table::new("CREATE TABLE dup (a TEXT)", "*.a", |_| vec![]); - let t2 = Table::new("CREATE TABLE dup (b TEXT)", "*.b", |_| vec![]); + let t1 = Table::new("dup", "CREATE TABLE dup (a TEXT)", "*.a", |_| vec![]); + let t2 = Table::new("dup", "CREATE TABLE dup (b TEXT)", "*.b", |_| vec![]); let db = AsyncDirSQL::new(root.path(), vec![t1, t2]).unwrap(); let ready = db.ready().await; diff --git a/packages/rust/tests/cli_e2e.rs b/packages/rust/tests/cli_e2e.rs index 79ffe16d5..2f5a8bc39 100644 --- a/packages/rust/tests/cli_e2e.rs +++ b/packages/rust/tests/cli_e2e.rs @@ -39,6 +39,7 @@ fn blog_fixture() -> TempDir { root.path().join(".dirsql.toml"), r#" [[table]] +name = "posts" ddl = "CREATE TABLE posts (basename TEXT, size INTEGER)" glob = "posts/*/*.json" on-file = '''sh -c 'base=${1##*/}; size=$(wc -c < "$1" | tr -d " "); printf "[{\"basename\":\"%s\",\"size\":%s}]" "$base" "$size"' sh {path}''' @@ -60,6 +61,7 @@ fn quoted_blog_fixture() -> TempDir { // Single-quoted TOML string so the embedded double quotes are literal. r#" [[table]] +name = "posts" ddl = 'CREATE TABLE "posts" (basename TEXT)' glob = "posts/*/*.json" on-file = '''sh -c 'printf "[{\"basename\":\"%s\"}]" "${1##*/}"' sh {path}''' @@ -820,6 +822,7 @@ fn query_subcommand_rejects_capture_column_collision() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "comments" ddl = "CREATE TABLE comments (thread_id TEXT, basename TEXT)" glob = "_comments/{thread_id}/*.txt" on-file = "cat {path}" @@ -868,11 +871,13 @@ fn query_subcommand_fans_out_file_to_overlapping_tables() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "ta" ddl = "CREATE TABLE ta (path TEXT)" glob = "data/*/metadata.json" on-file = '''sh -c 'rel=${1#"$2"/}; printf "[{\"path\":\"%s\"}]" "$rel"' sh {path} {root}''' [[table]] +name = "tb" ddl = "CREATE TABLE tb (path TEXT)" glob = "data/**/metadata.json" on-file = '''sh -c 'rel=${1#"$2"/}; printf "[{\"path\":\"%s\"}]" "$rel"' sh {path} {root}''' @@ -964,7 +969,7 @@ fn hookless_table_config_exits_nonzero_pointing_at_the_path_table() { let dir = TempDir::new().unwrap(); fs::write( dir.path().join(".dirsql.toml"), - "[[table]]\nddl = \"CREATE TABLE files (path TEXT, size INTEGER)\"\nglob = \"**/*.md\"\n", + "[[table]]\nname = \"files\"\nddl = \"CREATE TABLE files (path TEXT, size INTEGER)\"\nglob = \"**/*.md\"\n", ) .unwrap(); let out = std::process::Command::cargo_bin("dirsql") @@ -1110,6 +1115,7 @@ fn include_default_conflicting_records_table_exits_nonzero_naming_records() { dir.path().join("dup.toml"), r#" [[table]] +name = "records" ddl = "CREATE TABLE records (x TEXT)" glob = "**/*" on-file = "cat {path}" @@ -1310,6 +1316,7 @@ fn config_elsewhere_indexes_invocation_cwd_not_config_parent() { elsewhere.path().join(".dirsql.toml"), r#" [[table]] +name = "posts" ddl = "CREATE TABLE posts (basename TEXT)" glob = "posts/*/*.json" on-file = "printf '[{}]'" diff --git a/packages/rust/tests/cli_integration.rs b/packages/rust/tests/cli_integration.rs index a437395cd..172ab4f93 100644 --- a/packages/rust/tests/cli_integration.rs +++ b/packages/rust/tests/cli_integration.rs @@ -40,6 +40,7 @@ fn blog_fixture() -> (TempDir, DirSQL) { root.path().join(".dirsql.toml"), r#" [[table]] +name = "posts" ddl = "CREATE TABLE posts (basename TEXT, size INTEGER)" glob = "posts/*/*.json" on-file = '''sh -c 'base=${1##*/}; size=$(wc -c < "$1" | tr -d " "); printf "[{\"basename\":\"%s\",\"size\":%s}]" "$base" "$size"' sh {path}''' @@ -293,6 +294,7 @@ async fn get_events_surfaces_parse_errors_as_error_events_not_fatal() { fs::write(root.path().join("posts/second.json"), r#"{"ok":2}"#).unwrap(); let table = dirsql::Table::try_new( + "posts", "CREATE TABLE posts (ok INTEGER, basename TEXT)", "posts/*.json", |path| { diff --git a/packages/rust/tests/cli_repeatable_config_e2e.rs b/packages/rust/tests/cli_repeatable_config_e2e.rs index 0f70faf23..f0b117283 100644 --- a/packages/rust/tests/cli_repeatable_config_e2e.rs +++ b/packages/rust/tests/cli_repeatable_config_e2e.rs @@ -29,6 +29,7 @@ fn write_table_config(dir: &Path, table: &str, glob: &str) -> std::path::PathBuf format!( r#" [[table]] +name = "{table}" ddl = "CREATE TABLE {table} (basename TEXT)" glob = "{glob}" on-file = "printf '[{{}}]'" diff --git a/packages/rust/tests/code_review_findings.rs b/packages/rust/tests/code_review_findings.rs index 817fe5e30..e381af1fa 100644 --- a/packages/rust/tests/code_review_findings.rs +++ b/packages/rust/tests/code_review_findings.rs @@ -13,6 +13,7 @@ fn s1_ddl_with_semicolon_in_table_name_slot_is_rejected() { let result = DirSQL::new( dir.path(), vec![Table::new( + "evil;DROP_TABLE_bar--", "CREATE TABLE evil;DROP_TABLE_bar--(id TEXT)", "*.json", |_| vec![], @@ -39,11 +40,16 @@ fn s1_column_name_with_sql_syntax_produces_clean_error() { std::fs::write(dir.path().join("a.json"), b"{}").unwrap(); let result = DirSQL::new( dir.path(), - vec![Table::strict("CREATE TABLE t (id TEXT)", "*.json", |_| { - let mut row = Row::new(); - row.insert("id); DROP TABLE t; --".into(), Value::Text("x".into())); - vec![row] - })], + vec![Table::strict( + "t", + "CREATE TABLE t (id TEXT)", + "*.json", + |_| { + let mut row = Row::new(); + row.insert("id); DROP TABLE t; --".into(), Value::Text("x".into())); + vec![row] + }, + )], ); // The row is still rejected; since #714 the rejection costs that file // rather than the scan, so it surfaces as a reported skip instead of a @@ -73,11 +79,16 @@ fn one_row_db(dir: &TempDir) -> DirSQL { std::fs::write(dir.path().join("a.json"), b"{}").unwrap(); DirSQL::new( dir.path(), - vec![Table::new("CREATE TABLE t (id TEXT)", "*.json", |_| { - let mut row = Row::new(); - row.insert("id".into(), Value::Text("x".into())); - vec![row] - })], + vec![Table::new( + "t", + "CREATE TABLE t (id TEXT)", + "*.json", + |_| { + let mut row = Row::new(); + row.insert("id".into(), Value::Text("x".into())); + vec![row] + }, + )], ) .unwrap() } @@ -131,7 +142,12 @@ fn p7_builder_exposes_poll_interval() { let dir = TempDir::new().unwrap(); let _db = DirSQL::builder() .root(dir.path()) - .table(Table::new("CREATE TABLE t (id TEXT)", "*.json", |_| vec![])) + .table(Table::new( + "t", + "CREATE TABLE t (id TEXT)", + "*.json", + |_| vec![], + )) .poll_interval(Duration::from_millis(50)) .build() .unwrap(); @@ -142,7 +158,12 @@ fn i3_watch_error_exposes_underlying_source() { let dir = TempDir::new().unwrap(); let db = DirSQL::new( dir.path(), - vec![Table::new("CREATE TABLE t (id TEXT)", "*.json", |_| vec![])], + vec![Table::new( + "t", + "CREATE TABLE t (id TEXT)", + "*.json", + |_| vec![], + )], ) .unwrap(); // Delete the directory out from under the watcher so `start_watching` @@ -178,16 +199,21 @@ fn i8_ext_preserves_original_case() { std::fs::write(dir.path().join("Photo.JPG"), b"").unwrap(); let db = DirSQL::new( dir.path(), - vec![Table::new("CREATE TABLE pics (ext TEXT)", "**/*", |path| { - let mut row = Row::new(); - if let Some(ext) = std::path::Path::new(path).extension() { - row.insert( - "ext".into(), - Value::Text(ext.to_string_lossy().into_owned()), - ); - } - vec![row] - })], + vec![Table::new( + "pics", + "CREATE TABLE pics (ext TEXT)", + "**/*", + |path| { + let mut row = Row::new(); + if let Some(ext) = std::path::Path::new(path).extension() { + row.insert( + "ext".into(), + Value::Text(ext.to_string_lossy().into_owned()), + ); + } + vec![row] + }, + )], ) .unwrap(); let rows = db.query("SELECT ext FROM pics").unwrap(); diff --git a/packages/rust/tests/config.rs b/packages/rust/tests/config.rs index 40ad84bbc..565c5e01f 100644 --- a/packages/rust/tests/config.rs +++ b/packages/rust/tests/config.rs @@ -16,6 +16,7 @@ fn load_config_from_file() { &path, r#" [[table]] +name = "t" ddl = "CREATE TABLE t (path TEXT)" glob = "*.csv" on-file = "cat {path}" @@ -40,7 +41,7 @@ fn unknown_key_at_each_schema_level_errors() { ), ( "[[table]]", - "[[table]]\nddl = \"CREATE TABLE t (path TEXT)\"\nglob = \"*.json\"\non-file = \"cat {path}\"\nformat = \"json\"\n", + "[[table]]\nname = \"t\"\nddl = \"CREATE TABLE t (path TEXT)\"\nglob = \"*.json\"\non-file = \"cat {path}\"\nformat = \"json\"\n", "format", ), ( diff --git a/packages/rust/tests/config_root_derivation.rs b/packages/rust/tests/config_root_derivation.rs index 136f17533..e902aa670 100644 --- a/packages/rust/tests/config_root_derivation.rs +++ b/packages/rust/tests/config_root_derivation.rs @@ -57,6 +57,7 @@ fn config_without_root_indexes_process_cwd_not_config_parent() { &cfg_path, r#" [[table]] +name = "files" ddl = "CREATE TABLE files (path TEXT)" glob = "*.txt" on-file = '''sh -c 'rel=${1#"$2"/}; printf "[{\"path\":\"%s\"}]" "$rel"' sh {path} {root}''' @@ -86,6 +87,7 @@ fn explicit_root_wins_over_cwd() { &cfg_path, r#" [[table]] +name = "files" ddl = "CREATE TABLE files (path TEXT)" glob = "*.txt" on-file = '''sh -c 'rel=${1#"$2"/}; printf "[{\"path\":\"%s\"}]" "$rel"' sh {path} {root}''' diff --git a/packages/rust/tests/declared_table_name.rs b/packages/rust/tests/declared_table_name.rs new file mode 100644 index 000000000..a502c22fb --- /dev/null +++ b/packages/rust/tests/declared_table_name.rs @@ -0,0 +1,117 @@ +//! Integration tests for the declared `[[table]] name` key. +//! +//! A table's name is *declared*, never derived: dirsql does not tokenize +//! `ddl` to learn it. A `[[table]]` entry without `name` fails to load, and a +//! `name` that the entry's `ddl` does not actually create fails at load time +//! -- checked against SQLite's own catalog, before any file is ingested. + +use dirsql::{DirSQL, Value}; +use std::fs; +use tempfile::TempDir; + +/// An `on-file` hook emitting the file's root-relative `path`. +const HOOK: &str = r#"on-file = '''sh -c 'rel=${1#"$2"/}; printf "[{\"path\":\"%s\"}]" "$rel"' sh {path} {root}'''"#; + +/// A tempdir with one matched file and a `.dirsql.toml` holding `config`. +fn fixture(config: &str) -> TempDir { + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("data")).unwrap(); + fs::write(root.path().join("data/a.csv"), "anything").unwrap(); + fs::write(root.path().join(".dirsql.toml"), config).unwrap(); + root +} + +fn build(root: &TempDir) -> dirsql::Result { + DirSQL::builder() + .root(root.path()) + .config(root.path().join(".dirsql.toml")) + .build() +} + +#[test] +fn declared_name_registers_the_table() { + let root = fixture(&format!( + r#" +[[table]] +name = "notes" +ddl = "CREATE TABLE notes (path TEXT)" +glob = "data/*.csv" +{HOOK} +"# + )); + + let db = build(&root).expect("a [[table]] with a matching `name` must load"); + let rows = db.query("SELECT path FROM notes").unwrap(); + + assert_eq!(rows.len(), 1); + assert_eq!(rows[0]["path"], Value::Text("data/a.csv".into())); +} + +#[test] +fn missing_name_is_a_load_error() { + let root = fixture(&format!( + r#" +[[table]] +ddl = "CREATE TABLE notes (path TEXT)" +glob = "data/*.csv" +{HOOK} +"# + )); + + let message = match build(&root) { + Ok(_) => panic!("a [[table]] without `name` must fail to load"), + Err(err) => err.to_string(), + }; + assert!( + message.contains("name") && message.contains("[[table]]"), + "the error must name the missing `name` key of the [[table]] entry, got {message:?}" + ); +} + +#[test] +fn name_the_ddl_never_creates_is_a_load_error() { + let root = fixture(&format!( + r#" +[[table]] +name = "messages" +ddl = "CREATE TABLE notes (path TEXT)" +glob = "data/*.csv" +{HOOK} +"# + )); + + let message = match build(&root) { + Ok(_) => panic!("a `name` absent from the catalog must fail to load"), + Err(err) => err.to_string(), + }; + assert!( + message.contains("table 'messages'"), + "the error must carry the config-entry prefix `table 'messages'`, got {message:?}" + ); +} + +/// A quoted or schema-qualified identifier is a SQL *delimiter* form, not part +/// of the name: SQLite records the bare name in its catalog, so the declared +/// `name` matches without dirsql interpreting the DDL text at all. +#[test] +fn quoted_and_schema_qualified_ddl_match_the_declared_name() { + for ddl in [ + r#"CREATE TABLE \"notes\" (path TEXT)"#, + "CREATE TABLE main.notes (path TEXT)", + "CREATE TABLE IF NOT EXISTS notes (path TEXT)", + ] { + let root = fixture(&format!( + r#" +[[table]] +name = "notes" +ddl = "{ddl}" +glob = "data/*.csv" +{HOOK} +"# + )); + + let db = build(&root).unwrap_or_else(|e| panic!("{ddl} must load, got {e}")); + let rows = db.query("SELECT path FROM notes").unwrap(); + assert_eq!(rows.len(), 1, "{ddl} must index the matched file"); + } +} diff --git a/packages/rust/tests/declared_table_name_e2e.rs b/packages/rust/tests/declared_table_name_e2e.rs new file mode 100644 index 000000000..70a9cd753 --- /dev/null +++ b/packages/rust/tests/declared_table_name_e2e.rs @@ -0,0 +1,112 @@ +//! End-to-end tests for the declared `[[table]] name` key. +//! +//! Spawns the real compiled `dirsql` binary against a real temp directory and +//! a real `.dirsql.toml`. Nothing is mocked (real process, real filesystem, +//! real SQLite, real `on-file` command spawn). +//! +//! Gated behind `--features cli` (the `dirsql` bin needs it) and Unix (the +//! fixture shells out to `cat`); the Rust CI test job runs on Linux. + +#![cfg(all(feature = "cli", unix))] + +use std::fs; +use std::process::Output; + +use assert_cmd::prelude::*; +use serde_json::Value; +use tempfile::TempDir; + +/// A tempdir holding one JSON file of rows and a `.dirsql.toml` with `config`. +fn fixture(config: &str) -> TempDir { + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("data")).unwrap(); + fs::write( + root.path().join("data/a.json"), + r#"[{"id": "one"}, {"id": "two"}]"#, + ) + .unwrap(); + fs::write(root.path().join(".dirsql.toml"), config).unwrap(); + root +} + +fn query(root: &TempDir, sql: &str) -> Output { + std::process::Command::cargo_bin("dirsql") + .expect("`dirsql` binary must be built with --features cli") + .arg("query") + .arg(sql) + .arg("--config") + .arg(root.path().join(".dirsql.toml")) + .current_dir(root.path()) + .output() + .expect("spawning `dirsql query` failed") +} + +#[test] +fn declared_name_is_queryable_through_the_cli() { + let root = fixture( + r#" +[[table]] +name = "records" +ddl = "CREATE TABLE records (id TEXT)" +glob = "data/*.json" +on-file = "cat {path}" +"#, + ); + + let out = query(&root, "SELECT id FROM records ORDER BY id"); + assert!( + out.status.success(), + "a [[table]] with a matching `name` must query cleanly, stderr: {}", + String::from_utf8_lossy(&out.stderr) + ); + let rows: Vec = serde_json::from_slice(&out.stdout).expect("stdout must be JSON"); + let ids: Vec<&str> = rows.iter().map(|r| r["id"].as_str().unwrap()).collect(); + assert_eq!(ids, ["one", "two"]); +} + +#[test] +fn missing_name_exits_nonzero_and_names_the_key() { + let root = fixture( + r#" +[[table]] +ddl = "CREATE TABLE records (id TEXT)" +glob = "data/*.json" +on-file = "cat {path}" +"#, + ); + + let out = query(&root, "SELECT id FROM records"); + assert!( + !out.status.success(), + "a [[table]] without `name` must exit non-zero" + ); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("name") && stderr.contains("[[table]]"), + "stderr must name the missing `name` key of the [[table]] entry, got {stderr:?}" + ); +} + +#[test] +fn name_the_ddl_never_creates_exits_nonzero_before_ingestion() { + let root = fixture( + r#" +[[table]] +name = "messages" +ddl = "CREATE TABLE records (id TEXT)" +glob = "data/*.json" +on-file = "cat {path}" +"#, + ); + + let out = query(&root, "SELECT id FROM messages"); + assert!( + !out.status.success(), + "a `name` absent from the catalog must exit non-zero" + ); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("table 'messages'"), + "stderr must carry the config-entry prefix `table 'messages'`, got {stderr:?}" + ); +} diff --git a/packages/rust/tests/extensions.rs b/packages/rust/tests/extensions.rs index 0d1ef577d..dd5d00172 100644 --- a/packages/rust/tests/extensions.rs +++ b/packages/rust/tests/extensions.rs @@ -76,6 +76,7 @@ fn missing_extension_path_fails_construction() { path = "/nonexistent/dirsql-no-such-extension.so" [[table]] +name = "files" ddl = "CREATE TABLE files (path TEXT)" glob = "*.txt" on-file = "cat {path}" @@ -166,6 +167,7 @@ path = "{}" entrypoint = "sqlite3_extension_init" [[table]] +name = "files" ddl = "CREATE TABLE files (path TEXT)" glob = "*.txt" on-file = "cat {{path}}" @@ -216,6 +218,7 @@ fn suppress_config_extensions_ignores_config_entries_and_uses_overrides() { path = "does/not/exist.so" [[table]] +name = "files" ddl = "CREATE TABLE files (path TEXT)" glob = "*.txt" on-file = "cat {path}" diff --git a/packages/rust/tests/from_config.rs b/packages/rust/tests/from_config.rs index 0b4c558e8..56568902e 100644 --- a/packages/rust/tests/from_config.rs +++ b/packages/rust/tests/from_config.rs @@ -15,6 +15,7 @@ fn from_config_produces_one_row_per_matched_file() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "files" ddl = "CREATE TABLE files (path TEXT, basename TEXT)" glob = "data/*.csv" on-file = '''sh -c 'rel=${1#"$2"/}; base=${1##*/}; printf "[{\"path\":\"%s\",\"basename\":\"%s\"}]" "$rel" "$base"' sh {path} {root}''' @@ -53,6 +54,7 @@ fn from_config_honors_ignore_patterns() { ignore = ["ignored/**"] [[table]] +name = "files" ddl = "CREATE TABLE files (path TEXT)" glob = "**/*.csv" on-file = '''sh -c 'rel=${1#"$2"/}; printf "[{\"path\":\"%s\"}]" "$rel"' sh {path} {root}''' @@ -87,6 +89,7 @@ fn from_config_capture_column_collision_errors() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "comments" ddl = "CREATE TABLE comments (thread_id TEXT, basename TEXT)" glob = "_comments/{thread_id}/*.txt" on-file = "cat {path}" @@ -137,6 +140,7 @@ fn from_config_capture_placeholder_without_column_still_matches() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "comments" ddl = "CREATE TABLE comments (path TEXT, basename TEXT)" glob = "_comments/{thread_id}/*.txt" on-file = '''sh -c 'rel=${1#"$2"/}; base=${1##*/}; printf "[{\"path\":\"%s\",\"basename\":\"%s\"}]" "$rel" "$base"' sh {path} {root}''' @@ -189,6 +193,7 @@ fn from_config_exposes_stat_virtuals() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "files" ddl = "CREATE TABLE files (path TEXT, basename TEXT, dir TEXT, ext TEXT, size INTEGER, mtime INTEGER)" glob = "docs/*.md" on-file = '''sh -c 'rel=${1#"$2"/}; base=${1##*/}; case "$rel" in */*) dir=${rel%/*};; *) dir="";; esac; ext=${base##*.}; [ "$ext" = "$base" ] && ext=""; size=$(wc -c < "$1" | tr -d " "); mtime=$(stat -c %Y "$1"); printf "[{\"path\":\"%s\",\"basename\":\"%s\",\"dir\":\"%s\",\"ext\":\"%s\",\"size\":%s,\"mtime\":%s}]" "$rel" "$base" "$dir" "$ext" "$size" "$mtime"' sh {path} {root}''' @@ -235,6 +240,7 @@ fn from_config_undeclared_stat_columns_are_silently_dropped() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "minimal" ddl = "CREATE TABLE minimal (path TEXT)" glob = "*.txt" on-file = '''sh -c 'rel=${1#"$2"/}; base=${1##*/}; printf "[{\"path\":\"%s\",\"basename\":\"%s\"}]" "$rel" "$base"' sh {path} {root}''' @@ -274,6 +280,7 @@ fn from_config_hookless_table_errors() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "files" ddl = "CREATE TABLE files (path TEXT, size INTEGER)" glob = "**/*.md" "#, @@ -303,6 +310,7 @@ fn from_config_with_no_matching_files_yields_empty_table() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "empty_t" ddl = "CREATE TABLE empty_t (path TEXT)" glob = "nothing_here/*.txt" on-file = "cat {path}" @@ -335,6 +343,7 @@ fn from_config_rejects_removed_persist_keys() { persist = true [[table]] +name = "files" ddl = "CREATE TABLE files (path TEXT)" glob = "*.csv" on-file = "cat {path}" @@ -364,6 +373,7 @@ fn from_config_strict_table_builds() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "files" ddl = "CREATE TABLE files (path TEXT)" glob = "*.csv" strict = true @@ -390,6 +400,7 @@ async fn async_from_config_works() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "files" ddl = "CREATE TABLE files (path TEXT, basename TEXT)" glob = "*.csv" on-file = '''sh -c 'rel=${1#"$2"/}; base=${1##*/}; printf "[{\"path\":\"%s\",\"basename\":\"%s\"}]" "$rel" "$base"' sh {path} {root}''' diff --git a/packages/rust/tests/internal_rows.rs b/packages/rust/tests/internal_rows.rs index 146033e90..7a9760860 100644 --- a/packages/rust/tests/internal_rows.rs +++ b/packages/rust/tests/internal_rows.rs @@ -51,7 +51,7 @@ fn assert_mapping_consistent(db: &Db, table: &str, expected: i64) { #[test] fn mapping_tracks_create_modify_delete() { let db = Db::new().unwrap(); - db.create_table("CREATE TABLE t (id TEXT)").unwrap(); + db.create_table("t", "CREATE TABLE t (id TEXT)").unwrap(); for (i, r) in ["a0", "a1"].iter().enumerate() { db.insert_row("t", &row(r), "a.jsonl", i).unwrap(); @@ -125,7 +125,7 @@ fn table_exists(conn: &Connection, name: &str) -> bool { } fn csv_table() -> Table { - Table::new("CREATE TABLE rows (col TEXT)", "**/*.csv", |path| { + Table::new("rows", "CREATE TABLE rows (col TEXT)", "**/*.csv", |path| { let content = fs::read_to_string(path).unwrap(); content .lines() diff --git a/packages/rust/tests/internal_table_auth.rs b/packages/rust/tests/internal_table_auth.rs index 290ebe553..bc49322f3 100644 --- a/packages/rust/tests/internal_table_auth.rs +++ b/packages/rust/tests/internal_table_auth.rs @@ -14,7 +14,7 @@ use std::path::Path; use tempfile::TempDir; fn user_table() -> Table { - Table::new("CREATE TABLE items (name TEXT)", "*.txt", |path| { + Table::new("items", "CREATE TABLE items (name TEXT)", "*.txt", |path| { let content = std::fs::read_to_string(path).unwrap(); vec![HashMap::from([( "name".into(), diff --git a/packages/rust/tests/multi_config.rs b/packages/rust/tests/multi_config.rs index e7e37376b..008ff038d 100644 --- a/packages/rust/tests/multi_config.rs +++ b/packages/rust/tests/multi_config.rs @@ -39,6 +39,7 @@ fn tables_accumulate_across_config_entries() { cfg_a.path(), r#" [[table]] +name = "alpha" ddl = "CREATE TABLE alpha (basename TEXT)" glob = "a.json" on-file = '''sh -c 'printf "[{\"basename\":\"%s\"}]" "${1##*/}"' sh {path}''' @@ -49,6 +50,7 @@ on-file = '''sh -c 'printf "[{\"basename\":\"%s\"}]" "${1##*/}"' sh {path}''' cfg_b.path(), r#" [[table]] +name = "beta" ddl = "CREATE TABLE beta (basename TEXT)" glob = "b.json" on-file = '''sh -c 'printf "[{\"basename\":\"%s\"}]" "${1##*/}"' sh {path}''' @@ -93,6 +95,7 @@ fn each_on_file_runs_from_its_declaring_config_dir() { cfg_a.path(), r#" [[table]] +name = "alpha" ddl = "CREATE TABLE alpha (v TEXT)" glob = "a.json" on-file = "sh ./emit.sh {path}" @@ -109,6 +112,7 @@ on-file = "sh ./emit.sh {path}" cfg_b.path(), r#" [[table]] +name = "beta" ddl = "CREATE TABLE beta (v TEXT)" glob = "b.json" on-file = "sh ./emit.sh {path}" @@ -152,6 +156,7 @@ fn a_timeout_wrapped_hook_in_one_config_leaves_the_other_untouched() { cfg_a.path(), r#" [[table]] +name = "slow" ddl = "CREATE TABLE slow (v TEXT)" glob = "a.json" on-file = "timeout 0.5 sh ./slow.sh {path}" @@ -168,6 +173,7 @@ on-file = "timeout 0.5 sh ./slow.sh {path}" cfg_b.path(), r#" [[table]] +name = "fast" ddl = "CREATE TABLE fast (v TEXT)" glob = "b.json" on-file = "sh ./fast.sh {path}" @@ -215,6 +221,7 @@ fn ignore_patterns_accumulate_across_config_entries() { ignore = ["**/skip_a/**"] [[table]] +name = "files" ddl = "CREATE TABLE files (basename TEXT)" glob = "**/*.json" on-file = '''sh -c 'printf "[{\"basename\":\"%s\"}]" "${1##*/}"' sh {path}''' @@ -256,6 +263,7 @@ fn duplicate_table_names_across_config_entries_error() { cfg_a.path(), r#" [[table]] +name = "dup" ddl = "CREATE TABLE dup (basename TEXT)" glob = "*.json" on-file = "cat {path}" @@ -266,6 +274,7 @@ on-file = "cat {path}" cfg_b.path(), r#" [[table]] +name = "dup" ddl = "CREATE TABLE dup (basename TEXT)" glob = "*.json" on-file = "cat {path}" diff --git a/packages/rust/tests/no_implicit_files_table.rs b/packages/rust/tests/no_implicit_files_table.rs index d42af38e6..45e5d192d 100644 --- a/packages/rust/tests/no_implicit_files_table.rs +++ b/packages/rust/tests/no_implicit_files_table.rs @@ -18,12 +18,17 @@ fn fixture() -> TempDir { /// A user-declared table that is deliberately *not* named `files`. fn posts_table() -> Table { - Table::new("CREATE TABLE posts (path TEXT)", "**/*.md", |path| { - vec![HashMap::from([( - "path".into(), - Value::Text(path.to_string()), - )])] - }) + Table::new( + "posts", + "CREATE TABLE posts (path TEXT)", + "**/*.md", + |path| { + vec![HashMap::from([( + "path".into(), + Value::Text(path.to_string()), + )])] + }, + ) } #[test] @@ -86,7 +91,7 @@ fn a_configured_table_set_that_omits_files_gets_the_plain_error() { let config = root.path().join("dirsql.toml"); fs::write( &config, - "[[table]]\nddl = \"CREATE TABLE posts (path TEXT)\"\nglob = \"**/*.md\"\non-file = \"cat {path}\"\n", + "[[table]]\nname = \"posts\"\nddl = \"CREATE TABLE posts (path TEXT)\"\nglob = \"**/*.md\"\non-file = \"cat {path}\"\n", ) .unwrap(); diff --git a/packages/rust/tests/on_file.rs b/packages/rust/tests/on_file.rs index e384da9f2..f19dc5afd 100644 --- a/packages/rust/tests/on_file.rs +++ b/packages/rust/tests/on_file.rs @@ -25,6 +25,7 @@ fn on_file_rows_appear_in_query_results() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "papers" ddl = "CREATE TABLE papers (paper_id TEXT, title TEXT, basename TEXT)" glob = "**/meta.json" on-file = "cat {path}" @@ -73,6 +74,7 @@ fn on_file_abspath_token_is_no_longer_substituted() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "items" ddl = "CREATE TABLE items (q TEXT)" glob = "*.json" on-file = "sh echo_args.sh {path} {abspath}" @@ -101,6 +103,7 @@ fn on_file_omitting_path_no_longer_appends_it() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT)" glob = "*.json" on-file = "cat" @@ -137,6 +140,7 @@ fn on_file_receives_absolute_path() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT)" glob = "*.json" on-file = "sh abscheck.sh {path}" @@ -176,6 +180,7 @@ fn on_file_absolute_path_resolves_when_root_differs_from_config_dir() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT)" glob = "**/meta.json" on-file = "sh abscheck.sh {path}" @@ -217,6 +222,7 @@ fn a_failing_command_skips_only_that_file() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT)" glob = "*.txt" on-file = "sh extract.sh {path}" @@ -253,6 +259,7 @@ fn malformed_output_skips_only_that_file() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT)" glob = "*.txt" on-file = "sh extract.sh {path}" @@ -287,6 +294,7 @@ fn a_timeout_wrapped_hook_that_overruns_skips_the_file() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT)" glob = "*.txt" on-file = "timeout 0.5 sh slow.sh {path}" @@ -322,6 +330,7 @@ fn a_slow_unwrapped_hook_runs_to_completion() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT)" glob = "*.txt" on-file = "sh slowish.sh {path}" @@ -356,6 +365,7 @@ fn a_strict_violation_skips_only_that_file() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT)" glob = "*.txt" strict = true @@ -391,6 +401,7 @@ fn build_attempts_every_matched_file_even_after_one_fails() { let built = DirSQL::new( dir.path(), vec![Table::try_new( + "items", "CREATE TABLE items (name TEXT)", "**/*.txt", move |path| { @@ -427,6 +438,7 @@ fn build_reports_every_failing_file_not_only_the_first() { let db = DirSQL::new( dir.path(), vec![Table::try_new( + "items", "CREATE TABLE items (name TEXT)", "**/*.txt", |path| Err(format!("boom for {path}").into()), diff --git a/packages/rust/tests/on_file_e2e.rs b/packages/rust/tests/on_file_e2e.rs index 2df026195..3d54efd62 100644 --- a/packages/rust/tests/on_file_e2e.rs +++ b/packages/rust/tests/on_file_e2e.rs @@ -95,6 +95,7 @@ fn on_file_rows_are_served_over_http() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "papers" ddl = "CREATE TABLE papers (paper_id TEXT, title TEXT)" glob = "**/meta.json" on-file = "cat {path}" @@ -146,6 +147,7 @@ fn on_file_receives_absolute_path_over_http() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "papers" ddl = "CREATE TABLE papers (paper_id TEXT)" glob = "**/meta.json" on-file = "sh abscheck.sh {path}" @@ -192,6 +194,7 @@ fn on_file_absolute_path_resolves_when_config_dir_differs_from_root() { configdir.path().join(".dirsql.toml"), r#" [[table]] +name = "papers" ddl = "CREATE TABLE papers (paper_id TEXT)" glob = "**/meta.json" on-file = "sh abscheck.sh {path}" @@ -235,6 +238,7 @@ fn on_file_abspath_token_is_no_longer_substituted() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "items" ddl = "CREATE TABLE items (q TEXT)" glob = "*.json" on-file = "sh echo_args.sh {path} {abspath}" @@ -268,6 +272,7 @@ fn on_file_omitting_path_yields_no_rows() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT)" glob = "*.json" on-file = "cat" @@ -306,6 +311,7 @@ fn a_file_whose_command_errors_is_skipped_while_the_rest_succeed() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT)" glob = "*.txt" on-file = "sh extract.sh {path}" diff --git a/packages/rust/tests/path_table_query.rs b/packages/rust/tests/path_table_query.rs index 7f7baa1db..844c7b248 100644 --- a/packages/rust/tests/path_table_query.rs +++ b/packages/rust/tests/path_table_query.rs @@ -24,6 +24,7 @@ fn fixture() -> TempDir { /// against ordinary tables. fn csv_table() -> Table { Table::new( + "rows_csv", "CREATE TABLE rows_csv (path TEXT, cells TEXT)", "docs/*.csv", |path| { diff --git a/packages/rust/tests/persist.rs b/packages/rust/tests/persist.rs index 9bf31b399..d8e84722e 100644 --- a/packages/rust/tests/persist.rs +++ b/packages/rust/tests/persist.rs @@ -17,15 +17,20 @@ use tempfile::TempDir; /// Returns a CSV table whose extract function increments `counter` every time /// it runs. Used to verify that warm starts skip extract for unchanged files. fn counting_csv_table(counter: Arc) -> Table { - Table::new("CREATE TABLE rows (col TEXT)", "**/*.csv", move |path| { - let content = std::fs::read_to_string(path).unwrap(); - counter.fetch_add(1, Ordering::SeqCst); - content - .lines() - .skip(1) // header - .map(|line| HashMap::from([("col".into(), Value::Text(line.trim().to_string()))])) - .collect::>() - }) + Table::new( + "rows", + "CREATE TABLE rows (col TEXT)", + "**/*.csv", + move |path| { + let content = std::fs::read_to_string(path).unwrap(); + counter.fetch_add(1, Ordering::SeqCst); + content + .lines() + .skip(1) // header + .map(|line| HashMap::from([("col".into(), Value::Text(line.trim().to_string()))])) + .collect::>() + }, + ) } fn write_csv(root: &Path, name: &str, body_lines: &[&str]) { @@ -241,18 +246,25 @@ fn glob_config_change_forces_full_rebuild() { let csv_counter = Arc::new(AtomicUsize::new(0)); let tsv_counter = Arc::new(AtomicUsize::new(0)); let csv_table = counting_csv_table(csv_counter.clone()); - let tsv_table = Table::new("CREATE TABLE tsv_rows (col TEXT)", "**/*.tsv", { - let c = tsv_counter.clone(); - move |path| { - let content = std::fs::read_to_string(path).unwrap(); - c.fetch_add(1, Ordering::SeqCst); - content - .lines() - .skip(1) - .map(|line| HashMap::from([("col".into(), Value::Text(line.trim().to_string()))])) - .collect::>() - } - }); + let tsv_table = Table::new( + "tsv_rows", + "CREATE TABLE tsv_rows (col TEXT)", + "**/*.tsv", + { + let c = tsv_counter.clone(); + move |path| { + let content = std::fs::read_to_string(path).unwrap(); + c.fetch_add(1, Ordering::SeqCst); + content + .lines() + .skip(1) + .map(|line| { + HashMap::from([("col".into(), Value::Text(line.trim().to_string()))]) + }) + .collect::>() + } + }, + ); let db = DirSQL::builder() .root(root.path()) @@ -357,6 +369,7 @@ fn dirsql_directory_excluded_when_persist_disabled() { /// A counting table over `**/*.csv` with a distinct name/column. fn counting_named_table(name: &'static str, col: &'static str, counter: Arc) -> Table { Table::new( + name, &format!("CREATE TABLE {name} ({col} TEXT)"), "**/*.csv", move |path| { @@ -529,6 +542,7 @@ fn a_hook_failure_commits_the_files_that_parsed() { let db = DirSQL::builder() .root(root.path()) .table(Table::try_new( + "rows", "CREATE TABLE rows (col TEXT)", "**/*.csv", move |path| { diff --git a/packages/rust/tests/query_attach_detach.rs b/packages/rust/tests/query_attach_detach.rs index b08d5898b..c7b110516 100644 --- a/packages/rust/tests/query_attach_detach.rs +++ b/packages/rust/tests/query_attach_detach.rs @@ -17,6 +17,7 @@ fn items_db(root: &Path) -> DirSQL { DirSQL::new( root, vec![Table::new( + "items", "CREATE TABLE items (name TEXT)", "*.txt", |path| { diff --git a/packages/rust/tests/readonly_query.rs b/packages/rust/tests/readonly_query.rs index d492cfcde..43270a9d5 100644 --- a/packages/rust/tests/readonly_query.rs +++ b/packages/rust/tests/readonly_query.rs @@ -19,6 +19,7 @@ fn items_db(root: &Path) -> DirSQL { DirSQL::new( root, vec![Table::new( + "items", "CREATE TABLE items (name TEXT)", "*.txt", |path| { diff --git a/packages/rust/tests/repeatable_config.rs b/packages/rust/tests/repeatable_config.rs index ac88adaa6..918fec649 100644 --- a/packages/rust/tests/repeatable_config.rs +++ b/packages/rust/tests/repeatable_config.rs @@ -22,6 +22,7 @@ fn table_config(name: &str) -> String { format!( r#" [[table]] +name = "{name}" ddl = "CREATE TABLE {name} (basename TEXT)" glob = "*.json" on-file = '''sh -c 'printf "[{{\"basename\":\"%s\"}}]" "${{1##*/}}"' sh {{path}}''' diff --git a/packages/rust/tests/scan_failures.rs b/packages/rust/tests/scan_failures.rs index c054a462d..a1bda01e9 100644 --- a/packages/rust/tests/scan_failures.rs +++ b/packages/rust/tests/scan_failures.rs @@ -27,6 +27,7 @@ fn skipped_files_are_reported_on_the_built_database() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT)" glob = "*.txt" on-file = "sh extract.sh {path}" @@ -58,6 +59,7 @@ fn a_clean_scan_reports_no_skipped_files() { root.path().join(".dirsql.toml"), r#" [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT)" glob = "*.txt" on-file = "printf '[{\"name\":\"ok\"}]'" diff --git a/packages/rust/tests/scan_failures_e2e.rs b/packages/rust/tests/scan_failures_e2e.rs index c3608718e..af4ba15be 100644 --- a/packages/rust/tests/scan_failures_e2e.rs +++ b/packages/rust/tests/scan_failures_e2e.rs @@ -47,6 +47,7 @@ fn fixture(script: &str, config: &str) -> TempDir { const LENIENT_CONFIG: &str = r#" [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT)" glob = "*.txt" on-file = "sh hook.sh {path}" @@ -54,6 +55,7 @@ on-file = "sh hook.sh {path}" const STRICT_CONFIG: &str = r#" [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT)" glob = "*.txt" strict = true @@ -182,6 +184,7 @@ fn a_sqlite_error_still_fails_the_whole_run() { EXTRACT, r#" [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT" glob = "*.txt" on-file = "sh hook.sh {path}" diff --git a/packages/rust/tests/sdk.rs b/packages/rust/tests/sdk.rs index b15b5c9cb..daea35784 100644 --- a/packages/rust/tests/sdk.rs +++ b/packages/rust/tests/sdk.rs @@ -8,6 +8,7 @@ use tempfile::TempDir; fn comments_table() -> Table { Table::new( + "comments", "CREATE TABLE comments (id TEXT, body TEXT, author TEXT)", "comments/**/index.txt", |path| { @@ -37,13 +38,18 @@ fn comments_table() -> Table { } fn items_table() -> Table { - Table::new("CREATE TABLE items (name TEXT)", "**/*.txt", |path| { - let content = std::fs::read_to_string(path).unwrap(); - vec![HashMap::from([( - "name".into(), - Value::Text(content.trim().to_string()), - )])] - }) + Table::new( + "items", + "CREATE TABLE items (name TEXT)", + "**/*.txt", + |path| { + let content = std::fs::read_to_string(path).unwrap(); + vec![HashMap::from([( + "name".into(), + Value::Text(content.trim().to_string()), + )])] + }, + ) } #[test] @@ -96,6 +102,7 @@ fn it_supports_multiple_tables_and_joins() { fs::write(root.path().join("authors").join("alice.txt"), "1|Alice").unwrap(); let posts = Table::new( + "posts", "CREATE TABLE posts (title TEXT, author_id TEXT)", "posts/*.txt", |path| { @@ -119,6 +126,7 @@ fn it_supports_multiple_tables_and_joins() { }, ); let authors = Table::new( + "authors", "CREATE TABLE authors (id TEXT, name TEXT)", "authors/*.txt", |path| { @@ -179,6 +187,7 @@ fn it_ignores_extra_keys_by_default() { let db = DirSQL::new( root.path(), vec![Table::new( + "items", "CREATE TABLE items (name TEXT)", "*.txt", |path| { @@ -211,6 +220,7 @@ fn it_fills_missing_keys_with_null() { let db = DirSQL::new( root.path(), vec![Table::new( + "items", "CREATE TABLE items (name TEXT, color TEXT, count INTEGER)", "*.txt", |path| { @@ -239,6 +249,7 @@ fn it_does_not_inject_stat_columns_the_hook_omits() { let db = DirSQL::new( root.path(), vec![Table::new( + "items", "CREATE TABLE items (path TEXT, size INTEGER)", "*.txt", |_path| vec![HashMap::from([("size".into(), Value::Integer(42))])], @@ -260,6 +271,7 @@ fn it_raises_on_extra_keys_in_strict_mode() { let result = DirSQL::new( root.path(), vec![Table::strict( + "items", "CREATE TABLE items (name TEXT)", "*.txt", |path| { @@ -290,6 +302,7 @@ fn it_reports_missing_keys_in_strict_mode() { let result = DirSQL::new( root.path(), vec![Table::strict( + "items", "CREATE TABLE items (name TEXT, color TEXT)", "*.txt", |path| { @@ -315,6 +328,7 @@ fn it_allows_exact_match_in_strict_mode() { let db = DirSQL::new( root.path(), vec![Table::strict( + "items", "CREATE TABLE items (name TEXT, color TEXT)", "*.txt", |path| { @@ -348,6 +362,7 @@ fn it_round_trips_blob_values_through_the_sdk() { let db = DirSQL::new( root.path(), vec![Table::new( + "blobs", "CREATE TABLE blobs (name TEXT, data BLOB)", "*.json", move |_path| { @@ -428,6 +443,7 @@ fn it_streams_watch_error_events() { let db = DirSQL::new( root.path(), vec![Table::try_new( + "items", "CREATE TABLE items (name TEXT)", "**/*.txt", |_| Err("intentional parse failure".into()), @@ -513,14 +529,19 @@ fn it_splits_scan_and_build_for_async_bindings() { // `finish_build` should call it, once per scanned file. let extract_calls = Arc::new(AtomicUsize::new(0)); let counter = extract_calls.clone(); - let table = Table::new("CREATE TABLE items (name TEXT)", "**/*.txt", move |path| { - let content = std::fs::read_to_string(path).unwrap(); - counter.fetch_add(1, Ordering::SeqCst); - vec![HashMap::from([( - "name".into(), - Value::Text(content.trim().to_string()), - )])] - }); + let table = Table::new( + "items", + "CREATE TABLE items (name TEXT)", + "**/*.txt", + move |path| { + let content = std::fs::read_to_string(path).unwrap(); + counter.fetch_add(1, Ordering::SeqCst); + vec![HashMap::from([( + "name".into(), + Value::Text(content.trim().to_string()), + )])] + }, + ); let prepared = DirSQL::builder() .root(root.path().to_path_buf()) @@ -592,6 +613,7 @@ fn builder_config_loads_tables_with_explicit_root() { &cfg_path, r#" [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT)" glob = "*.json" on-file = "printf '[{}]'" @@ -628,6 +650,7 @@ fn builder_explicit_root_wins_over_config_directory() { &cfg_path, r#" [[table]] +name = "items" ddl = "CREATE TABLE items (basename TEXT)" glob = "*.json" on-file = '''sh -c 'printf "[{\"basename\":\"%s\"}]" "${1##*/}"' sh {path}''' @@ -657,6 +680,7 @@ fn builder_appends_programmatic_tables_to_config_tables() { &cfg_path, r#" [[table]] +name = "items" ddl = "CREATE TABLE items (name TEXT)" glob = "*.json" on-file = "printf '[{}]'" @@ -664,13 +688,18 @@ on-file = "printf '[{}]'" ) .unwrap(); - let notes_table = Table::new("CREATE TABLE notes (body TEXT)", "notes/*.txt", |path| { - let content = std::fs::read_to_string(path).unwrap(); - vec![HashMap::from([( - "body".into(), - Value::Text(content.trim().to_string()), - )])] - }); + let notes_table = Table::new( + "notes", + "CREATE TABLE notes (body TEXT)", + "notes/*.txt", + |path| { + let content = std::fs::read_to_string(path).unwrap(); + vec![HashMap::from([( + "body".into(), + Value::Text(content.trim().to_string()), + )])] + }, + ); let db = DirSQL::builder() .root(root.path()) @@ -754,17 +783,35 @@ fn wait_file_events_after_watch_errors() { } #[test] -fn unparseable_ddl_errors() { +fn ddl_sqlite_rejects_errors() { + let root = TempDir::new().unwrap(); + let table = Table::new("t", "THIS IS NOT A CREATE TABLE", "*.txt", |_| vec![]); + let result = DirSQL::new(root.path(), vec![table]); + assert!(matches!(result, Err(dirsql::DirSqlError::Core(_)))); +} + +/// A `ddl` that runs but creates something else is caught against SQLite's +/// catalog, before any file is ingested. +#[test] +fn declared_name_absent_from_the_catalog_errors() { let root = TempDir::new().unwrap(); - let table = Table::new("THIS IS NOT A CREATE TABLE", "*.txt", |_| vec![]); + let table = Table::new( + "messages", + "CREATE TABLE notes (x TEXT)", + "*.txt", + |_| vec![], + ); let result = DirSQL::new(root.path(), vec![table]); - assert!(matches!(result, Err(dirsql::DirSqlError::Ddl(_)))); + assert!(matches!( + result, + Err(dirsql::DirSqlError::TableNotCreated { ref name }) if name == "messages" + )); } #[test] fn invalid_glob_errors() { let root = TempDir::new().unwrap(); - let table = Table::new("CREATE TABLE t (x TEXT)", "a[b", |_| vec![]); + let table = Table::new("t", "CREATE TABLE t (x TEXT)", "a[b", |_| vec![]); let result = DirSQL::new(root.path(), vec![table]); assert!(matches!(result, Err(dirsql::DirSqlError::Matcher { .. }))); } @@ -777,6 +824,7 @@ fn undeclared_capture_is_dropped() { fs::create_dir_all(root.path().join("logs")).unwrap(); fs::write(root.path().join("logs").join("a.txt"), "x").unwrap(); let table = Table::new( + "entries", "CREATE TABLE entries (path TEXT)", "logs/{kind}.txt", |_| vec![Row::new()], @@ -791,8 +839,8 @@ fn undeclared_capture_is_dropped() { #[test] fn duplicate_table_name_errors() { let root = TempDir::new().unwrap(); - let t1 = Table::new("CREATE TABLE dup (a TEXT)", "*.a", |_| vec![]); - let t2 = Table::new("CREATE TABLE dup (b TEXT)", "*.b", |_| vec![]); + let t1 = Table::new("dup", "CREATE TABLE dup (a TEXT)", "*.a", |_| vec![]); + let t2 = Table::new("dup", "CREATE TABLE dup (b TEXT)", "*.b", |_| vec![]); let result = DirSQL::new(root.path(), vec![t1, t2]); assert!(matches!(result, Err(dirsql::DirSqlError::DuplicateTable(name)) if name == "dup")); } @@ -804,7 +852,7 @@ fn on_file_error_surfaces_as_a_reported_skip() { // can tell *why* rather than only *that*. let root = TempDir::new().unwrap(); fs::write(root.path().join("boom.txt"), "data").unwrap(); - let table = Table::try_new("CREATE TABLE items (name TEXT)", "*.txt", |_| { + let table = Table::try_new("items", "CREATE TABLE items (name TEXT)", "*.txt", |_| { Err("kaboom".into()) }); @@ -832,6 +880,7 @@ fn fanout_root() -> TempDir { fn table_returning(name: &str, glob: &str, col: &'static str, val: &'static str) -> Table { Table::new( + name, &format!("CREATE TABLE {name} ({col} TEXT)"), glob, move |_path| vec![HashMap::from([(col.into(), Value::Text(val.into()))])], @@ -877,6 +926,7 @@ fn fanout_overlapping_distinct_globs_populate_both_tables() { fn capture_column_collision_errors_on_construction() { let root = fanout_root(); let a = Table::new( + "a", "CREATE TABLE a (id TEXT, col_a TEXT)", "data/{id}/metadata.json", |_path| vec![HashMap::from([("col_a".into(), Value::Text("A".into()))])], @@ -905,6 +955,7 @@ fn binary_file_under_glob_does_not_break_build() { .unwrap(); let table = Table::new( + "assets", "CREATE TABLE assets (path TEXT, basename TEXT)", "*.png", |path| { diff --git a/packages/rust/tests/table_name_resolution.rs b/packages/rust/tests/table_name_resolution.rs index 27277c0e5..cfcbc4268 100644 --- a/packages/rust/tests/table_name_resolution.rs +++ b/packages/rust/tests/table_name_resolution.rs @@ -1,10 +1,10 @@ -//! Integration tests for robust table-name resolution. +//! Integration tests for a quoted-identifier DDL. //! -//! dirsql keeps `ddl` as the schema input (bring-your-own DDL, hand-written -//! or emitted by any ORM / schema tool) and needs only the table *name* from -//! it. A **quoted identifier** -- the canonical form emitted by Drizzle / -//! SQLAlchemy / Diesel / sea-query -- must register and be queryable under -//! the bare name. +//! A table's name is declared, but its `ddl` is bring-your-own -- hand-written +//! or emitted by any ORM / schema tool. A **quoted identifier** (the canonical +//! form emitted by Drizzle / SQLAlchemy / Diesel / sea-query) carries SQL +//! *delimiters*, not part of the name, so SQLite records the bare identifier +//! and the declared `name` matches it. use dirsql::{DirSQL, Row, Table, Value}; use std::collections::HashMap; @@ -12,10 +12,11 @@ use std::fs; use tempfile::TempDir; /// The `comments` fixture from `sdk.rs::comments_table`, differing only in -/// the **quoted** table identifier -- anything that breaks is the name -/// resolution, not the surrounding plumbing. +/// the **quoted** table identifier -- anything that breaks is the delimiter +/// handling, not the surrounding plumbing. fn quoted_comments_table() -> Table { Table::new( + "comments", r#"CREATE TABLE "comments" (id TEXT, body TEXT, author TEXT)"#, "comments/**/index.txt", |path| { diff --git a/packages/rust/tests/watch_relative_root.rs b/packages/rust/tests/watch_relative_root.rs index fd690e18e..58d4f3fe1 100644 --- a/packages/rust/tests/watch_relative_root.rs +++ b/packages/rust/tests/watch_relative_root.rs @@ -50,6 +50,7 @@ impl Drop for CwdGuard { fn items_table() -> Table { Table::new( + "items", "CREATE TABLE items (name TEXT, path TEXT)", "**/*.txt", |path| { diff --git a/packages/rust/tests/watch_scan_correctness.rs b/packages/rust/tests/watch_scan_correctness.rs index 8b2564dc8..6c9ec8620 100644 --- a/packages/rust/tests/watch_scan_correctness.rs +++ b/packages/rust/tests/watch_scan_correctness.rs @@ -13,6 +13,7 @@ use std::time::{Duration, Instant}; fn files_table(root: &std::path::Path) -> Table { let root = fs::canonicalize(root).unwrap_or_else(|_| root.to_path_buf()); Table::new( + "files", "CREATE TABLE files (name TEXT, path TEXT)", "**/*", move |path| { diff --git a/packages/rust/tests/watcher.rs b/packages/rust/tests/watcher.rs index a83b90f5d..fdf355448 100644 --- a/packages/rust/tests/watcher.rs +++ b/packages/rust/tests/watcher.rs @@ -136,11 +136,13 @@ fn watch_create_and_delete_fan_out_to_both_tables() { fs::create_dir_all(dir.path().join("data").join("2401.00001")).unwrap(); let ta = Table::new( + "ta", "CREATE TABLE ta (col_a TEXT)", "data/*/metadata.json", |_| vec![HashMap::from([("col_a".into(), Value::Text("A".into()))])], ); let tb = Table::new( + "tb", "CREATE TABLE tb (col_b TEXT)", "data/**/metadata.json", |_| vec![HashMap::from([("col_b".into(), Value::Text("B".into()))])], diff --git a/packages/ts/changelog.d/2026-08-20-declared-table-name.md b/packages/ts/changelog.d/2026-08-20-declared-table-name.md new file mode 100644 index 000000000..890fafdd6 --- /dev/null +++ b/packages/ts/changelog.d/2026-08-20-declared-table-name.md @@ -0,0 +1,7 @@ +**Changed** + +- **A table's name is now declared, not derived from `ddl`.** `TableDef` (and the `Table` class) gains a required `name`, and a `[[table]]` entry in a `.dirsql.toml` requires a `name` key. A `name` its `ddl` does not create fails at load with an error naming the entry. (#962) + +**Removed** + +- **The `parseTableName(ddl)` export.** It wrapped the core's DDL tokenizer, which no longer exists now that a table's name is declared rather than parsed. (#962) diff --git a/packages/ts/e2e-attestations/claude-issue-962-0jjrfv.json b/packages/ts/e2e-attestations/claude-issue-962-0jjrfv.json new file mode 100644 index 000000000..6bd33aa43 --- /dev/null +++ b/packages/ts/e2e-attestations/claude-issue-962-0jjrfv.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1787251654, + "exit_code": 0, + "commit": "e235584fe642aa2561d6b06c6212ebdeb1d3ca5a", + "branch": "claude/issue-962-0jjrfv" +} diff --git a/packages/ts/migrations.d/2026-08-20-declared-table-name.md b/packages/ts/migrations.d/2026-08-20-declared-table-name.md new file mode 100644 index 000000000..affacfca4 --- /dev/null +++ b/packages/ts/migrations.d/2026-08-20-declared-table-name.md @@ -0,0 +1,54 @@ +### `TableDef` requires an explicit `name`; `parseTableName` removed (#962) + +#### Summary + +A table's SQL name was derived by parsing the `CREATE TABLE` head of `ddl`, and +that parser was exposed as the top-level `parseTableName(ddl)` export. The name +is now **declared**: `TableDef` (and the `Table` class) requires a `name`, and a +`[[table]]` entry in a `.dirsql.toml` requires a `name` key. Whether the `ddl` +creates a table by that name is checked against SQLite's catalog at load time, +so the parser — and its export — are gone. + +#### Required changes + +| Before | After | Fix | +| ------ | ----- | --- | +| `{ ddl, glob, onFile }` / `new Table({ ddl, glob, onFile })` | `Property 'name' is missing in type … but required in type 'TableDef'` | Add `name: "
"`, matching the table the `ddl` creates. | +| `[[table]]` with `ddl` + `glob` + `on-file` | `await db.ready` rejects: `Missing required field 'name' in [[table]] entry` | Add `name = "
"` to the entry. | +| `import { parseTableName } from "dirsql"` | Export no longer exists | Use the name you declared on the table. | + +#### Deprecations removed + +- `parseTableName(ddl)` — removed outright, along with the core DDL tokenizer + it wrapped. + +#### Behavior changes without code changes + +- A `[[table]]` whose `ddl` creates a table under a different name than its + `name` now rejects at `ready` with an error naming the entry + (`table 'messages': …`), before any file is ingested. +- Quoted (`CREATE TABLE "messages"`), schema-qualified and `IF NOT EXISTS` DDL + keep working unchanged: SQLite records the bare identifier, so a plain + `name: "messages"` matches. + +#### Verification + +```typescript +import { readFileSync } from "node:fs"; +import { DirSQL } from "dirsql"; + +const db = new DirSQL({ + root: "./data-root", + tables: [ + { + name: "records", + ddl: "CREATE TABLE records (id TEXT)", + glob: "data/*.json", + onFile: (path) => JSON.parse(readFileSync(path, "utf8")), + }, + ], +}); +await db.query("SELECT id FROM records"); +// -> the file's rows. Omitting `name` fails to typecheck; a `name` the ddl +// never creates rejects `db.ready`. +``` diff --git a/packages/ts/napi/src/lib.rs b/packages/ts/napi/src/lib.rs index 284410f03..64e9d6bf9 100644 --- a/packages/ts/napi/src/lib.rs +++ b/packages/ts/napi/src/lib.rs @@ -20,7 +20,7 @@ use dirsql::{ DirSQL as CoreDirSQL, Extension, PreparedBuild, RawFileEvent, Row, RowEvent as CoreRowEvent, - Table, Value, db::parse_table_name as core_parse_table_name, + Table, Value, }; use napi::Task; use napi::bindgen_prelude::*; @@ -31,14 +31,6 @@ use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; -/// Parse the table name out of a `CREATE TABLE (...)` DDL. -/// Returns `null` if the DDL doesn't match (e.g. empty, missing -/// `CREATE TABLE`, or the identifier slot is empty). -#[napi(js_name = "parseTableName")] -pub fn parse_table_name(ddl: String) -> Option { - core_parse_table_name(&ddl) -} - /// Run the `dirsql` CLI in this process and return its exit code. /// /// The npm launcher calls this instead of spawning a bundled binary, so the @@ -404,13 +396,14 @@ fn parse_tables_from_js(env: Env, tables: Array<'_>) -> Result> { })?; let raw_obj = table_element.raw(); + let name = unsafe { get_string_property(raw_env, raw_obj, "name")? }; let ddl = unsafe { get_string_property(raw_env, raw_obj, "ddl")? }; let glob = unsafe { get_string_property(raw_env, raw_obj, "glob")? }; let on_file_val = unsafe { get_function_property(raw_env, raw_obj, "onFile")? }; let strict = unsafe { get_bool_property(raw_env, raw_obj, "strict", false) }; let fn_ref = unsafe { Arc::new(FnRef::new(raw_env, on_file_val)?) }; - let mut table = Table::try_new(ddl, glob, make_on_file_closure(fn_ref)); + let mut table = Table::try_new(name, ddl, glob, make_on_file_closure(fn_ref)); table.strict = strict; rust_tables.push(table); } diff --git a/packages/ts/src/core.test.ts b/packages/ts/src/core.test.ts index 74bea17b9..fd36e32b7 100644 --- a/packages/ts/src/core.test.ts +++ b/packages/ts/src/core.test.ts @@ -12,7 +12,6 @@ describe("getCore", () => { it("lazily loads the native core on first access and caches it", () => { const fake = { DirSQL: {}, - parseTableName: () => null, } as ReturnType; vi.mocked(loadNativeCore).mockReturnValue(fake); diff --git a/packages/ts/src/core.ts b/packages/ts/src/core.ts index 19cf42672..c2b711352 100644 --- a/packages/ts/src/core.ts +++ b/packages/ts/src/core.ts @@ -34,7 +34,6 @@ export interface NativeDirSQLConstructor { // native binary (`dirsql.node`); tests may substitute a fake. export interface CoreModule { DirSQL: NativeDirSQLConstructor; - parseTableName(ddl: string): string | null; } // Unit tests `vi.mock("./core.js")` to fake `getCore` directly, so diff --git a/packages/ts/src/dirsql.test.ts b/packages/ts/src/dirsql.test.ts index 8e42ab3ec..53849970d 100644 --- a/packages/ts/src/dirsql.test.ts +++ b/packages/ts/src/dirsql.test.ts @@ -77,7 +77,12 @@ describe("DirSQL", () => { const openAsync = installFakeCore(makeInner()); const onFile = () => []; const tables = [ - { ddl: "CREATE TABLE t (n INTEGER)", glob: "**/*.json", onFile }, + { + name: "t", + ddl: "CREATE TABLE t (n INTEGER)", + glob: "**/*.json", + onFile, + }, ]; const db = new DirSQL({ root: "/data", tables }); await db.ready; diff --git a/packages/ts/src/index.test.ts b/packages/ts/src/index.test.ts index 3b3535e5e..4d10a4b4e 100644 --- a/packages/ts/src/index.test.ts +++ b/packages/ts/src/index.test.ts @@ -6,7 +6,6 @@ describe("public barrel", () => { it("re-exports the runtime values", () => { expect(typeof api.DirSQL).toBe("function"); expect(typeof api.Table).toBe("function"); - expect(typeof api.parseTableName).toBe("function"); }); it("re-exports the ScanFailure type with its documented shape (#715)", () => { @@ -19,8 +18,6 @@ describe("public barrel", () => { }); it("exposes exactly the public runtime exports", () => { - expect(Object.keys(api).sort()).toEqual( - ["DirSQL", "Table", "parseTableName"].sort(), - ); + expect(Object.keys(api).sort()).toEqual(["DirSQL", "Table"].sort()); }); }); diff --git a/packages/ts/src/index.ts b/packages/ts/src/index.ts index be3603d6e..3a209118e 100644 --- a/packages/ts/src/index.ts +++ b/packages/ts/src/index.ts @@ -7,6 +7,5 @@ export type { RowEvent, ScanFailure, } from "./dirsql.js"; -export { parseTableName } from "./parse-table-name.js"; export { Table } from "./table.js"; export type { TableDef } from "./table.js"; diff --git a/packages/ts/src/parse-table-name.test.ts b/packages/ts/src/parse-table-name.test.ts deleted file mode 100644 index a7059aacd..000000000 --- a/packages/ts/src/parse-table-name.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { getCore } from "./core.js"; -import { parseTableName } from "./parse-table-name.js"; - -vi.mock("./core.js"); - -describe("parseTableName", () => { - it("delegates to the core parser and returns its result", () => { - const parse = vi.fn().mockReturnValue("users"); - vi.mocked(getCore).mockReturnValue({ - parseTableName: parse, - } as unknown as ReturnType); - - expect(parseTableName("CREATE TABLE users (a INTEGER)")).toBe("users"); - expect(parse).toHaveBeenCalledWith("CREATE TABLE users (a INTEGER)"); - }); - - it("propagates null for an unrecognized DDL", () => { - const parse = vi.fn().mockReturnValue(null); - vi.mocked(getCore).mockReturnValue({ - parseTableName: parse, - } as unknown as ReturnType); - - expect(parseTableName("not ddl")).toBeNull(); - }); -}); diff --git a/packages/ts/src/parse-table-name.ts b/packages/ts/src/parse-table-name.ts deleted file mode 100644 index fbc5c5959..000000000 --- a/packages/ts/src/parse-table-name.ts +++ /dev/null @@ -1,17 +0,0 @@ -// `parseTableName` — table-name resolution delegated to the Rust core. -// -// Split out of the public barrel (`index.ts`) so it carries a colocated -// unit test instead of an exemption (#239). - -import { getCore } from "./core.js"; - -/** - * Parse the table name out of a `CREATE TABLE (...)` DDL. - * - * Returns `null` for a DDL the parser doesn't recognize. Backed by the - * core Rust implementation (`dirsql::db::parse_table_name`) so the JS - * SDK and the orchestrator agree on table-name resolution. - */ -export function parseTableName(ddl: string): string | null { - return getCore().parseTableName(ddl); -} diff --git a/packages/ts/src/resolve-config-extensions.test.ts b/packages/ts/src/resolve-config-extensions.test.ts index 7689cdbe8..87f77b769 100644 --- a/packages/ts/src/resolve-config-extensions.test.ts +++ b/packages/ts/src/resolve-config-extensions.test.ts @@ -49,7 +49,9 @@ describe("resolveConfigExtensionSpecs", () => { }); it("returns null when the config has no [dirsql] section at all", () => { - stubConfig({ table: [{ ddl: "CREATE TABLE t (x TEXT)", glob: "*" }] }); + stubConfig({ + table: [{ name: "t", ddl: "CREATE TABLE t (x TEXT)", glob: "*" }], + }); expect(resolveConfigExtensionSpecs("/x/.dirsql.toml")).toBeNull(); }); @@ -126,6 +128,7 @@ describe("resolveConfigsExtensionSpecs", () => { dirsql: { extension: [{ path: "ext/a.so" }] }, } as ReturnType) .mockReturnValueOnce({ + name: "t", table: [{ ddl: "CREATE TABLE t (x TEXT)", glob: "*" }], } as ReturnType); expect(resolveConfigsExtensionSpecs(["/a.toml", "/b.toml"])).toBeNull(); diff --git a/packages/ts/src/table.test.ts b/packages/ts/src/table.test.ts index 9b9beb700..85e14fffd 100644 --- a/packages/ts/src/table.test.ts +++ b/packages/ts/src/table.test.ts @@ -4,26 +4,38 @@ import { Table } from "./table.js"; describe("Table", () => { const onFile = () => [{ a: 1 }]; - it("copies ddl, glob, and onFile onto the instance", () => { + it("copies name, ddl, glob, and onFile onto the instance", () => { const t = new Table({ + name: "t", ddl: "CREATE TABLE t (a INTEGER)", glob: "*.json", onFile, }); + expect(t.name).toBe("t"); expect(t.ddl).toBe("CREATE TABLE t (a INTEGER)"); expect(t.glob).toBe("*.json"); expect(t.onFile).toBe(onFile); }); it("copies strict when present", () => { - const t = new Table({ ddl: "d", glob: "g", onFile, strict: true }); + const t = new Table({ + name: "t", + ddl: "d", + glob: "g", + onFile, + strict: true, + }); expect(t.strict).toBe(true); - expect(Object.keys(t).sort()).toEqual(["ddl", "glob", "onFile", "strict"]); + expect(Object.keys(t).sort()).toEqual( + ["name", "ddl", "glob", "onFile", "strict"].sort(), + ); }); it("omits strict from enumerable keys when absent", () => { - const t = new Table({ ddl: "d", glob: "g", onFile }); + const t = new Table({ name: "t", ddl: "d", glob: "g", onFile }); expect(t.strict).toBeUndefined(); - expect(Object.keys(t).sort()).toEqual(["ddl", "glob", "onFile"]); + expect(Object.keys(t).sort()).toEqual( + ["name", "ddl", "glob", "onFile"].sort(), + ); }); }); diff --git a/packages/ts/src/table.ts b/packages/ts/src/table.ts index 836ca83bc..85d6b0d5b 100644 --- a/packages/ts/src/table.ts +++ b/packages/ts/src/table.ts @@ -5,6 +5,11 @@ /** Definition of a SQL-indexed table backed by files on disk. */ export interface TableDef { + /** + * The table's SQL name, declared rather than derived from `ddl`. The `ddl` + * must create a table by this name; a mismatch is a load-time error. + */ + name: string; /** SQL DDL statement, e.g. `CREATE TABLE users (name TEXT, age INTEGER)`. */ ddl: string; /** Glob pattern (relative to the DirSQL root) for files backing this table. */ @@ -22,7 +27,7 @@ export interface TableDef { /** * Thin class wrapper around {@link TableDef} for parity with the Python - * `Table(ddl=..., glob=..., on_file=...)` and Rust `Table::new(...)` + * `Table(name=..., ddl=..., glob=..., on_file=...)` and Rust `Table::new(...)` * constructors. `new Table({...})` is structurally identical to a plain * object literal satisfying `TableDef` — anything accepting `TableDef[]` * (e.g. {@link DirSQL}'s `tables` option) takes either form. @@ -33,12 +38,14 @@ export interface TableDef { * `strict` to `undefined` under `useDefineForClassFields`. */ export class Table implements TableDef { + declare readonly name: string; declare readonly ddl: string; declare readonly glob: string; declare readonly onFile: (filePath: string) => Record[]; declare readonly strict?: boolean; constructor(def: TableDef) { + this.name = def.name; this.ddl = def.ddl; this.glob = def.glob; this.onFile = def.onFile; diff --git a/packages/ts/tests/e2e/declared-table-name.test.ts b/packages/ts/tests/e2e/declared-table-name.test.ts new file mode 100644 index 000000000..2d8f499e5 --- /dev/null +++ b/packages/ts/tests/e2e/declared-table-name.test.ts @@ -0,0 +1,139 @@ +// CLI e2e: the declared `[[table]] name` key through the real launcher. +// +// A table's name is declared, never derived from `ddl`. The launcher must +// query a config table under its declared `name`, exit non-zero when a +// `[[table]]` entry omits `name`, and exit non-zero when the entry's `ddl` +// never creates that name. Nothing is mocked: real launcher, real process, +// real filesystem, real SQLite, real `on-file` command spawn. + +import { spawn } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const PKG_ROOT = join(__dirname, "..", ".."); + +const PKG: { bin: { dirsql: string } } = JSON.parse( + await readFile(join(PKG_ROOT, "package.json"), "utf8"), +); +const CLI_ENTRY = join(PKG_ROOT, PKG.bin.dirsql); + +interface RunResult { + code: number | null; + stderr: string; + stdout: string; +} + +function runLauncher(args: string[], cwd: string): Promise { + return new Promise((resolve) => { + const proc = spawn(process.execPath, [CLI_ENTRY, ...args], { + cwd, + stdio: ["pipe", "pipe", "pipe"], + }); + let stderr = ""; + let stdout = ""; + proc.stderr.setEncoding("utf8"); + proc.stdout.setEncoding("utf8"); + proc.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + proc.stdout.on("data", (chunk: string) => { + stdout += chunk; + }); + proc.stdin.end(); + proc.on("close", (code) => resolve({ code, stderr, stdout })); + setTimeout(() => { + if (proc.exitCode === null) { + proc.kill("SIGKILL"); + } + }, 20_000); + }); +} + +describe("declared [[table]] name (CLI)", () => { + let dir: string; + let configPath: string; + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "dirsql-declared-name-e2e-")); + configPath = join(dir, ".dirsql.toml"); + await mkdir(join(dir, "data"), { recursive: true }); + await writeFile( + join(dir, "data", "a.json"), + '[{"id": "one"}, {"id": "two"}]', + ); + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it("queries a table under its declared name", async () => { + await writeFile( + configPath, + ` +[[table]] +name = "records" +ddl = "CREATE TABLE records (id TEXT)" +glob = "data/*.json" +on-file = "cat {path}" +`, + ); + + const { code, stdout, stderr } = await runLauncher( + ["query", "SELECT id FROM records ORDER BY id", "--config", configPath], + dir, + ); + + expect(code, `expected a clean run, stderr: ${stderr}`).toBe(0); + expect((JSON.parse(stdout) as { id: string }[]).map((r) => r.id)).toEqual([ + "one", + "two", + ]); + }, 30_000); + + it("exits non-zero when a [[table]] entry has no name", async () => { + await writeFile( + configPath, + ` +[[table]] +ddl = "CREATE TABLE records (id TEXT)" +glob = "data/*.json" +on-file = "cat {path}" +`, + ); + + const { code, stderr } = await runLauncher( + ["query", "SELECT id FROM records", "--config", configPath], + dir, + ); + + expect(code).not.toBe(0); + expect(stderr).toContain("name"); + expect(stderr).toContain("[[table]]"); + }, 30_000); + + it("exits non-zero when the ddl never creates the declared name", async () => { + await writeFile( + configPath, + ` +[[table]] +name = "messages" +ddl = "CREATE TABLE records (id TEXT)" +glob = "data/*.json" +on-file = "cat {path}" +`, + ); + + const { code, stderr } = await runLauncher( + ["query", "SELECT id FROM messages", "--config", configPath], + dir, + ); + + expect(code).not.toBe(0); + expect(stderr).toContain("table 'messages'"); + }, 30_000); +}); diff --git a/packages/ts/tests/e2e/extension-package.test.ts b/packages/ts/tests/e2e/extension-package.test.ts index b4b621b66..18d770522 100644 --- a/packages/ts/tests/e2e/extension-package.test.ts +++ b/packages/ts/tests/e2e/extension-package.test.ts @@ -266,7 +266,7 @@ describe("dirsql CLI --config: extension by package name (#299)", () => { await writeFile(join(dir, "a.txt"), "x"); await writeFile( join(dir, ".dirsql.toml"), - `[[dirsql.extension]]\npath = "${PKG_NAME}"\nentrypoint = "sqlite3_extension_init"\n\n[[table]]\nddl = "CREATE TABLE files (path TEXT)"\nglob = "*.txt"\non-file = "cat {path}"\n`, + `[[dirsql.extension]]\npath = "${PKG_NAME}"\nentrypoint = "sqlite3_extension_init"\n\n[[table]]\nname = "files"\nddl = "CREATE TABLE files (path TEXT)"\nglob = "*.txt"\non-file = "cat {path}"\n`, ); const rows = await runServerAndQuery(join(dir, ".dirsql.toml")); expect(rows).toEqual([{ a: 42 }]); diff --git a/packages/ts/tests/e2e/fanout.test.ts b/packages/ts/tests/e2e/fanout.test.ts index 1922109ee..54b56806d 100644 --- a/packages/ts/tests/e2e/fanout.test.ts +++ b/packages/ts/tests/e2e/fanout.test.ts @@ -161,7 +161,7 @@ beforeAll(async () => { const pathHook = `on-file = '''sh -c 'rel=\${1#"$2"/}; printf "[{\\"path\\":\\"%s\\"}]" "$rel"' sh {path} {root}'''`; await writeFile( cfg, - `[[table]]\nddl = "CREATE TABLE ta (path TEXT)"\nglob = "data/*/metadata.json"\n${pathHook}\n\n[[table]]\nddl = "CREATE TABLE tb (path TEXT)"\nglob = "data/**/metadata.json"\n${pathHook}\n`, + `[[table]]\nname = "ta"\nddl = "CREATE TABLE ta (path TEXT)"\nglob = "data/*/metadata.json"\n${pathHook}\n\n[[table]]\nname = "tb"\nddl = "CREATE TABLE tb (path TEXT)"\nglob = "data/**/metadata.json"\n${pathHook}\n`, ); serverPort = await freePort(); diff --git a/packages/ts/tests/e2e/query-contract.test.ts b/packages/ts/tests/e2e/query-contract.test.ts index bf0883f3a..39c5b0fa1 100644 --- a/packages/ts/tests/e2e/query-contract.test.ts +++ b/packages/ts/tests/e2e/query-contract.test.ts @@ -176,7 +176,7 @@ beforeAll(async () => { const pathHook = `on-file = '''sh -c 'rel=\${1#"$2"/}; printf "[{\\"path\\":\\"%s\\"}]" "$rel"' sh {path} {root}'''`; await writeFile( cfg, - `[[table]]\nddl = "CREATE TABLE files (path TEXT)"\nglob = "*.txt"\n${pathHook}\n`, + `[[table]]\nname = "files"\nddl = "CREATE TABLE files (path TEXT)"\nglob = "*.txt"\n${pathHook}\n`, ); serverPort = await freePort(); diff --git a/packages/ts/tests/integration/binding/declared-table-name.test.ts b/packages/ts/tests/integration/binding/declared-table-name.test.ts new file mode 100644 index 000000000..72e740f0b --- /dev/null +++ b/packages/ts/tests/integration/binding/declared-table-name.test.ts @@ -0,0 +1,84 @@ +// Binding-tier tests (real core, real fs) for the declared table `name`. +// +// A table's name is declared, never derived from `ddl`. A `[[table]]` entry +// without `name` fails to load, and a `name` the entry's `ddl` never creates +// fails at load time -- checked against SQLite's own catalog, before any +// ingestion. + +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { DirSQL } from "dirsql"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +// Emits the file's root-relative `path`. `{path}` is the absolute path, +// `{root}` the index root. +const pathHook = `on-file = '''sh -c 'rel=\${1#"$2"/}; printf "[{\\"path\\":\\"%s\\"}]" "$rel"' sh {path} {root}'''`; + +async function seedFile(path: string, content: string): Promise { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, content); +} + +describe("declared [[table]] name", () => { + let dir: string; + let configPath: string; + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "dirsql-declared-name-")); + configPath = join(dir, ".dirsql.toml"); + await seedFile(join(dir, "data", "a.csv"), "anything"); + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it("registers a config table under its declared name", async () => { + await seedFile( + configPath, + ` +[[table]] +name = "notes" +ddl = "CREATE TABLE notes (path TEXT)" +glob = "data/*.csv" +${pathHook} +`, + ); + + const db = new DirSQL({ root: dir, config: configPath }); + const rows = await db.query("SELECT path FROM notes"); + expect(rows.map((r) => r.path)).toEqual(["data/a.csv"]); + }); + + it("rejects a [[table]] entry with no name", async () => { + await seedFile( + configPath, + ` +[[table]] +ddl = "CREATE TABLE notes (path TEXT)" +glob = "data/*.csv" +${pathHook} +`, + ); + + const db = new DirSQL({ root: dir, config: configPath }); + await expect(db.ready).rejects.toThrow(/name/); + }); + + it("rejects a declared name the ddl never creates", async () => { + await seedFile( + configPath, + ` +[[table]] +name = "messages" +ddl = "CREATE TABLE notes (path TEXT)" +glob = "data/*.csv" +${pathHook} +`, + ); + + const db = new DirSQL({ root: dir, config: configPath }); + await expect(db.ready).rejects.toThrow(/table 'messages'/); + }); +}); diff --git a/packages/ts/tests/integration/binding/fanout.test.ts b/packages/ts/tests/integration/binding/fanout.test.ts index 4e58e54fe..4cbdd1881 100644 --- a/packages/ts/tests/integration/binding/fanout.test.ts +++ b/packages/ts/tests/integration/binding/fanout.test.ts @@ -26,11 +26,13 @@ describe("DirSQL fan-out", () => { root: dir, tables: [ { + name: "ta", ddl: "CREATE TABLE ta (col_a TEXT)", glob: "data/*/metadata.json", onFile: () => [{ col_a: "A" }], }, { + name: "tb", ddl: "CREATE TABLE tb (col_b TEXT)", glob: "data/*/metadata.json", onFile: () => [{ col_b: "B" }], @@ -49,11 +51,13 @@ describe("DirSQL fan-out", () => { root: dir, tables: [ { + name: "ta", ddl: "CREATE TABLE ta (col_a TEXT)", glob: "data/*/metadata.json", onFile: () => [{ col_a: "A" }], }, { + name: "tb", ddl: "CREATE TABLE tb (col_b TEXT)", glob: "data/**/metadata.json", onFile: () => [{ col_b: "B" }], @@ -71,6 +75,7 @@ describe("DirSQL fan-out", () => { root: dir, tables: [ { + name: "a", ddl: "CREATE TABLE a (id TEXT, col_a TEXT)", glob: "data/{id}/metadata.json", onFile: () => [{ col_a: "A" }], diff --git a/packages/ts/tests/integration/binding/from-config.test.ts b/packages/ts/tests/integration/binding/from-config.test.ts index c7fc85584..a2a5af6f4 100644 --- a/packages/ts/tests/integration/binding/from-config.test.ts +++ b/packages/ts/tests/integration/binding/from-config.test.ts @@ -49,6 +49,7 @@ describe("new DirSQL(configPath)", () => { configPath, ` [[table]] +name = "items" ddl = "CREATE TABLE items (path TEXT, basename TEXT)" glob = "items/*.csv" ${pathBasenameHook} @@ -73,6 +74,7 @@ ${pathBasenameHook} configPath, ` [[table]] +name = "comments" ddl = "CREATE TABLE comments (thread_id TEXT, basename TEXT)" glob = "comments/{thread_id}/*.txt" ${pathBasenameHook} @@ -90,6 +92,7 @@ ${pathBasenameHook} configPath, ` [[table]] +name = "comments" ddl = "CREATE TABLE comments (path TEXT, basename TEXT)" glob = "comments/{thread_id}/*.txt" ${pathBasenameHook} @@ -114,6 +117,7 @@ ${pathBasenameHook} configPath, ` [[table]] +name = "files" ddl = "CREATE TABLE files (path TEXT, basename TEXT, dir TEXT, ext TEXT, size INTEGER, mtime INTEGER)" glob = "docs/*.md" ${statHook} @@ -146,6 +150,7 @@ ${statHook} ignore = ["**/node_modules/**"] [[table]] +name = "items" ddl = "CREATE TABLE items (path TEXT)" glob = "data/**/*.json" ${pathHook} @@ -166,11 +171,13 @@ ${pathHook} configPath, ` [[table]] +name = "posts" ddl = "CREATE TABLE posts (basename TEXT)" glob = "posts/*.txt" ${basenameHook} [[table]] +name = "authors" ddl = "CREATE TABLE authors (basename TEXT)" glob = "authors/*.txt" ${basenameHook} @@ -204,6 +211,7 @@ ${basenameHook} configPath, ` [[table]] +name = "files" ddl = "CREATE TABLE files (path TEXT, size INTEGER)" glob = "**/*.md" `, @@ -217,6 +225,7 @@ glob = "**/*.md" configPath, ` [[table]] +name = "t" glob = "*.json" `, ); diff --git a/packages/ts/tests/integration/binding/index.test.ts b/packages/ts/tests/integration/binding/index.test.ts index d2567c58c..dd0614764 100644 --- a/packages/ts/tests/integration/binding/index.test.ts +++ b/packages/ts/tests/integration/binding/index.test.ts @@ -38,6 +38,7 @@ describe("DirSQL", () => { root: dir, tables: [ { + name: "users", ddl: "CREATE TABLE users (name TEXT, age INTEGER)", glob: "data/users.json", onFile: (filePath: string) => @@ -59,12 +60,14 @@ describe("DirSQL", () => { root: dir, tables: [ { + name: "users", ddl: "CREATE TABLE users (name TEXT, age INTEGER)", glob: "data/users.json", onFile: (filePath: string) => JSON.parse(readFileSync(filePath, "utf8")), }, { + name: "products", ddl: "CREATE TABLE products (name TEXT, price REAL)", glob: "data/products.json", onFile: (filePath: string) => @@ -91,6 +94,7 @@ describe("DirSQL", () => { root: dir, tables: [ { + name: "orders", ddl: 'CREATE TABLE orders (name TEXT, "order" INTEGER)', glob: "data/orders.json", onFile: (filePath: string) => @@ -110,6 +114,7 @@ describe("DirSQL", () => { root: dir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT)", glob: "data/*.json", onFile: (filePath: string) => @@ -131,6 +136,7 @@ describe("DirSQL", () => { root: dir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT)", glob: "data/*.json", onFile: (filePath: string) => @@ -153,6 +159,7 @@ describe("DirSQL", () => { root: dir, tables: [ { + name: "users", ddl: "CREATE TABLE users (name TEXT, age INTEGER)", glob: "data/users.json", onFile: (filePath: string) => @@ -173,6 +180,7 @@ describe("DirSQL", () => { root: emptyDir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT)", glob: "**/*.json", onFile: (filePath: string) => @@ -193,6 +201,7 @@ describe("DirSQL", () => { root: dir, tables: [ { + name: "users", ddl: "CREATE TABLE users (name TEXT)", glob: "data/users.json", onFile: (filePath: string) => @@ -213,6 +222,7 @@ describe("DirSQL", () => { root: dir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT)", glob: "items/*.json", onFile: (filePath: string) => [ @@ -248,6 +258,7 @@ describe("DirSQL", () => { root: dir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT)", glob: "items/*.json", onFile: (filePath: string) => [ @@ -275,6 +286,7 @@ describe("DirSQL", () => { root: dir, tables: [ { + name: "t", ddl: "NOT VALID SQL", glob: "**/*.json", onFile: () => [], @@ -308,6 +320,7 @@ describe("DirSQL strict mode", () => { root: dir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT)", glob: "items/*.json", onFile: (filePath: string) => [ @@ -334,6 +347,7 @@ describe("DirSQL strict mode", () => { root: dir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT, color TEXT)", glob: "items/*.json", onFile: (filePath: string) => [ @@ -360,6 +374,7 @@ describe("DirSQL strict mode", () => { root: dir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT, color TEXT)", glob: "items/*.json", onFile: (filePath: string) => [ @@ -402,6 +417,7 @@ describe("DirSQL relaxed schema (default)", () => { root: dir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT)", glob: "*.json", onFile: readJson, @@ -420,6 +436,7 @@ describe("DirSQL relaxed schema (default)", () => { root: dir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT, color TEXT)", glob: "*.json", onFile: readJson, @@ -451,6 +468,7 @@ describe("DirSQL onFile path argument", () => { root: dir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT)", glob: "*.json", onFile: (filePath: string) => { @@ -487,6 +505,7 @@ describe("DirSQL watch events", () => { root: dir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT)", glob: "**/*.json", onFile: (filePath: string) => [ @@ -520,6 +539,7 @@ describe("DirSQL watch events", () => { root: dir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT)", glob: "**/*.json", onFile: (filePath: string) => [ @@ -552,6 +572,7 @@ describe("DirSQL watch events", () => { root: dir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT)", glob: "**/*.json", onFile: (filePath: string) => [ @@ -585,6 +606,7 @@ describe("DirSQL watch events", () => { root: dir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT)", glob: "items/*.json", onFile: (filePath: string) => [ @@ -614,6 +636,7 @@ describe("DirSQL watch events", () => { root: dir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT)", glob: "*.json", onFile: (filePath: string) => [ @@ -650,6 +673,7 @@ describe("Table class", () => { it("is importable as a constructable class", () => { const t = new Table({ + name: "users", ddl: "CREATE TABLE users (name TEXT, age INTEGER)", glob: "data/users.json", onFile: (filePath: string) => JSON.parse(readFileSync(filePath, "utf8")), @@ -662,6 +686,7 @@ describe("Table class", () => { it("instances are assignable to TableDef and accepted by DirSQL", async () => { const tableInstance: TableDef = new Table({ + name: "users", ddl: "CREATE TABLE users (name TEXT, age INTEGER)", glob: "data/users.json", onFile: (filePath: string) => JSON.parse(readFileSync(filePath, "utf8")), @@ -681,16 +706,17 @@ describe("Table class", () => { it("plain object and Table instance produce identical query behavior", async () => { const onFile = (filePath: string) => JSON.parse(readFileSync(filePath, "utf8")); + const name = "users"; const ddl = "CREATE TABLE users (name TEXT, age INTEGER)"; const glob = "data/users.json"; const dbFromLiteral = new DirSQL({ root: dir, - tables: [{ ddl, glob, onFile }], + tables: [{ name, ddl, glob, onFile }], }); const dbFromClass = new DirSQL({ root: dir, - tables: [new Table({ ddl, glob, onFile })], + tables: [new Table({ name, ddl, glob, onFile })], }); const literalRows = await dbFromLiteral.query( @@ -704,6 +730,7 @@ describe("Table class", () => { it("has the same enumerable keys as the equivalent plain object literal", () => { const def = { + name: "users", ddl: "CREATE TABLE users (name TEXT)", glob: "data/users.json", onFile: (filePath: string) => JSON.parse(readFileSync(filePath, "utf8")), @@ -718,6 +745,7 @@ describe("Table class", () => { JSON.stringify([{ name: "Alice", age: 30, extra: "nope" }]), ); const t = new Table({ + name: "users", ddl: "CREATE TABLE users (name TEXT, age INTEGER)", glob: "data/users.json", onFile: (filePath: string) => JSON.parse(readFileSync(filePath, "utf8")), diff --git a/packages/ts/tests/integration/binding/on-file-rename.test.ts b/packages/ts/tests/integration/binding/on-file-rename.test.ts index 2fbd28e7b..0fdbacc68 100644 --- a/packages/ts/tests/integration/binding/on-file-rename.test.ts +++ b/packages/ts/tests/integration/binding/on-file-rename.test.ts @@ -32,6 +32,7 @@ describe("the onFile table seam against the real core", () => { root: dir, tables: [ { + name: "users", ddl: "CREATE TABLE users (name TEXT, age INTEGER)", glob: "data/users.json", onFile: (filePath: string) => diff --git a/packages/ts/tests/integration/binding/path-table.test.ts b/packages/ts/tests/integration/binding/path-table.test.ts index 2db516e92..80eb737e3 100644 --- a/packages/ts/tests/integration/binding/path-table.test.ts +++ b/packages/ts/tests/integration/binding/path-table.test.ts @@ -16,6 +16,7 @@ describe("path-tables (#627)", () => { root: dir, tables: [ { + name: "rows_csv", ddl: "CREATE TABLE rows_csv (path TEXT)", glob: "docs/*.csv", onFile: () => [{ path: "docs/c.csv" }], diff --git a/packages/ts/tests/integration/binding/persist.test.ts b/packages/ts/tests/integration/binding/persist.test.ts index 09d462c22..2d867af09 100644 --- a/packages/ts/tests/integration/binding/persist.test.ts +++ b/packages/ts/tests/integration/binding/persist.test.ts @@ -73,6 +73,7 @@ describe("DirSQL persist", () => { function makeTable(box: { count: number }) { return { + name: "items", ddl: "CREATE TABLE items (name TEXT, price REAL)", glob: "items/*.json", onFile: (filePath: string) => { @@ -241,6 +242,7 @@ describe("DirSQL persist", () => { root: dir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT, price REAL, sku TEXT)", glob: "items/*.json", onFile: (filePath: string) => { @@ -270,6 +272,7 @@ describe("DirSQL persist", () => { root: dir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT, price REAL)", glob: "**/*.json", onFile: (filePath: string) => [ diff --git a/packages/ts/tests/integration/binding/repeatable-config.test.ts b/packages/ts/tests/integration/binding/repeatable-config.test.ts index 318ea4f16..3439bbc12 100644 --- a/packages/ts/tests/integration/binding/repeatable-config.test.ts +++ b/packages/ts/tests/integration/binding/repeatable-config.test.ts @@ -16,7 +16,7 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; const basenameHook = `on-file = '''sh -c 'printf "[{\\"basename\\":\\"%s\\"}]" "\${1##*/}"' sh {path}'''`; function tableConfig(name: string, glob: string): string { - return `[[table]]\nddl = "CREATE TABLE ${name} (basename TEXT)"\nglob = "${glob}"\n${basenameHook}\n`; + return `[[table]]\nname = "${name}"\nddl = "CREATE TABLE ${name} (basename TEXT)"\nglob = "${glob}"\n${basenameHook}\n`; } describe("repeatable config", () => { diff --git a/packages/ts/tests/integration/binding/scan-failures.test.ts b/packages/ts/tests/integration/binding/scan-failures.test.ts index 61cacb99e..d6d56f663 100644 --- a/packages/ts/tests/integration/binding/scan-failures.test.ts +++ b/packages/ts/tests/integration/binding/scan-failures.test.ts @@ -37,6 +37,7 @@ describe("DirSQL scanFailures", () => { root: dir, tables: [ { + name: "items", ddl: "CREATE TABLE items (name TEXT)", glob: "*.json", onFile, diff --git a/packages/ts/tests/integration/binding/table-name-resolution.test.ts b/packages/ts/tests/integration/binding/table-name-resolution.test.ts deleted file mode 100644 index 7668a9a5d..000000000 --- a/packages/ts/tests/integration/binding/table-name-resolution.test.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { readFileSync } from "node:fs"; -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { DirSQL, parseTableName } from "dirsql"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; - -// A quoted identifier -- the canonical shape emitted by ORMs / schema tools -- -// must resolve to the bare table name (the surrounding quotes are SQL -// delimiters, not part of the name). -describe("quoted-identifier DDL (#204)", () => { - it("parseTableName resolves to the bare identifier", () => { - expect(parseTableName('CREATE TABLE "comments" (id TEXT)')).toBe( - "comments", - ); - }); - - describe("end to end", () => { - let dir: string; - - beforeEach(async () => { - dir = await mkdtemp(join(tmpdir(), "dirsql-204-")); - await mkdir(join(dir, "data"), { recursive: true }); - await writeFile( - join(dir, "data", "users.json"), - JSON.stringify([{ name: "Alice" }, { name: "Bob" }]), - ); - }); - - afterEach(async () => { - await rm(dir, { recursive: true, force: true }); - }); - - it("registers and queries a quoted-DDL table by its bare name", async () => { - const db = new DirSQL({ - root: dir, - tables: [ - { - ddl: 'CREATE TABLE "users" (name TEXT)', - glob: "data/users.json", - // `onFile` is synchronous (returns rows, not a Promise), so the - // file body is read with the sync API. - onFile: (filePath: string) => - JSON.parse(readFileSync(filePath, "utf8")), - }, - ], - }); - - const rows = await db.query("SELECT name FROM users ORDER BY name"); - expect(rows).toHaveLength(2); - expect(rows[0].name).toBe("Alice"); - expect(rows[1].name).toBe("Bob"); - }); - }); -}); diff --git a/packages/ts/tests/integration/binding/value-fidelity.test.ts b/packages/ts/tests/integration/binding/value-fidelity.test.ts index 9dc70b51a..9ac07040c 100644 --- a/packages/ts/tests/integration/binding/value-fidelity.test.ts +++ b/packages/ts/tests/integration/binding/value-fidelity.test.ts @@ -20,7 +20,7 @@ afterEach(async () => { function mkdb(onFile: () => Array>): DirSQL { return new DirSQL({ root: dir, - tables: [{ ddl: "CREATE TABLE t (v)", glob: "*.json", onFile }], + tables: [{ name: "t", ddl: "CREATE TABLE t (v)", glob: "*.json", onFile }], }); } @@ -89,6 +89,7 @@ describe("DirSQL Buffer -> BLOB", () => { root: dir, tables: [ { + name: "blobs", ddl: "CREATE TABLE blobs (name TEXT, data BLOB)", glob: "*.json", onFile: () => [{ name: "bin", data }], diff --git a/packages/ts/tests/integration/binding/watch.test.ts b/packages/ts/tests/integration/binding/watch.test.ts index 59f85682e..7d4a47da3 100644 --- a/packages/ts/tests/integration/binding/watch.test.ts +++ b/packages/ts/tests/integration/binding/watch.test.ts @@ -36,6 +36,7 @@ async function collectFromWatch( function jsonTable(glob: string) { return { + name: "items", ddl: "CREATE TABLE items (name TEXT)", glob, onFile: (filePath: string) => [JSON.parse(readFileSync(filePath, "utf8"))], @@ -219,6 +220,7 @@ describe("DirSQL watch() async iterator", () => { root: dir, tables: [ { + name: "rows", ddl: "CREATE TABLE rows (idx INTEGER, name TEXT)", glob: "*.jsonl", onFile: (filePath: string) => diff --git a/packages/ts/tests/integration/hermetic/extensions.test.ts b/packages/ts/tests/integration/hermetic/extensions.test.ts index 4488191eb..de8c4c2bd 100644 --- a/packages/ts/tests/integration/hermetic/extensions.test.ts +++ b/packages/ts/tests/integration/hermetic/extensions.test.ts @@ -10,7 +10,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; // Fake native core module, delivered through the mocked `createRequire` so // the real `loadNativeCore()` / `getCore()` chain runs unmodified. const { fakeCore } = vi.hoisted(() => ({ - fakeCore: { DirSQL: { openAsync: vi.fn() }, parseTableName: vi.fn() }, + fakeCore: { DirSQL: { openAsync: vi.fn() } }, })); vi.mock("node:fs", async (importOriginal) => ({ diff --git a/packages/ts/tests/integration/hermetic/index.test.ts b/packages/ts/tests/integration/hermetic/index.test.ts index 8565fffd8..833ebef67 100644 --- a/packages/ts/tests/integration/hermetic/index.test.ts +++ b/packages/ts/tests/integration/hermetic/index.test.ts @@ -3,7 +3,7 @@ // requires the napi binary — it hands back a fake core module instead. // Real-core behaviour is covered by `tests/binding/`. -import { DirSQL, type RowEvent, Table, parseTableName } from "dirsql"; +import { DirSQL, type RowEvent, Table } from "dirsql"; import { beforeEach, describe, expect, it, vi } from "vitest"; // The mocked `createRequire` resolves every specifier to this object, so the @@ -12,7 +12,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const { fakeCore } = vi.hoisted(() => ({ fakeCore: { DirSQL: { openAsync: vi.fn() }, - parseTableName: vi.fn(), }, })); @@ -75,6 +74,7 @@ describe("DirSQL construction", () => { openAsync.mockResolvedValue(makeInner()); const tables = [ { + name: "t", ddl: "CREATE TABLE t (n INTEGER)", glob: "**/*.json", onFile: () => [], @@ -106,12 +106,14 @@ describe("DirSQL construction", () => { openAsync.mockResolvedValue(makeInner()); const onFile = () => []; const asClass = new Table({ + name: "a", ddl: "CREATE TABLE a (n INTEGER)", glob: "a/**", onFile, strict: true, }); const asLiteral = { + name: "b", ddl: "CREATE TABLE b (n INTEGER)", glob: "b/**", onFile, @@ -121,6 +123,7 @@ describe("DirSQL construction", () => { const [, forwarded] = openAsync.mock.calls[0] as [unknown, unknown[]]; expect(forwarded).toEqual([ { + name: "a", ddl: "CREATE TABLE a (n INTEGER)", glob: "a/**", onFile, @@ -221,17 +224,3 @@ describe("DirSQL watch", () => { expect(seen.map((e) => e.action)).toEqual(["insert", "insert", "delete"]); }); }); - -describe("parseTableName", () => { - it("delegates DDL to the core's parser", () => { - fakeCore.parseTableName.mockImplementation((ddl: string) => - ddl.includes("comments") ? "comments" : null, - ); - expect(parseTableName('CREATE TABLE "comments" (n INTEGER)')).toBe( - "comments", - ); - expect(fakeCore.parseTableName).toHaveBeenCalledWith( - 'CREATE TABLE "comments" (n INTEGER)', - ); - }); -}); diff --git a/packages/ts/tests/integration/hermetic/on-file-rename.test.ts b/packages/ts/tests/integration/hermetic/on-file-rename.test.ts index 3dafc9a71..dc2e8f445 100644 --- a/packages/ts/tests/integration/hermetic/on-file-rename.test.ts +++ b/packages/ts/tests/integration/hermetic/on-file-rename.test.ts @@ -12,7 +12,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const { fakeCore } = vi.hoisted(() => ({ fakeCore: { DirSQL: { openAsync: vi.fn() }, - parseTableName: vi.fn(), }, })); @@ -50,7 +49,12 @@ describe("the onFile table seam", () => { const db = new DirSQL({ root: "/data", tables: [ - { ddl: "CREATE TABLE t (n INTEGER)", glob: "**/*.json", onFile }, + { + name: "t", + ddl: "CREATE TABLE t (n INTEGER)", + glob: "**/*.json", + onFile, + }, ], }); await db.ready; @@ -59,13 +63,19 @@ describe("the onFile table seam", () => { Array>, ]; expect(forwarded).toEqual([ - { ddl: "CREATE TABLE t (n INTEGER)", glob: "**/*.json", onFile }, + { + name: "t", + ddl: "CREATE TABLE t (n INTEGER)", + glob: "**/*.json", + onFile, + }, ]); }); it("Table copies onFile and carries no extract property", () => { const onFile = () => []; const table = new Table({ + name: "t", ddl: "CREATE TABLE t (n INTEGER)", glob: "**/*.json", onFile,