Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ kind, and dirsql never injects a column the table did not produce:
`content` column — the one place dirsql supplies columns for you. Attaching a
`--on-file` parser replaces the stat columns with the parser's output.

There is no automatic fact injection and no glob capture: a `{name}` glob
segment is rewritten to `*` and captures nothing. (Both mechanisms were
There is no automatic fact injection and no glob capture: a `{name}` in a glob is
a literal, not a wildcard. (Both mechanisms were
removed in the fact-removal epic, [#624](https://github.com/thekevinscott/dirsql/issues/624).)

**Content interpretation is intentionally out of scope.** dirsql does not
Expand Down Expand Up @@ -224,7 +224,7 @@ Walks a directory tree and matches files against table globs. Returns a list of

### `matcher` -- Glob-to-table mapping

Maps glob patterns to table names and handles ignore patterns. A file is matched against every glob in registration order; every matching pattern fires, so a file can belong to multiple tables. A `{name}` placeholder in a glob is rewritten to `*` before compilation, so it matches a single path segment but captures no value (glob captures were removed in #624).
Maps glob patterns to table names and handles ignore patterns. A file is matched against every glob in registration order; every matching pattern fires, so a file can belong to multiple tables. A `{name}` in a glob is a literal; `{a,b}` is an alternation.

### `watcher` -- Filesystem monitoring

Expand Down
3 changes: 3 additions & 0 deletions packages/python/changelog.d/2026-10-06-config-braces.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
**Changed** In a declared `[[table]] glob`, `{name}` is a literal, not a
wildcard, as in a path-table and in bash. The load-time error for a `{name}`
that collides with a declared column is gone. Write `*` for the old meaning.
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"command": "uv run python -m pytest tests/e2e/glob_speed_matchers_test.py tests/e2e/glob_speed_one_level_test.py tests/e2e/query_contract_test.py -x -q",
"ran_at": 1791333540,
"exit_code": 0,
"commit": "37586d9320ebca5d4be159cfe3a523ebcf017884",
"branch": "claude/1258-config-braces"
}
40 changes: 40 additions & 0 deletions packages/python/migrations.d/2026-10-06-config-braces.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
### A braced name in a config glob is a literal

#### Summary

`glob = "data/{id}/metadata.json"` used to treat `{id}` as `*`. It now matches
a directory literally named `{id}`. `{a,b}` alternation is unchanged. A column
named like a `{name}` no longer errors at load.

#### Required changes

| Before | After |
| --- | --- |
| `glob = "data/{id}/metadata.json"` | `glob = "data/*/metadata.json"` |

#### Deprecations removed

- The `CaptureColumnCollision` error: a `{name}` is no longer a capture, so
nothing can collide.

#### Behavior changes without code changes

- A config that kept `{name}` in a glob stops matching files and loads no rows.

#### Verification

```bash
cd "$(mktemp -d)"
mkdir -p 'data/{id}' data/x
echo '[{"p":"lit"}]' > 'data/{id}/m.json'
echo '[{"p":"other"}]' > data/x/m.json
cat > .dirsql.toml <<'TOML'
[[table]]
name = "t"
glob = "data/{id}/m.json"
ddl = "CREATE TABLE t (p TEXT)"
on-file = "jq -sc add"
TOML
dirsql query "SELECT p FROM t" -c .dirsql.toml
# expected: [{"p":"lit"}]
```
74 changes: 0 additions & 74 deletions packages/python/tests/e2e/capture_collision_test.py

This file was deleted.

17 changes: 0 additions & 17 deletions packages/python/tests/integration/binding/fanout_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,20 +65,3 @@ async def it_populates_both_tables_with_overlapping_distinct_globs(tmp_dir):
b_rows = await db.query("SELECT col_b FROM tb")
assert len(b_rows) == 1, "second-declared table must be populated"
assert b_rows[0]["col_b"] == "B"

@pytest.mark.asyncio
async def it_errors_when_a_placeholder_collides_with_a_column(tmp_dir):
_write_fanout_file(tmp_dir)
db = DirSQL(
tmp_dir,
tables=[
Table(
name="a",
ddl="CREATE TABLE a (id TEXT, col_a TEXT)",
glob="data/{id}/metadata.json",
on_file=lambda _path: [{"col_a": "A"}],
),
],
)
with pytest.raises(Exception, match="id"):
await db.ready()
57 changes: 0 additions & 57 deletions packages/python/tests/integration/binding/from_config_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,63 +64,6 @@ async def it_produces_one_row_per_matched_file(config_dir):
assert results[1]["path"] == "items/b.csv"
assert results[1]["basename"] == "b.csv"

def describe_glob_placeholders():
@pytest.mark.asyncio
async def it_errors_when_a_placeholder_collides_with_a_column(config_dir):
_write(
os.path.join(config_dir, "comments", "thread-1", "a.txt"),
"hello",
)
_write(
os.path.join(config_dir, ".dirsql.toml"),
"""\
[[table]]
name = "comments"
ddl = "CREATE TABLE comments (thread_id TEXT, basename TEXT)"
glob = "comments/{thread_id}/*.txt"
"""
+ _HOOK_PATH_BASENAME
+ "\n",
)

db = DirSQL(
root=config_dir, config=os.path.join(config_dir, ".dirsql.toml")
)
with pytest.raises(Exception, match="thread_id"):
await db.ready()

@pytest.mark.asyncio
async def it_treats_a_non_colliding_placeholder_as_a_wildcard(config_dir):
_write(
os.path.join(config_dir, "comments", "thread-1", "a.txt"),
"hello",
)
_write(
os.path.join(config_dir, "comments", "thread-2", "b.txt"),
"world",
)
_write(
os.path.join(config_dir, ".dirsql.toml"),
"""\
[[table]]
name = "comments"
ddl = "CREATE TABLE comments (path TEXT, basename TEXT)"
glob = "comments/{thread_id}/*.txt"
"""
+ _HOOK_PATH_BASENAME
+ "\n",
)

db = DirSQL(
root=config_dir, config=os.path.join(config_dir, ".dirsql.toml")
)
await db.ready()
results = await db.query("SELECT basename FROM comments ORDER BY basename")
assert len(results) == 2
assert results[0]["basename"] == "a.txt"
assert results[1]["basename"] == "b.txt"
assert "thread_id" not in results[0]

def describe_stat_virtuals():
@pytest.mark.asyncio
async def it_exposes_stat_virtuals(config_dir):
Expand Down
3 changes: 3 additions & 0 deletions packages/rust/changelog.d/2026-10-06-config-braces.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
**Changed** In a declared `[[table]] glob`, `{name}` is a literal, not a
wildcard, as in a path-table and in bash. The load-time error for a `{name}`
that collides with a declared column is gone. Write `*` for the old meaning.
40 changes: 40 additions & 0 deletions packages/rust/migrations.d/2026-10-06-config-braces.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
### A braced name in a config glob is a literal

#### Summary

`glob = "data/{id}/metadata.json"` used to treat `{id}` as `*`. It now matches
a directory literally named `{id}`. `{a,b}` alternation is unchanged. A column
named like a `{name}` no longer errors at load.

#### Required changes

| Before | After |
| --- | --- |
| `glob = "data/{id}/metadata.json"` | `glob = "data/*/metadata.json"` |

#### Deprecations removed

- The `CaptureColumnCollision` error: a `{name}` is no longer a capture, so
nothing can collide.

#### Behavior changes without code changes

- A config that kept `{name}` in a glob stops matching files and loads no rows.

#### Verification

```bash
cd "$(mktemp -d)"
mkdir -p 'data/{id}' data/x
echo '[{"p":"lit"}]' > 'data/{id}/m.json'
echo '[{"p":"other"}]' > data/x/m.json
cat > .dirsql.toml <<'TOML'
[[table]]
name = "t"
glob = "data/{id}/m.json"
ddl = "CREATE TABLE t (p TEXT)"
on-file = "jq -sc add"
TOML
dirsql query "SELECT p FROM t" -c .dirsql.toml
# expected: [{"p":"lit"}]
```
80 changes: 0 additions & 80 deletions packages/rust/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -199,14 +199,6 @@ pub enum DirSqlError {
source: Option<BoxError>,
},

#[error(
"glob capture `{{{placeholder}}}` collides with declared column `{column}`: \
captures no longer populate columns, so `{column}` would always be NULL. \
Remove `{column}` from the table's DDL, or emit its value from the on-file \
hook by splitting the path yourself."
)]
CaptureColumnCollision { placeholder: String, column: String },

#[error(
"query() only accepts read-only statements; SQLite classified this statement as a write"
)]
Expand Down Expand Up @@ -1323,18 +1315,6 @@ impl DirSQL {
eprintln!("{warning}");
}
}
// Reject a `{name}` glob placeholder whose name is also a declared
// column: captures no longer populate columns, so it would read
// NULL forever. The table exists here either way (freshly created
// or restored from cache), so its columns are knowable, and this
// runs before any file is ingested — a load-time failure.
let declared_columns = db.get_table_columns(&table_name).map_err(map_db_error)?;
if let Some(name) = find_capture_column_collision(&table.glob, &declared_columns) {
return Err(DirSqlError::CaptureColumnCollision {
placeholder: name.clone(),
column: name,
});
}
on_file_map.insert(table_name.clone(), table.hook);
strict_map.insert(table_name.clone(), table.strict);
ddl_map.insert(table_name, table.ddl);
Expand Down Expand Up @@ -2519,18 +2499,6 @@ pub(crate) fn json_to_value(value: &serde_json::Value) -> Value {
}
}

/// Returns the first `{name}` placeholder in `glob` whose name is also one of
/// `declared_columns`. `None` when the glob has no placeholders or none of
/// them names a declared column. Pure: the sole input is the glob string and
/// the column list, so it is exhaustively unit-testable.
fn find_capture_column_collision(glob: &str, declared_columns: &[String]) -> Option<String> {
let declared: std::collections::HashSet<&str> =
declared_columns.iter().map(String::as_str).collect();
crate::matcher::placeholder_names(glob)
.into_iter()
.find(|name| declared.contains(name.as_str()))
}

/// Async wrapper around [`DirSQL`] whose constructor returns immediately while
/// the initial scan runs on a background thread.
///
Expand Down Expand Up @@ -4506,54 +4474,6 @@ mod internal_tests {
assert_eq!(deleted, vec![("gone.txt".to_string(), "t".to_string())]);
}

#[test]
fn find_capture_column_collision_flags_a_placeholder_naming_a_column() {
let declared = vec!["thread_id".to_string(), "basename".to_string()];
assert_eq!(
find_capture_column_collision("_comments/{thread_id}/*.txt", &declared),
Some("thread_id".to_string())
);
}

#[test]
fn find_capture_column_collision_ignores_a_placeholder_with_no_column() {
let declared = vec!["path".to_string(), "basename".to_string()];
assert_eq!(
find_capture_column_collision("_comments/{thread_id}/*.txt", &declared),
None
);
}

#[test]
fn find_capture_column_collision_none_without_placeholders() {
let declared = vec!["thread_id".to_string()];
assert_eq!(
find_capture_column_collision("_comments/*/*.txt", &declared),
None
);
}

#[test]
fn find_capture_column_collision_returns_first_colliding_placeholder() {
let declared = vec!["repo".to_string(), "org".to_string()];
assert_eq!(
find_capture_column_collision("{org}/{repo}/data.json", &declared),
Some("org".to_string())
);
}

#[test]
fn capture_column_collision_error_names_placeholder_and_fix() {
let err = DirSqlError::CaptureColumnCollision {
placeholder: "thread_id".to_string(),
column: "thread_id".to_string(),
};
let msg = err.to_string();
assert!(msg.contains("thread_id"));
assert!(msg.contains("collides"));
assert!(msg.contains("on-file"));
}

/// A `name` the batch never created: the error has to say what it *did*
/// create, or a typo gives the reader nothing to compare against.
#[test]
Expand Down
Loading
Loading