From f8634a5fdce6c647ef527a709bad7fa31326064a Mon Sep 17 00:00:00 2001 From: helson Date: Fri, 16 Oct 2026 10:49:59 +0800 Subject: [PATCH] fix(queuefs): serve list_unacked in id order so a large queue cannot fail startup list_unacked filters status IN ('pending','processing') and orders by id, but idx_queue_status is (queue_name, status, id). With two status values that index cannot deliver rows in id order, so SQLite inserts a sorter: SEARCH queue_messages USING INDEX idx_queue_status (queue_name=? AND status=?) USE TEMP B-TREE FOR ORDER BY Once the unacknowledged set is large enough the sorter spills to a temporary file, and with no writable temp directory list_unacked fails with 'unable to open database file' even though the database is intact. Task recovery snapshots every queue before any consumer starts, so that aborts server startup, and the queue only drains while the server runs, so it never heals on its own. Add idx_queue_unacked on (queue_name, id) so that query is served in id order without a sorter. The single-status queries keep using idx_queue_status, so no existing plan changes. The index needs its own name: CREATE INDEX IF NOT EXISTS cannot redefine an existing index, so changing the column order of idx_queue_status would silently not apply to databases that already have it, which is exactly the population hitting this failure. Verified against SQLite via EXPLAIN QUERY PLAN for a fresh database and for one migrated from the old schema. Fixes #6127 --- crates/ragfs/src/plugins/queuefs/backend.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/crates/ragfs/src/plugins/queuefs/backend.rs b/crates/ragfs/src/plugins/queuefs/backend.rs index 73407d7080..e1add4b9c0 100644 --- a/crates/ragfs/src/plugins/queuefs/backend.rs +++ b/crates/ragfs/src/plugins/queuefs/backend.rs @@ -462,6 +462,17 @@ impl SQLiteQueueBackend { "ALTER TABLE queue_messages ADD COLUMN processing_started_at INTEGER", "CREATE INDEX IF NOT EXISTS idx_queue_status ON queue_messages(queue_name, status, id)", "CREATE INDEX IF NOT EXISTS idx_queue_message_id ON queue_messages(queue_name, message_id)", + // idx_queue_status is (queue_name, status, id). list_unacked matches two + // status values ("pending", "processing"), so SQLite cannot use that index + // to satisfy ORDER BY id and adds a sorter; once the queue is large enough + // the sorter spills to a temporary file and list_unacked fails with + // "unable to open database file" when no temp dir is writable, which aborts + // startup because task recovery snapshots every queue first. An index whose + // leading columns are (queue_name, id) serves that query in id order without a + // sort, while the single-status queries keep using idx_queue_status. It needs + // its own name: CREATE INDEX IF NOT EXISTS cannot redefine an index that + // already exists under the old column order on an older database. + "CREATE INDEX IF NOT EXISTS idx_queue_unacked ON queue_messages(queue_name, id)", ] { if let Err(err) = conn.execute(stmt, []) { let text = err.to_string(); @@ -1242,7 +1253,7 @@ mod tests { .prepare( "SELECT name FROM sqlite_master WHERE type = 'index' - AND name IN ('idx_queue_message_id', 'idx_queue_status') + AND name IN ('idx_queue_message_id', 'idx_queue_status', 'idx_queue_unacked') ORDER BY name", ) .unwrap() @@ -1254,7 +1265,8 @@ mod tests { migrated_indexes, vec![ "idx_queue_message_id".to_string(), - "idx_queue_status".to_string() + "idx_queue_status".to_string(), + "idx_queue_unacked".to_string() ] ); }