From e58a9162e5774d2351a98b39193d64cc59943932 Mon Sep 17 00:00:00 2001 From: Raunak Raj <71929976+bajrangCoder@users.noreply.github.com> Date: Sat, 12 Sep 2026 14:08:11 +0530 Subject: [PATCH 1/2] fix(search): prevent discovery restarts and stream results efficiently --- .../sdcard/src/android/WorkspaceIndex.java | 160 +++++++++--------- src/sidebarApps/searchInFiles/index.js | 112 +++++++----- src/sidebarApps/searchInFiles/worker.js | 152 ++++++++--------- tests/unit/nativeSearchProcessing.test.js | 99 +++++++++++ tests/unit/searchDiscovery.test.js | 55 ++++++ tests/unit/searchStreamingResults.test.js | 61 +++++++ tests/unit/searchWorker.test.js | 67 ++++++++ 7 files changed, 493 insertions(+), 213 deletions(-) create mode 100644 tests/unit/nativeSearchProcessing.test.js create mode 100644 tests/unit/searchDiscovery.test.js create mode 100644 tests/unit/searchStreamingResults.test.js create mode 100644 tests/unit/searchWorker.test.js diff --git a/src/plugins/sdcard/src/android/WorkspaceIndex.java b/src/plugins/sdcard/src/android/WorkspaceIndex.java index b8b867f156..f3c6a56e5c 100644 --- a/src/plugins/sdcard/src/android/WorkspaceIndex.java +++ b/src/plugins/sdcard/src/android/WorkspaceIndex.java @@ -50,8 +50,7 @@ class WorkspaceIndex { private static final int EXPLICIT_INCLUDE_READ_LIMIT_BYTES = 128 * 1024 * 1024; private static final int SAMPLE_BYTES = 8192; private static final int MAX_MATCHES_PER_FILE = 5000; - private static final int SEARCH_RESULT_BATCH_SIZE = 12; - private static final int SEARCH_RESULT_BATCH_MATCHES = 600; + private static final int SEARCH_RESULT_BATCH_MATCHES = 200; private static final Set BINARY_EXTENSIONS = new HashSet<>(); private static final Set TEXT_EXTENSIONS = new HashSet<>(); @@ -965,8 +964,6 @@ private void runSearch(Job job, JSONObject options, CallbackContext callback) boolean batchResults = options.optBoolean("batchResults", false); Pattern pattern = compileSearchPattern(search, searchOptions); - JSONArray searchResultBatch = new JSONArray(); - int batchedMatches = 0; int total = files.length(); int processed = 0; int lastProgress = -1; @@ -1013,38 +1010,18 @@ private void runSearch(Job job, JSONObject options, CallbackContext callback) if ("replace".equals(mode)) { String replacement = Matcher.quoteReplacement(replace == null ? "" : replace); - String text = pattern.matcher(content).replaceAll(replacement); + String text = pattern.matcher(new SearchInput(content, job)).replaceAll(replacement); JSONObject result = baseEvent(job.id, "replace-result"); result.put("file", file); result.put("text", text); send(callback, result, true); } else { - JSONObject result = searchInContent(file, content, pattern); - if (result != null) { - if (batchResults) { - searchResultBatch.put(result); - batchedMatches += result.getJSONArray("matches").length(); - if ( - searchResultBatch.length() >= SEARCH_RESULT_BATCH_SIZE || - batchedMatches >= SEARCH_RESULT_BATCH_MATCHES - ) { - flushSearchResultBatch(callback, job.id, searchResultBatch); - batchedMatches = 0; - } - } else { - JSONObject event = baseEvent(job.id, "search-result"); - event.put("data", result); - send(callback, event, true); - } - } + searchInContent(file, content, pattern, job, callback, batchResults); } processed += 1; } - if (batchResults) { - flushSearchResultBatch(callback, job.id, searchResultBatch); - } sendProgress(callback, job.id, 100); send(callback, baseEvent(job.id, "replace".equals(mode) ? "done-replacing" : "done-searching"), false); } @@ -1063,32 +1040,30 @@ private Pattern compileSearchPattern(String search, JSONObject options) return Pattern.compile(pattern, flags); } - private JSONObject searchInContent( + private void searchInContent( JSONObject file, String content, - Pattern pattern + Pattern pattern, + Job job, + CallbackContext callback, + boolean batchResults ) throws JSONException { - Matcher matcher = pattern.matcher(content); + SearchInput input = new SearchInput(content, job); + Matcher matcher = pattern.matcher(input); JSONArray matches = new JSONArray(); - StringBuilder text = new StringBuilder(file.optString("name")); - if (text.length() > 30) { - text = new StringBuilder("..." + text.substring(text.length() - 30)); - } - - boolean limited = false; + int matchCount = 0; int cursor = 0; int row = 0; int column = 0; - while (matcher.find()) { - if (matches.length() >= MAX_MATCHES_PER_FILE) { - limited = true; - break; - } - String word = matcher.group(); + while (true) { + input.check(); + if (!matcher.find()) break; + String word = content.substring(matcher.start(), Math.min(matcher.end(), matcher.start() + 160)); int start = matcher.start(); int end = matcher.end(); String[] surrounding = getSurrounding(content, word, start, end); while (cursor < start) { + if ((cursor & 1023) == 0) input.check(); if (content.charAt(cursor) == '\n') { row += 1; column = 0; @@ -1099,6 +1074,7 @@ private JSONObject searchInContent( } JSONObject startPosition = lineColumn(row, column); while (cursor < end) { + if ((cursor & 1023) == 0) input.check(); if (content.charAt(cursor) == '\n') { row += 1; column = 0; @@ -1114,34 +1090,31 @@ private JSONObject searchInContent( match.put("line", surrounding[0].trim()); match.put("position", position(startPosition, endPosition)); matches.put(match); - text.append("\n\t").append(surrounding[0].trim()); + matchCount++; + if (matchCount == MAX_MATCHES_PER_FILE) { + sendSearchMatches(callback, job.id, file, matches, batchResults, matcher.find()); + return; + } + if (matches.length() >= SEARCH_RESULT_BATCH_MATCHES) { + sendSearchMatches(callback, job.id, file, matches, batchResults, false); + matches = new JSONArray(); + input.renewDeadline(); + } } - if (matches.length() == 0) return null; + sendSearchMatches(callback, job.id, file, matches, batchResults, false); + } + private void sendSearchMatches(CallbackContext callback, String id, JSONObject file, + JSONArray matches, boolean batchResults, boolean limited) throws JSONException { + if (matches.length() == 0) return; JSONObject data = new JSONObject(); data.put("file", file); data.put("matches", matches); - if (limited) { - text - .append("\n\t") - .append("... result limit reached for this file"); - } data.put("limited", limited); - data.put("text", text.toString()); - return data; - } - - private void flushSearchResultBatch( - CallbackContext callback, - String id, - JSONArray batch - ) throws JSONException { - if (batch.length() == 0) return; - JSONObject event = baseEvent(id, "search-results"); - event.put("data", new JSONArray(batch.toString())); + JSONObject event = baseEvent(id, batchResults ? "search-results" : "search-result"); + event.put("data", batchResults ? new JSONArray().put(data) : data); send(callback, event, true); - while (batch.length() > 0) batch.remove(0); } private String getFileContent( @@ -1590,37 +1563,56 @@ private JSONObject lineColumn(int row, int column) throws JSONException { return result; } - private String[] getSurrounding(String content, String word, int start, int end) { - int max = 160; - int lineStart = start; - while (lineStart > 0) { - char previous = content.charAt(lineStart - 1); - if (previous == '\n' || previous == '\r') break; - lineStart--; + // Java's Matcher does not honor thread interruption. Check its input access + // so backtracking is subject to the same cancellation and deadline as results. + private static final class SearchInput implements CharSequence { + final String content; + final Job job; + long deadline = System.nanoTime() + 2_000_000_000L; + int accesses; + + SearchInput(String content, Job job) { + this.content = content; + this.job = job; } - int lineEnd = end; - while (lineEnd < content.length()) { - char current = content.charAt(lineEnd); - if (current == '\n' || current == '\r') break; - lineEnd++; + void renewDeadline() { deadline = System.nanoTime() + 2_000_000_000L; } + + void check() { + if (job.cancelled) throw new IllegalStateException("Search cancelled"); + if (System.nanoTime() > deadline) throw new IllegalStateException("Search timed out; simplify the expression"); } - int snippetStart = lineStart; - int snippetEnd = lineEnd; - if (lineEnd - lineStart > max) { - int matchLength = Math.max(1, end - start); - int remaining = Math.max(0, max - matchLength); - int left = remaining / 2; - int right = remaining - left; - snippetStart = Math.max(lineStart, start - left); - snippetEnd = Math.min(lineEnd, end + right); + public int length() { return content.length(); } + public char charAt(int index) { + if ((accesses++ & 1023) == 0) check(); + return content.charAt(index); } + public CharSequence subSequence(int start, int end) { return content.subSequence(start, end); } + public String toString() { return content; } + } + private String[] getSurrounding(String content, String word, int start, int end) { + int max = 160; + int remaining = Math.max(0, max - (end - start)); + int snippetStart = start; + int leftLimit = Math.max(0, start - remaining / 2); + while (snippetStart > leftLimit) { + char c = content.charAt(snippetStart - 1); + if (c == '\n' || c == '\r') break; + snippetStart--; + } + int snippetEnd = Math.min(end, start + max); + int rightLimit = Math.min(content.length(), snippetStart + max); + while (snippetEnd < rightLimit) { + char c = content.charAt(snippetEnd); + if (c == '\n' || c == '\r') break; + snippetEnd++; + } StringBuilder line = new StringBuilder(); - if (snippetStart > lineStart) line.append("..."); + if (snippetStart > 0 && content.charAt(snippetStart - 1) != '\n' && content.charAt(snippetStart - 1) != '\r') line.append("..."); line.append(content.substring(snippetStart, snippetEnd).trim()); - if (snippetEnd < lineEnd) line.append("..."); + if (snippetEnd < content.length() && content.charAt(snippetEnd) != '\n' && content.charAt(snippetEnd) != '\r') line.append("..."); String renderText = word; return new String[] { diff --git a/src/sidebarApps/searchInFiles/index.js b/src/sidebarApps/searchInFiles/index.js index 9efae14b03..e0253fa124 100644 --- a/src/sidebarApps/searchInFiles/index.js +++ b/src/sidebarApps/searchInFiles/index.js @@ -316,16 +316,32 @@ async function onWorkerMessage(e) { } switch (action) { + case "processing": + clearTimeout(e.target.searchWatchdog); + e.target.searchWatchdog = setTimeout(() => { + if (version !== searchVersion) return; + $error.value = "Search timed out; simplify the expression"; + terminateWorker(false); + if (replacing) finishReplaceTask(version); + else finishSearchTask(version); + }, 2000); + break; + case "processed": + clearTimeout(e.target.searchWatchdog); + break; case "get-file": { let readError; let content = ""; try { - content = await readSearchFileContent(data); + content = await withTimeout(readSearchFileContent(data), 30000); + if (content === TIMEOUT) throw new Error("File read timed out"); } catch (er) { - readError = er; + readError = er?.message || String(er); + if (version === searchVersion) $error.value = readError; } + if (version !== searchVersion) return; e.target.postMessage({ id, action: "get-file", @@ -336,7 +352,9 @@ async function onWorkerMessage(e) { } case "search-result": { + clearTimeout(e.target.searchWatchdog); appendSearchResult(data); + e.target.postMessage({ action: "result-ack", id }); break; } @@ -388,30 +406,26 @@ async function onWorkerMessage(e) { function appendSearchResult(data) { const { file, matches, limited } = data; - + const hasResults = results.length > 0; if (!matches.length) return; - if (filesSearched.find((item) => item.url === file.url)) return; - - filesSearched.push(Tree.fromJSON(file)); - if (filesSearched.length === 1) { - searchResult.setValue(""); + let index = filesSearched.findIndex((item) => item.url === file.url); + if (index < 0) { + index = filesSearched.length; + filesSearched.push(Tree.fromJSON(file)); + if (filesSearched.length === 1) searchResult.setValue(""); + resultOverview.filesCount += 1; + fileNames.push({ name: file.name, path: file.path, count: 0 }); } - resultOverview.filesCount += 1; + fileNames[index].count += matches.length; resultOverview.matchesCount += matches.length; $resultOverview.innerHTML = searchResultText( resultOverview.filesCount, resultOverview.matchesCount, ); - - const index = filesSearched.length - 1; + const continuation = + results.length && results[results.length - 1].file === index; const displayRows = groupMatchesForDisplay(matches); - results.push({ - file: index, - match: null, - position: null, - }); - - fileNames.push({ name: file.name, path: file.path, count: matches.length }); + if (!continuation) results.push({ file: index, match: null, position: null }); for (const result of matches) { result.file = index; if (words.length < MAX_HL_WORDS) { @@ -419,24 +433,11 @@ function appendSearchResult(data) { if (!words.includes(token)) words.push(token); } } - for (const { result } of displayRows) { - results.push(result); - } - if (limited) { - results.push({ - file: index, - match: null, - position: null, - notice: true, - }); - } - - const text = formatSearchResultText(file, displayRows, limited); - if (fileNames.length > 1) { - appendSearchResultText(`\n${text}`); - } else { - appendSearchResultText(text); - } + for (const { result } of displayRows) results.push(result); + if (limited) + results.push({ file: index, match: null, position: null, notice: true }); + const text = formatSearchResultText(file, displayRows, limited, continuation); + appendSearchResultText(`${hasResults ? "\n" : ""}${text}`); } function enqueueNativeSearchResults(batch, version) { @@ -510,8 +511,13 @@ function groupMatchesForDisplay(matches) { return rows; } -function formatSearchResultText(file, displayRows, limited) { - const lines = [file.name]; +function formatSearchResultText( + file, + displayRows, + limited, + continuation = false, +) { + const lines = continuation ? [] : [file.name]; for (const { result, preview } of displayRows) { const row = result.position?.start?.row; const lineNumber = Number.isInteger(row) ? `${row + 1}: ` : ""; @@ -632,12 +638,11 @@ async function searchAll() { return; } - addEvents(); - const version = searchVersion; - await waitForFileListIfReady(); + await waitForFileListIfReady(version); if (version !== searchVersion) return; + addEvents(); const allFiles = files().filter((file) => !helpers.isBinary(file)); const nativeRoots = addedFolder .filter(({ listFiles }) => listFiles) @@ -843,11 +848,15 @@ function getOpenFileOverlays() { return overlays; } -async function waitForFileListIfReady() { - const result = await withTimeout(waitForFileList(), FILE_LIST_WAIT_TIMEOUT); +async function waitForFileListIfReady(version) { + const ready = waitForFileList(); + const result = await withTimeout(ready, FILE_LIST_WAIT_TIMEOUT); + if (version !== searchVersion) return; if (result === TIMEOUT) { $indexStatus.value = "Scanning project files..."; + await ready; } + if (version === searchVersion) $indexStatus.value = ""; } function markIndexDirty(urls) { @@ -879,10 +888,13 @@ function clearPendingResultText() { const TIMEOUT = Symbol("timeout"); function withTimeout(promise, ms) { + let timer; return Promise.race([ promise, - new Promise((resolve) => setTimeout(() => resolve(TIMEOUT), ms)), - ]); + new Promise((resolve) => { + timer = setTimeout(() => resolve(TIMEOUT), ms); + }), + ]).finally(() => clearTimeout(timer)); } /** @@ -958,6 +970,11 @@ function sendMessage(action, files, search, options, replace) { */ function onErrorMessage(e) { console.error(e); + if (e.target.searchVersion !== searchVersion) return; + $error.value = e.message || "Search worker failed"; + terminateWorker(false); + if (replacing) void finishReplaceTask(searchVersion); + else void finishSearchTask(searchVersion); } /** @@ -966,7 +983,10 @@ function onErrorMessage(e) { * @param {boolean} [initializeNewWorkers=true] - Whether to initialize new workers after terminating the existing ones. */ function terminateWorker(initializeNewWorkers = true) { - workers.forEach((worker) => worker.terminate()); + workers.forEach((worker) => { + clearTimeout(worker.searchWatchdog); + worker.terminate(); + }); workers.length = 0; if (!initializeNewWorkers) return; diff --git a/src/sidebarApps/searchInFiles/worker.js b/src/sidebarApps/searchInFiles/worker.js index 529735dc08..e4c3931403 100644 --- a/src/sidebarApps/searchInFiles/worker.js +++ b/src/sidebarApps/searchInFiles/worker.js @@ -3,7 +3,9 @@ import picomatch from "picomatch/posix"; import { isBinaryFile } from "utils/binaryExtensions"; const resolvers = {}; +let requestId = 0; const MAX_CONCURRENT_FILE_READS = 2; +const RESULT_BATCH_SIZE = 200; self.onmessage = (ev) => { const { action, data, error, id } = ev.data; @@ -16,6 +18,7 @@ self.onmessage = (ev) => { processFiles(data, "replace"); break; + case "result-ack": case "get-file": { if (!resolvers[id]) return; const cb = resolvers[id]; @@ -86,13 +89,21 @@ function processFiles(data, mode = "search") { return; } - getFile(file.url, (res, err) => { + getFile(file.url, async (res, err) => { if (err) { finishOne(); return; } - process({ file, content: res, search, replace, options }); + self.postMessage({ action: "processing" }); + await process({ + file, + content: res, + search, + replace, + options, + }); + self.postMessage({ action: "processed" }); finishOne(); }); } @@ -105,37 +116,54 @@ function processFiles(data, mode = "search") { * @param {string} arg.content - The file content. * @param {RegExp} arg.search - The string to search for. */ -function searchInFile({ file, content, search }) { - const matches = []; - - let text = `${file.name}`; - let match; - - if (text.length > 30) { - text = `...${text.slice(-30)}`; +async function searchInFile({ file, content, search }) { + let matches = []; + search = new RegExp(search.source, search.flags); + async function flush() { + if (!matches.length) return; + const batch = matches; + matches = []; + await new Promise((resolve) => { + const id = ++requestId; + resolvers[id] = resolve; + self.postMessage({ + action: "search-result", + id, + data: { file, matches: batch }, + }); + }); + self.postMessage({ action: "processing" }); } - + let cursor = 0; + let row = 0; + let column = 0; + function positionAt(offset) { + while (cursor < offset) { + if (content[cursor++] === "\n") { + row++; + column = 0; + } else column++; + } + return { row, column }; + } + search.lastIndex = 0; + let match; while ((match = search.exec(content))) { - const [word] = match; + const word = match[0]; const start = match.index; const end = start + word.length; - const position = { - start: getLineColumn(content, start), - end: getLineColumn(content, end), - }; + const position = { start: positionAt(start), end: positionAt(end) }; const [line, renderText] = getSurrounding(content, word, start, end); - text += `\n\t${line.trim()}`; - matches.push({ match: word, position, renderText, line: line.trim() }); + matches.push({ match: word.slice(0, 160), position, renderText, line }); + if (matches.length >= RESULT_BATCH_SIZE) await flush(); + if (!search.global && !search.sticky) break; + if (!word.length) { + // AdvanceStringIndex: don't restart inside a Unicode surrogate pair. + search.lastIndex = + end + (search.unicode && content.codePointAt(end) > 0xffff ? 2 : 1); + } } - - self.postMessage({ - action: "search-result", - data: { - file, - matches, - text, - }, - }); + await flush(); } /** @@ -164,61 +192,19 @@ function replaceInFile({ file, content, search, replace }) { */ function getSurrounding(content, word, start, end) { const max = 160; - let lineStart = start; - while (lineStart > 0) { - const previous = content[lineStart - 1]; - if (previous === "\n" || previous === "\r") break; - lineStart--; - } - - let lineEnd = end; - while (lineEnd < content.length) { - const current = content[lineEnd]; - if (current === "\n" || current === "\r") break; - lineEnd++; - } - - let snippetStart = lineStart; - let snippetEnd = lineEnd; - if (lineEnd - lineStart > max) { - const matchLength = Math.max(1, end - start); - const remaining = Math.max(0, max - matchLength); - const left = Math.floor(remaining / 2); - const right = remaining - left; - snippetStart = Math.max(lineStart, start - left); - snippetEnd = Math.min(lineEnd, end + right); - } - - let line = content.substring(snippetStart, snippetEnd).trim(); - if (snippetStart > lineStart) line = `...${line}`; - if (snippetEnd < lineEnd) line = `${line}...`; - - return [line, word].map((text) => text.replace(/[\r\n]+/g, " ⏎ ")); -} - -/** - * Determines the line and column numbers for a given position in the file. - * - * @param {string} file - The file content as a string. - * @param {number} position - The position in the file for which line and column - * numbers are to be determined. - * - * @returns {Object} An object with 'line' and 'column' properties, representing - * the line and column numbers respectively for the given position. - * - * @example - * - * const file = 'Hello, this is a test.\nAnother test is here.'; - * const position = 15; - * const lineColumn = getLineColumn(file, position); - * - * // lineColumn: { line: 1, column: 16 } - */ -function getLineColumn(file, position) { - const lines = file.substring(0, position).split("\n"); - const lineNumber = lines.length - 1; - const columnNumber = lines[lineNumber].length; - return { row: lineNumber, column: columnNumber }; + const remaining = Math.max(0, max - (end - start)); + let left = start; + const leftLimit = Math.max(0, start - Math.floor(remaining / 2)); + while (left > leftLimit && !/[\r\n]/.test(content[left - 1])) left--; + let right = Math.min(end, start + max); + const rightLimit = Math.min(content.length, start + max - (start - left)); + while (right < rightLimit && !/[\r\n]/.test(content[right])) right++; + let line = content.slice(left, right).trim(); + if (left > 0 && !/[\r\n]/.test(content[left - 1])) line = `...${line}`; + if (right < content.length && !/[\r\n]/.test(content[right])) line += "..."; + return [line, word.slice(0, max)].map((text) => + text.replace(/[\r\n]+/g, " ⏎ "), + ); } /** @@ -227,7 +213,7 @@ function getLineColumn(file, position) { * @param {function} cb */ function getFile(url, cb) { - const id = Number.parseInt(Date.now() + Math.random() * 1000000); + const id = ++requestId; resolvers[id] = cb; self.postMessage({ action: "get-file", diff --git a/tests/unit/nativeSearchProcessing.test.js b/tests/unit/nativeSearchProcessing.test.js new file mode 100644 index 0000000000..752bc04bf5 --- /dev/null +++ b/tests/unit/nativeSearchProcessing.test.js @@ -0,0 +1,99 @@ +import { readFileSync, mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; +import { expect, it } from "vitest"; + +const hasJava = spawnSync("javac", ["-version"]).status === 0; +it.skipIf(!hasJava)( + "native preview scans stay bounded and regex input observes cancellation/deadlines", + () => { + const source = readFileSync( + new URL( + "../../src/plugins/sdcard/src/android/WorkspaceIndex.java", + import.meta.url, + ), + "utf8", + ); + const helpers = source.slice( + source.indexOf(" private static final class SearchInput"), + source.indexOf(" private void sendStatus("), + ); + const matching = source.slice( + source.indexOf(" private void searchInContent("), + source.indexOf(" private String getFileContent("), + ); + const positions = source.slice( + source.indexOf(" private JSONObject position("), + source.indexOf(" // Java's Matcher"), + ); + const directory = mkdtempSync(join(tmpdir(), "acode-native-search-")); + try { + writeFileSync( + join(directory, "SearchCheck.java"), + ` +import java.util.regex.Pattern; +import java.util.regex.Matcher; +class SearchCheck { + static final int MAX_MATCHES_PER_FILE = 5000; + static final int SEARCH_RESULT_BATCH_MATCHES = 200; + static class Job { volatile boolean cancelled; String id = "test"; } + static class JSONException extends Exception {} + static class JSONObject extends java.util.HashMap { + public JSONObject put(String key, Object value) { super.put(key, value); return this; } + } + static class JSONArray extends java.util.ArrayList { + int length() { return size(); } + JSONArray put(Object value) { add(value); return this; } + } + static class CallbackContext { java.util.List events = new java.util.ArrayList<>(); } + JSONObject baseEvent(String id, String type) { return new JSONObject().put("type", type); } + void send(CallbackContext callback, JSONObject event, boolean keep) { callback.events.add(event); } + ${matching} + ${positions} + ${helpers} + public static void main(String[] args) throws Exception { + SearchCheck test = new SearchCheck(); + CallbackContext callback = new CallbackContext(); + test.searchInContent(new JSONObject(), "x ".repeat(6000), Pattern.compile("x"), new Job(), callback, false); + int count = 0; + for (JSONObject event : callback.events) { + JSONObject data = (JSONObject) event.get("data"); + int batchSize = ((JSONArray) data.get("matches")).length(); + if (batchSize > 200) throw new AssertionError("batch size"); + count += batchSize; + } + if (count != 5000 || !Boolean.TRUE.equals(((JSONObject) callback.events.get(callback.events.size() - 1).get("data")).get("limited"))) throw new AssertionError("original file limit"); + String line = "x ".repeat(100000); + for (int i = 0; i < 100000; i++) { + if (test.getSurrounding(line, "x", i * 2, i * 2 + 1)[0].length() > 166) throw new AssertionError("preview size"); + } + Job job = new Job(); job.cancelled = true; + try { Pattern.compile("x").matcher(new SearchInput(line, job)).find(); throw new AssertionError("not cancelled"); } + catch (IllegalStateException expected) {} + long start = System.nanoTime(); + try { Pattern.compile("(a+)+$").matcher(new SearchInput("a".repeat(100000) + "!", new Job())).find(); throw new AssertionError("no timeout"); } + catch (IllegalStateException expected) { + if (!expected.getMessage().contains("timed out")) throw expected; + } + if (System.nanoTime() - start > 4_000_000_000L) throw new AssertionError("deadline exceeded"); + } +}`, + ); + const compile = spawnSync( + "javac", + [join(directory, "SearchCheck.java")], + { encoding: "utf8" }, + ); + expect(compile.status, compile.stderr).toBe(0); + const run = spawnSync("java", ["-cp", directory, "SearchCheck"], { + encoding: "utf8", + timeout: 6000, + }); + expect(run.status, run.stderr).toBe(0); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }, + 10000, +); diff --git a/tests/unit/searchDiscovery.test.js b/tests/unit/searchDiscovery.test.js new file mode 100644 index 0000000000..c4f8620056 --- /dev/null +++ b/tests/unit/searchDiscovery.test.js @@ -0,0 +1,55 @@ +import { readFileSync } from "node:fs"; +import vm from "node:vm"; +import { expect, it, vi } from "vitest"; + +const source = readFileSync( + new URL("../../src/sidebarApps/searchInFiles/index.js", import.meta.url), + "utf8", +); +function extract(name, next) { + return source.slice( + source.indexOf(`async function ${name}(`), + source.indexOf(`\n${next}`, source.indexOf(`async function ${name}(`)), + ); +} +it("waits for discovery before subscribing to file events or taking the search snapshot", async () => { + let resolve; + const ready = new Promise((r) => { + resolve = r; + }); + const addEvents = vi.fn(); + const files = vi.fn(() => []); + const context = vm.createContext({ + $search: { value: "needle" }, + getOptions: () => ({}), + toRegex: () => /needle/g, + searchVersion: 1, + waitForFileList: () => ready, + addEvents, + files, + helpers: { isBinary: () => false }, + addedFolder: [], + editorManager: { files: [] }, + searchResult: { setGhostText() {} }, + strings: {}, + $progress: {}, + $indexStatus: {}, + TIMEOUT: Symbol(), + FILE_LIST_WAIT_TIMEOUT: 250, + withTimeout: async () => context.TIMEOUT, + }); + vm.runInContext( + extract("searchAll", "async function readSearchFileContent") + + extract("waitForFileListIfReady", "function markIndexDirty"), + context, + ); + const pending = vm.runInContext("searchAll()", context); + await new Promise((resolve) => setImmediate(resolve)); + expect(addEvents).not.toHaveBeenCalled(); + expect(files).not.toHaveBeenCalled(); + expect(context.$indexStatus.value).toBe("Scanning project files..."); + resolve(); + await pending; + expect(addEvents).toHaveBeenCalledTimes(1); + expect(files).toHaveBeenCalledTimes(1); +}); diff --git a/tests/unit/searchStreamingResults.test.js b/tests/unit/searchStreamingResults.test.js new file mode 100644 index 0000000000..7aae84054b --- /dev/null +++ b/tests/unit/searchStreamingResults.test.js @@ -0,0 +1,61 @@ +import { readFileSync } from "node:fs"; +import vm from "node:vm"; +import { expect, it } from "vitest"; + +it("appends interleaved batches without losing matches or counting files twice", () => { + const source = readFileSync( + new URL("../../src/sidebarApps/searchInFiles/index.js", import.meta.url), + "utf8", + ); + let text = ""; + const context = vm.createContext({ + filesSearched: [], + fileNames: [], + words: [], + results: [], + Tree: { fromJSON: (file) => file }, + resultOverview: { filesCount: 0, matchesCount: 0 }, + $resultOverview: {}, + searchResultText: () => "", + MAX_HL_WORDS: 0, + searchResult: { + setValue(value) { + text = value; + }, + }, + appendSearchResultText(value) { + text += value; + }, + }); + vm.runInContext( + source.slice( + source.indexOf("function appendSearchResult(data)"), + source.indexOf("function enqueueNativeSearchResults"), + ) + + source.slice( + source.indexOf("function groupMatchesForDisplay"), + source.indexOf("async function finishSearchTask"), + ), + context, + ); + const add = (url, row, limited = false) => { + context.batch = { + file: { name: url, url }, + matches: [{ line: "match", position: { start: { row, column: 0 } } }], + limited, + }; + vm.runInContext("appendSearchResult(batch)", context); + }; + add("a", 0); + add("a", 1); + add("b", 0); + add("a", 2, true); + expect(context.resultOverview).toEqual({ filesCount: 2, matchesCount: 4 }); + expect(context.fileNames.map((file) => file.count)).toEqual([3, 1]); + expect(text).toBe( + "a\n\t1: match\n\t2: match\nb\n\t1: match\na\n\t3: match\n\t... result limit reached for this file", + ); + expect(context.results.filter((r) => r.position).map((r) => r.file)).toEqual([ + 0, 0, 1, 0, + ]); +}); diff --git a/tests/unit/searchWorker.test.js b/tests/unit/searchWorker.test.js new file mode 100644 index 0000000000..42d94b9700 --- /dev/null +++ b/tests/unit/searchWorker.test.js @@ -0,0 +1,67 @@ +import { readFileSync } from "node:fs"; +import vm from "node:vm"; +import { describe, expect, it } from "vitest"; + +const source = readFileSync( + new URL("../../src/sidebarApps/searchInFiles/worker.js", import.meta.url), + "utf8", +).replace(/^import .*;\n/gm, ""); +async function search(content, regex) { + const messages = []; + const context = vm.createContext({ + self: { + postMessage(message) { + messages.push(message); + if (message.action === "search-result") + queueMicrotask(() => + context.self.onmessage({ + data: { action: "result-ack", id: message.id }, + }), + ); + }, + }, + picomatch: { isMatch: () => true }, + isBinaryFile: () => false, + }); + vm.runInContext(source, context); + context.content = content; + context.regex = regex; + await vm.runInContext( + `searchInFile({ file: { name: "test" }, content, search: regex })`, + context, + { timeout: 1000 }, + ); + const batches = messages + .filter((m) => m.action === "search-result") + .map((m) => m.data); + return { batches, matches: batches.flatMap((b) => b.matches) }; +} +describe("fallback search matching", () => { + it("advances empty Unicode matches without hanging", async () => { + const result = await search("😀x", /(?:)/gu); + expect(result.matches.map((m) => m.position.start.column)).toEqual([ + 0, 2, 3, + ]); + }); + it("tracks multiline positions and resets regex state", async () => { + const regex = /ab\ncd/g; + regex.lastIndex = 100; + const result = await search("z\nab\ncd\nab\ncd", regex); + expect(result.matches.map((m) => m.position)).toEqual([ + { start: { row: 1, column: 0 }, end: { row: 2, column: 2 } }, + { start: { row: 3, column: 0 }, end: { row: 4, column: 2 } }, + ]); + }); + it("streams every dense long-line match in bounded batches", async () => { + const result = await search("x ".repeat(100000), /x/g); + expect(result.matches).toHaveLength(100000); + expect(result.batches.every((b) => b.matches.length <= 200)).toBe(true); + expect(result.matches.every((m) => m.line.length <= 166)).toBe(true); + }); + it("bounds even a match spanning the entire document", async () => { + const result = await search("x".repeat(100000), /x+/g); + expect(result.matches[0].position.end.column).toBe(100000); + expect(result.matches[0].renderText.length).toBe(160); + expect(result.matches[0].line.length).toBeLessThanOrEqual(166); + }); +}); From 3fa502f50291dc59d16df11874402ad54a0c0bb4 Mon Sep 17 00:00:00 2001 From: Raunak Raj <71929976+bajrangCoder@users.noreply.github.com> Date: Sat, 12 Sep 2026 14:25:12 +0530 Subject: [PATCH 2/2] fix --- src/sidebarApps/searchInFiles/cmResultView.js | 15 +-- src/sidebarApps/searchInFiles/index.js | 38 ++++++-- tests/unit/searchDiscovery.test.js | 94 +++++++++++++++---- tests/unit/searchStreamingResults.test.js | 29 +++++- 4 files changed, 137 insertions(+), 39 deletions(-) diff --git a/src/sidebarApps/searchInFiles/cmResultView.js b/src/sidebarApps/searchInFiles/cmResultView.js index 05d4d839fe..0ddc754533 100644 --- a/src/sidebarApps/searchInFiles/cmResultView.js +++ b/src/sidebarApps/searchInFiles/cmResultView.js @@ -15,11 +15,11 @@ import helpers from "utils/helpers"; * @param {object} opts * @param {(lineIndex:number)=>void} opts.onLineClick * @param {()=>string[]} opts.getWords - returns list of words to highlight - * @param {()=>string[]} opts.getFileNames - returns list of filenames (used to style header lines) + * @param {(lineIndex:number)=>object} opts.getFileInfo - file metadata for a result row */ export function createSearchResultView( container, - { onLineClick, getWords, getFileNames, getRegex }, + { onLineClick, getWords, getFileInfo, getRegex }, ) { let view; let isGhostText = false; @@ -145,15 +145,11 @@ export function createSearchResultView( function buildGroupDecos(state) { const doc = state.doc; const folded = state.field(foldState, false) || new Set(); - // No removed groups - const fns = - (typeof getFileNames === "function" ? getFileNames() : []) || []; - if (isGhostText || !fns.length || doc.length === 0 || doc.lines === 0) + if (isGhostText || doc.length === 0 || doc.lines === 0) return Decoration.none; const builder = []; // Build header chevrons and collapses per group - let groupIndex = 0; eachGroup(doc, ({ start, end }) => { const header = doc.line(start); const key = start - 1; @@ -163,9 +159,7 @@ export function createSearchResultView( Decoration.line({ class: "cm-fileName" }).range(header.from), ); // File icon - const fileNames = - (typeof getFileNames === "function" ? getFileNames() : []) || []; - const fileInfo = fileNames[groupIndex] || {}; + const fileInfo = getFileInfo?.(key) || {}; const fname = typeof fileInfo === "string" ? fileInfo : fileInfo.name || ""; const iconClass = helpers.getIconForFile(fname); @@ -210,7 +204,6 @@ export function createSearchResultView( }).range(first.from), ); } - groupIndex++; }); return Decoration.set(builder, true); diff --git a/src/sidebarApps/searchInFiles/index.js b/src/sidebarApps/searchInFiles/index.js index e0253fa124..9de507be65 100644 --- a/src/sidebarApps/searchInFiles/index.js +++ b/src/sidebarApps/searchInFiles/index.js @@ -42,6 +42,8 @@ const $progress = Reactive(); const $indexStatus = Reactive(""); const FILE_LIST_WAIT_TIMEOUT = 250; +const FILE_LIST_MAX_WAIT = 5000; +let pendingDiscoveryVersion = null; const SEARCH_WORKER_COUNT = 1; const resultOverview = { @@ -144,7 +146,7 @@ $container.onref = ($el) => { searchResult = createSearchResultView($el, { onLineClick: onCursorChange, getWords: () => words, - getFileNames: () => fileNames, + getFileInfo: (line) => fileNames[results[line]?.file], getRegex: () => currentSearchRegex, }); searchResult.view.scrollDOM?.addEventListener( @@ -850,13 +852,30 @@ function getOpenFileOverlays() { async function waitForFileListIfReady(version) { const ready = waitForFileList(); - const result = await withTimeout(ready, FILE_LIST_WAIT_TIMEOUT); + pendingDiscoveryVersion = version; + let result = await withTimeout(ready, FILE_LIST_WAIT_TIMEOUT); if (version !== searchVersion) return; if (result === TIMEOUT) { $indexStatus.value = "Scanning project files..."; - await ready; + result = await withTimeout( + ready, + FILE_LIST_MAX_WAIT - FILE_LIST_WAIT_TIMEOUT, + ); + } + if (version !== searchVersion) return; + $indexStatus.value = ""; + if (result !== TIMEOUT) { + pendingDiscoveryVersion = null; + return; } - if (version === searchVersion) $indexStatus.value = ""; + $error.value = + "Project scan is still running; search results may be incomplete."; + void ready.then(() => { + if (version !== searchVersion) return; + pendingDiscoveryVersion = null; + $error.value = + "Project scan finished; search again to include newly discovered files."; + }); } function markIndexDirty(urls) { @@ -1157,6 +1176,13 @@ async function onCursorChange(line) { * When a file is added or removed from the file list * @param {import('lib/fileList').Tree} tree */ +function onFileAdded(tree) { + // Discovery emits add-file for every entry. After a timeout, retain the + // snapshot results instead of repeatedly clearing them as entries arrive. + if (pendingDiscoveryVersion === searchVersion) return; + onFileUpdate(tree); +} + function onFileUpdate(tree) { if (!tree || tree?.children) return; markIndexDirty([tree.url]); @@ -1198,7 +1224,7 @@ function resetResultScroll() { * Add event listeners to file changes */ function addEvents() { - files.on("add-file", onFileUpdate); + files.on("add-file", onFileAdded); files.on("remove-file", onFileUpdate); files.on("add-folder", onInput); files.on("remove-folder", onInput); @@ -1211,7 +1237,7 @@ function addEvents() { * Remove event listeners to file changes */ function removeEvents() { - files.off("add-file", onFileUpdate); + files.off("add-file", onFileAdded); files.off("remove-file", onFileUpdate); files.off("add-folder", onInput); files.off("remove-folder", onInput); diff --git a/tests/unit/searchDiscovery.test.js b/tests/unit/searchDiscovery.test.js index c4f8620056..c16079ddfd 100644 --- a/tests/unit/searchDiscovery.test.js +++ b/tests/unit/searchDiscovery.test.js @@ -1,32 +1,30 @@ import { readFileSync } from "node:fs"; import vm from "node:vm"; -import { expect, it, vi } from "vitest"; +import { afterEach, expect, it, vi } from "vitest"; const source = readFileSync( new URL("../../src/sidebarApps/searchInFiles/index.js", import.meta.url), "utf8", ); -function extract(name, next) { - return source.slice( - source.indexOf(`async function ${name}(`), - source.indexOf(`\n${next}`, source.indexOf(`async function ${name}(`)), - ); +function extract(start, end) { + const offset = source.indexOf(start); + return source.slice(offset, source.indexOf(end, offset)); } -it("waits for discovery before subscribing to file events or taking the search snapshot", async () => { +function setup() { + vi.useFakeTimers(); let resolve; const ready = new Promise((r) => { resolve = r; }); - const addEvents = vi.fn(); - const files = vi.fn(() => []); const context = vm.createContext({ $search: { value: "needle" }, getOptions: () => ({}), toRegex: () => /needle/g, searchVersion: 1, + pendingDiscoveryVersion: null, waitForFileList: () => ready, - addEvents, - files, + addEvents: vi.fn(), + files: vi.fn(() => [{ url: "github://test/a.js" }]), helpers: { isBinary: () => false }, addedFolder: [], editorManager: { files: [] }, @@ -34,22 +32,78 @@ it("waits for discovery before subscribing to file events or taking the search s strings: {}, $progress: {}, $indexStatus: {}, + $error: {}, TIMEOUT: Symbol(), FILE_LIST_WAIT_TIMEOUT: 250, - withTimeout: async () => context.TIMEOUT, + FILE_LIST_MAX_WAIT: 5000, + setTimeout, + clearTimeout, + words: [], + fileNames: [], + supportsNativeSearch: () => false, + sendMessage: vi.fn(), + onFileUpdate: vi.fn(), }); vm.runInContext( - extract("searchAll", "async function readSearchFileContent") + - extract("waitForFileListIfReady", "function markIndexDirty"), + extract( + "async function searchAll(", + "async function readSearchFileContent", + ) + + extract( + "async function waitForFileListIfReady(", + "function markIndexDirty", + ) + + extract("function withTimeout(", "\n/**") + + extract("function onFileAdded(", "function onFileUpdate("), context, ); - const pending = vm.runInContext("searchAll()", context); - await new Promise((resolve) => setImmediate(resolve)); - expect(addEvents).not.toHaveBeenCalled(); - expect(files).not.toHaveBeenCalled(); + return { context, resolve, pending: vm.runInContext("searchAll()", context) }; +} +afterEach(() => vi.useRealTimers()); +it("waits for discovery before subscribing or taking the search snapshot", async () => { + const { context, resolve, pending } = setup(); + await vi.advanceTimersByTimeAsync(250); + expect(context.addEvents).not.toHaveBeenCalled(); + expect(context.files).not.toHaveBeenCalled(); expect(context.$indexStatus.value).toBe("Scanning project files..."); resolve(); await pending; - expect(addEvents).toHaveBeenCalledTimes(1); - expect(files).toHaveBeenCalledTimes(1); + expect(context.addEvents).toHaveBeenCalledTimes(1); + expect(context.sendMessage).toHaveBeenCalledTimes(1); + expect(context.pendingDiscoveryVersion).toBeNull(); +}); +it("starts snapshot search after five seconds and suppresses discovery restarts", async () => { + const { context, resolve, pending } = setup(); + await vi.advanceTimersByTimeAsync(5000); + await pending; + expect(context.sendMessage).toHaveBeenCalledTimes(1); + expect(context.$error.value).toContain("incomplete"); + vm.runInContext('onFileAdded({ url: "github://test/new.js" })', context); + expect(context.onFileUpdate).not.toHaveBeenCalled(); + resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(context.$error.value).toContain("search again"); + expect(context.sendMessage).toHaveBeenCalledTimes(1); + vm.runInContext('onFileAdded({ url: "github://test/manual.js" })', context); + expect(context.onFileUpdate).toHaveBeenCalledTimes(1); +}); +it("does not start a stale search when the query changes during discovery", async () => { + const { context, pending } = setup(); + await vi.advanceTimersByTimeAsync(250); + context.searchVersion = 2; + await vi.advanceTimersByTimeAsync(4750); + await pending; + expect(context.sendMessage).not.toHaveBeenCalled(); +}); +it("ignores late readiness from a superseded query", async () => { + const { context, resolve, pending } = setup(); + await vi.advanceTimersByTimeAsync(5000); + await pending; + context.searchVersion = 2; + context.pendingDiscoveryVersion = 2; + context.$error.value = "new query"; + resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(context.$error.value).toBe("new query"); + expect(context.pendingDiscoveryVersion).toBe(2); }); diff --git a/tests/unit/searchStreamingResults.test.js b/tests/unit/searchStreamingResults.test.js index 7aae84054b..2ad8e630b6 100644 --- a/tests/unit/searchStreamingResults.test.js +++ b/tests/unit/searchStreamingResults.test.js @@ -1,10 +1,16 @@ +// @vitest-environment happy-dom import { readFileSync } from "node:fs"; import vm from "node:vm"; -import { expect, it } from "vitest"; +import { expect, it, vi } from "vitest"; +import { createSearchResultView } from "sidebarApps/searchInFiles/cmResultView"; +vi.mock("lib/settings", () => ({ default: { value: {} } })); +vi.mock("utils/helpers", () => ({ + default: { getIconForFile: (name) => `icon-${name}` }, +})); it("appends interleaved batches without losing matches or counting files twice", () => { const source = readFileSync( - new URL("../../src/sidebarApps/searchInFiles/index.js", import.meta.url), + `${process.cwd()}/src/sidebarApps/searchInFiles/index.js`, "utf8", ); let text = ""; @@ -58,4 +64,23 @@ it("appends interleaved batches without losing matches or counting files twice", expect(context.results.filter((r) => r.position).map((r) => r.file)).toEqual([ 0, 0, 1, 0, ]); + const container = document.createElement("div"); + document.body.append(container); + const resultView = createSearchResultView(container, { + getFileInfo: (line) => context.fileNames[context.results[line]?.file], + getWords: () => [], + }); + try { + resultView.setValue(text); + expect( + [...container.querySelectorAll(".cm-fileCount")].map( + (el) => el.textContent, + ), + ).toEqual(["3", "1", "3"]); + expect(container.querySelectorAll(".icon-a")).toHaveLength(2); + expect(container.querySelectorAll(".icon-b")).toHaveLength(1); + } finally { + resultView.view.destroy(); + container.remove(); + } });