Skip to content

Commit a80c5e3

Browse files
chenjiahanchenjiahan
authored
perf(fmt): batch config ignore matching during discovery (#398)
1 parent bbef70a commit a80c5e3

6 files changed

Lines changed: 480 additions & 120 deletions

File tree

‎packages/rstack/src/fmt/discoverPaths.ts‎

Lines changed: 232 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,11 @@ import micromatch from 'micromatch';
55
import readdir, { type Dirent, type DirentLike } from 'tiny-readdir';
66
import type { GitIgnoreMatcher as NativeGitIgnoreMatcher } from '../../binding.cjs';
77
import { loadNativeBinding } from '../native/index.ts';
8+
import type {
9+
BatchIgnoreContext,
10+
IgnoreMatcher,
11+
IgnorePredicate,
12+
} from './ignore.ts';
813
import {
914
createRelativePathResolver,
1015
toPosixPath,
@@ -21,8 +26,18 @@ const defaultIgnoredDirNames = new Set([
2126
'node_modules',
2227
]);
2328

24-
const gitIgnored = Symbol('gitIgnored');
25-
type GitIgnoreDirent = Dirent & { [gitIgnored]?: true };
29+
const ignored = Symbol('ignored');
30+
type IgnoredDirent = Dirent & { [ignored]?: true };
31+
type TraversalIgnorePredicate = ((
32+
filePath: string,
33+
isDirectory: boolean,
34+
) => boolean) &
35+
Pick<IgnoreMatcher, 'batch'>;
36+
37+
interface GitIgnoreBatchContext {
38+
readonly matcher: NativeGitIgnoreMatcher;
39+
readonly relativeParent: string;
40+
}
2641

2742
interface DiscoverFmtPathsOptions {
2843
/** Absolute directory used to resolve input paths. */
@@ -31,7 +46,7 @@ interface DiscoverFmtPathsOptions {
3146
/** Whether files inside node_modules may be discovered. */
3247
withNodeModules?: boolean;
3348
/** Returns whether a candidate path should be excluded. */
34-
isIgnored?: (filePath: string, isDirectory: boolean) => boolean;
49+
isIgnored?: TraversalIgnorePredicate;
3550
}
3651

3752
const isErrnoException = (error: unknown): error is NodeJS.ErrnoException =>
@@ -141,55 +156,20 @@ class GitIgnoreFiles {
141156
return this.#matcher!.isIgnored(toPosixPath(relativePath), isDirectory);
142157
}
143158

144-
/** Matches one directory's entries in a single native call. */
145-
matchDirents(
146-
parentPath: string,
147-
dirents: Dirent[],
148-
): boolean | number | Uint8Array | undefined {
149-
if (!this.#hasRules || dirents.length === 0) {
159+
resolveBatchContext(parentPath: string): GitIgnoreBatchContext | undefined {
160+
if (!this.#hasRules) {
150161
return;
151162
}
152163

153-
const relativeParentPath = this.#resolveRelativePath(parentPath);
154-
if (!isRelativePathInside(relativeParentPath)) {
164+
const relativeParent = this.#resolveRelativePath(parentPath);
165+
if (!isRelativePathInside(relativeParent)) {
155166
return;
156167
}
157168

158-
const relativeParent = toPosixPath(relativeParentPath);
159-
160-
if (dirents.length === 1) {
161-
const dirent = dirents[0];
162-
return this.#matcher!.isIgnoredChild(
163-
relativeParent,
164-
dirent.name,
165-
dirent.isDirectory(),
166-
);
167-
}
168-
169-
const names = new Array<string>(dirents.length);
170-
171-
if (dirents.length <= 32) {
172-
let directoryMask = 0;
173-
for (let index = 0; index < dirents.length; index++) {
174-
const dirent = dirents[index];
175-
names[index] = dirent.name;
176-
directoryMask |= Number(dirent.isDirectory()) << index;
177-
}
178-
return this.#matcher!.isIgnoredBatchMask(
179-
relativeParent,
180-
names,
181-
directoryMask >>> 0,
182-
);
183-
}
184-
185-
const directoryFlags = new Uint8Array(dirents.length);
186-
for (let index = 0; index < dirents.length; index++) {
187-
const dirent = dirents[index];
188-
names[index] = dirent.name;
189-
directoryFlags[index] = Number(dirent.isDirectory());
190-
}
191-
192-
return this.#matcher!.isIgnoredBatch(relativeParent, names, directoryFlags);
169+
return {
170+
matcher: this.#matcher!,
171+
relativeParent: toPosixPath(relativeParent),
172+
};
193173
}
194174

195175
#load(directoryPath: string): Promise<void> {
@@ -218,39 +198,227 @@ class GitIgnoreFiles {
218198
}
219199
}
220200

201+
const isIgnoredBeforeNative = (
202+
parentPath: string,
203+
dirent: Dirent,
204+
ignoredDirNames: ReadonlySet<string>,
205+
isIncluded: ((filePath: string) => boolean) | undefined,
206+
precheck: IgnorePredicate | undefined,
207+
): boolean => {
208+
if (ignoredDirNames.has(dirent.name)) {
209+
return true;
210+
}
211+
212+
const isDirectory = dirent.isDirectory();
213+
let targetPath: string | undefined;
214+
if (!isDirectory && isIncluded) {
215+
targetPath = path.join(parentPath, dirent.name);
216+
if (!isIncluded(targetPath)) {
217+
return true;
218+
}
219+
}
220+
if (!isDirectory && isBinaryPath(dirent.name)) {
221+
return true;
222+
}
223+
224+
return (
225+
precheck?.(
226+
targetPath ?? path.join(parentPath, dirent.name),
227+
isDirectory,
228+
) === true
229+
);
230+
};
231+
232+
/** Matches one directory after earlier traversal rules have removed candidates. */
233+
const markIgnoredDirents = (
234+
parentPath: string,
235+
dirents: Dirent[],
236+
gitIgnore: GitIgnoreFiles,
237+
ignoredDirNames: ReadonlySet<string>,
238+
isIncluded: ((filePath: string) => boolean) | undefined,
239+
batchIgnore?: BatchIgnoreContext,
240+
): void => {
241+
const gitIgnoreContext = gitIgnore.resolveBatchContext(parentPath);
242+
243+
if (dirents.length === 1) {
244+
const dirent = dirents[0];
245+
if (
246+
gitIgnoreContext?.matcher.isIgnoredChild(
247+
gitIgnoreContext.relativeParent,
248+
dirent.name,
249+
dirent.isDirectory(),
250+
) === true ||
251+
isIgnoredBeforeNative(
252+
parentPath,
253+
dirent,
254+
ignoredDirNames,
255+
isIncluded,
256+
batchIgnore?.precheck,
257+
) ||
258+
batchIgnore?.matcher.isIgnoredChild(
259+
parentPath,
260+
dirent.name,
261+
dirent.isDirectory(),
262+
) === true
263+
) {
264+
(dirent as IgnoredDirent)[ignored] = true;
265+
}
266+
return;
267+
}
268+
269+
const names = new Array<string>(dirents.length);
270+
271+
if (dirents.length <= 32) {
272+
let directoryMask = 0;
273+
for (let index = 0; index < dirents.length; index++) {
274+
const dirent = dirents[index];
275+
names[index] = dirent.name;
276+
directoryMask |= Number(dirent.isDirectory()) << index;
277+
}
278+
279+
let ignoredMask = gitIgnoreContext
280+
? gitIgnoreContext.matcher.isIgnoredBatchMask(
281+
gitIgnoreContext.relativeParent,
282+
names,
283+
directoryMask >>> 0,
284+
)
285+
: 0;
286+
287+
if (batchIgnore) {
288+
for (let index = 0; index < dirents.length; index++) {
289+
const entryMask = 1 << index;
290+
if (
291+
(ignoredMask & entryMask) === 0 &&
292+
isIgnoredBeforeNative(
293+
parentPath,
294+
dirents[index],
295+
ignoredDirNames,
296+
isIncluded,
297+
batchIgnore.precheck,
298+
)
299+
) {
300+
ignoredMask |= entryMask;
301+
}
302+
}
303+
304+
const validMask = 0xffffffff >>> (32 - dirents.length);
305+
const candidateMask = (validMask & ~ignoredMask) >>> 0;
306+
if (candidateMask !== 0) {
307+
ignoredMask =
308+
(ignoredMask |
309+
batchIgnore.matcher.isIgnoredBatchMask(
310+
parentPath,
311+
names,
312+
directoryMask >>> 0,
313+
candidateMask,
314+
)) >>>
315+
0;
316+
}
317+
}
318+
319+
for (let index = 0; index < dirents.length; index++) {
320+
if ((ignoredMask & (1 << index)) !== 0) {
321+
(dirents[index] as IgnoredDirent)[ignored] = true;
322+
}
323+
}
324+
return;
325+
}
326+
327+
const directoryFlags = new Uint8Array(dirents.length);
328+
for (let index = 0; index < dirents.length; index++) {
329+
const dirent = dirents[index];
330+
names[index] = dirent.name;
331+
directoryFlags[index] = Number(dirent.isDirectory());
332+
}
333+
334+
const ignoredFlags = gitIgnoreContext
335+
? gitIgnoreContext.matcher.isIgnoredBatch(
336+
gitIgnoreContext.relativeParent,
337+
names,
338+
directoryFlags,
339+
)
340+
: new Uint8Array(dirents.length);
341+
342+
if (batchIgnore) {
343+
const candidateFlags = new Uint8Array(dirents.length);
344+
let candidateCount = 0;
345+
for (let index = 0; index < dirents.length; index++) {
346+
if (ignoredFlags[index] === 0) {
347+
if (
348+
isIgnoredBeforeNative(
349+
parentPath,
350+
dirents[index],
351+
ignoredDirNames,
352+
isIncluded,
353+
batchIgnore.precheck,
354+
)
355+
) {
356+
ignoredFlags[index] = 1;
357+
} else {
358+
candidateFlags[index] = 1;
359+
candidateCount++;
360+
}
361+
}
362+
}
363+
364+
if (candidateCount !== 0) {
365+
const nextIgnored = batchIgnore.matcher.isIgnoredBatch(
366+
parentPath,
367+
names,
368+
directoryFlags,
369+
candidateFlags,
370+
);
371+
for (let index = 0; index < dirents.length; index++) {
372+
ignoredFlags[index] |= nextIgnored[index];
373+
}
374+
}
375+
}
376+
377+
for (let index = 0; index < dirents.length; index++) {
378+
if (ignoredFlags[index] !== 0) {
379+
(dirents[index] as IgnoredDirent)[ignored] = true;
380+
}
381+
}
382+
};
383+
221384
const createTraversalOptions = (
222385
gitIgnore: GitIgnoreFiles,
223386
ignoredDirNames: ReadonlySet<string>,
224387
signal: { aborted: boolean },
225388
onError: (error: unknown) => void,
226389
isIncluded?: (filePath: string) => boolean,
227-
isIgnored?: (filePath: string, isDirectory: boolean) => boolean,
390+
isIgnored?: TraversalIgnorePredicate,
228391
) => {
392+
const batchIgnore = isIgnored?.batch;
393+
const scalarIgnore = batchIgnore ? undefined : isIgnored;
394+
229395
return {
230396
followSymlinks: false,
231397
signal,
232398
ignore: (targetPath: string, targetContext: DirentLike) => {
233399
// With symlink following disabled, tiny-readdir always provides a Dirent here.
234400
const dirent = targetContext as Dirent;
235-
if (ignoredDirNames.has(dirent.name)) {
401+
if (
402+
(dirent as IgnoredDirent)[ignored] === true ||
403+
ignoredDirNames.has(dirent.name)
404+
) {
236405
return true;
237406
}
238407

408+
if (batchIgnore) {
409+
return false;
410+
}
411+
239412
if (dirent.isDirectory()) {
240-
return (
241-
(dirent as GitIgnoreDirent)[gitIgnored] === true ||
242-
isIgnored?.(targetPath, true) === true
243-
);
413+
return scalarIgnore?.(targetPath, true) === true;
244414
}
245415

246416
if (isIncluded !== undefined && !isIncluded(targetPath)) {
247417
return true;
248418
}
249419

250420
return (
251-
isIgnored?.(targetPath, false) === true ||
252-
isBinaryPath(targetPath) ||
253-
(dirent as GitIgnoreDirent)[gitIgnored] === true
421+
scalarIgnore?.(targetPath, false) === true || isBinaryPath(targetPath)
254422
);
255423
},
256424
onDirents: async (dirents: Dirent[]) => {
@@ -268,24 +436,14 @@ const createTraversalOptions = (
268436
await gitIgnore.load(parentPath);
269437
}
270438

271-
const ignored = gitIgnore.matchDirents(parentPath, dirents);
272-
if (typeof ignored === 'boolean') {
273-
if (ignored) {
274-
(dirents[0] as GitIgnoreDirent)[gitIgnored] = true;
275-
}
276-
} else if (typeof ignored === 'number') {
277-
for (let index = 0; index < dirents.length; index++) {
278-
if (ignored & (1 << index)) {
279-
(dirents[index] as GitIgnoreDirent)[gitIgnored] = true;
280-
}
281-
}
282-
} else if (ignored) {
283-
for (let index = 0; index < ignored.length; index++) {
284-
if (ignored[index] === 1) {
285-
(dirents[index] as GitIgnoreDirent)[gitIgnored] = true;
286-
}
287-
}
288-
}
439+
markIgnoredDirents(
440+
parentPath,
441+
dirents,
442+
gitIgnore,
443+
ignoredDirNames,
444+
isIncluded,
445+
batchIgnore,
446+
);
289447
} catch (error) {
290448
onError(error);
291449
}
@@ -300,7 +458,7 @@ const discoverDirectoryFiles = async (
300458
gitIgnore: GitIgnoreFiles,
301459
ignoredDirNames: ReadonlySet<string>,
302460
isIncluded?: (filePath: string) => boolean,
303-
isIgnored?: (filePath: string, isDirectory: boolean) => boolean,
461+
isIgnored?: TraversalIgnorePredicate,
304462
): Promise<string[]> => {
305463
let failed = false;
306464
let failure: unknown;

0 commit comments

Comments
 (0)