|
| 1 | +import { isPlainRecord } from '@sim/utils/object' |
| 2 | +import { describe, expect, it } from 'vitest' |
| 3 | +import { crawlTool } from '@/tools/firecrawl/crawl' |
| 4 | + |
| 5 | +function crawlBody(options: { |
| 6 | + maxDepth: number |
| 7 | + maxDiscoveryDepth?: number |
| 8 | + includePaths?: string[] |
| 9 | + excludePaths?: string[] |
| 10 | +}) { |
| 11 | + if (typeof crawlTool.request.body !== 'function') throw new Error('Missing request formatter') |
| 12 | + const body = crawlTool.request.body({ apiKey: 'test', url: 'https://example.com', ...options }) |
| 13 | + if (!isPlainRecord(body)) throw new Error('Expected a JSON request body') |
| 14 | + return body |
| 15 | +} |
| 16 | + |
| 17 | +describe('Firecrawl legacy crawl depth', () => { |
| 18 | + it('bounds URL path segments without limiting discovery hops', () => { |
| 19 | + const body = crawlBody({ maxDepth: 1, maxDiscoveryDepth: 4 }) |
| 20 | + const patterns = (body.includePaths as string[]).map((pattern) => new RegExp(pattern)) |
| 21 | + const accepts = (path: string) => patterns.some((pattern) => pattern.test(path)) |
| 22 | + |
| 23 | + expect(body.maxDiscoveryDepth).toBe(4) |
| 24 | + expect(body.regexOnFullURL).toBe(false) |
| 25 | + expect(accepts('/')).toBe(true) |
| 26 | + expect(accepts('/docs')).toBe(true) |
| 27 | + expect(accepts('/docs/index.html')).toBe(true) |
| 28 | + expect(accepts('/docs/guide')).toBe(false) |
| 29 | + expect(accepts('/docs/guide/page')).toBe(false) |
| 30 | + }) |
| 31 | + |
| 32 | + it('keeps existing path filters while excluding paths beyond the legacy depth', () => { |
| 33 | + const body = crawlBody({ maxDepth: 1, includePaths: ['^/docs'], excludePaths: ['private'] }) |
| 34 | + const patterns = (body.excludePaths as string[]).map((pattern) => new RegExp(pattern)) |
| 35 | + const excludes = (path: string) => patterns.some((pattern) => pattern.test(path)) |
| 36 | + |
| 37 | + expect(body.includePaths).toEqual(['^/docs']) |
| 38 | + expect(excludes('/docs')).toBe(false) |
| 39 | + expect(excludes('/docs/guide')).toBe(true) |
| 40 | + expect(excludes('/private')).toBe(true) |
| 41 | + expect(body.maxDiscoveryDepth).toBeUndefined() |
| 42 | + }) |
| 43 | + |
| 44 | + it('honors a zero depth instead of dropping the limit', () => { |
| 45 | + const body = crawlBody({ maxDepth: 0 }) |
| 46 | + const patterns = (body.includePaths as string[]).map((pattern) => new RegExp(pattern)) |
| 47 | + expect(patterns.some((pattern) => pattern.test('/'))).toBe(true) |
| 48 | + expect(patterns.some((pattern) => pattern.test('/docs'))).toBe(false) |
| 49 | + }) |
| 50 | + |
| 51 | + it.each<[number, string, boolean]>([ |
| 52 | + [0, '/index.html', false], |
| 53 | + [0, '/index.php/', false], |
| 54 | + [1, '/docs/index.html', false], |
| 55 | + [1, '/docs/index.php', false], |
| 56 | + [1, '/docs//index.html/', false], |
| 57 | + [1, '/docs/guide/index.html', true], |
| 58 | + [1, '/docs/index.html/child', true], |
| 59 | + [1, '/docs/index.html.bak', true], |
| 60 | + [1, '/docs/index.phpx', true], |
| 61 | + [1, '/docs/inde', true], |
| 62 | + [1, '/docs/index.htm', true], |
| 63 | + ])('applies depth %s with existing includes to %s', (maxDepth, path, excluded) => { |
| 64 | + const body = crawlBody({ maxDepth, includePaths: ['^/'] }) |
| 65 | + const patterns = (body.excludePaths as string[]).map((pattern) => new RegExp(pattern)) |
| 66 | + expect(patterns.some((pattern) => pattern.test(path))).toBe(excluded) |
| 67 | + }) |
| 68 | + |
| 69 | + it.each([-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])( |
| 70 | + 'rejects an invalid legacy depth: %s', |
| 71 | + (maxDepth) => { |
| 72 | + expect(() => crawlBody({ maxDepth })).toThrow(/maxDepth/) |
| 73 | + } |
| 74 | + ) |
| 75 | +}) |
0 commit comments