fix(firecrawl): preserve index pages with crawl path filters

Author: waleedlatif1

apps/sim/tools/firecrawl/crawl.test.ts

@@ -48,6 +48,24 @@ describe('Firecrawl legacy crawl depth', () => {
     expect(patterns.some((pattern) => pattern.test('/docs'))).toBe(false)
   })
 
+  it.each<[number, string, boolean]>([
+    [0, '/index.html', false],
+    [0, '/index.php/', false],
+    [1, '/docs/index.html', false],
+    [1, '/docs/index.php', false],
+    [1, '/docs//index.html/', false],
+    [1, '/docs/guide/index.html', true],
+    [1, '/docs/index.html/child', true],
+    [1, '/docs/index.html.bak', true],
+    [1, '/docs/index.phpx', true],
+    [1, '/docs/inde', true],
+    [1, '/docs/index.htm', true],
+  ])('applies depth %s with existing includes to %s', (maxDepth, path, excluded) => {
+    const body = crawlBody({ maxDepth, includePaths: ['^/'] })
+    const patterns = (body.excludePaths as string[]).map((pattern) => new RegExp(pattern))
+    expect(patterns.some((pattern) => pattern.test(path))).toBe(excluded)
+  })
+
   it.each([-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])(
     'rejects an invalid legacy depth: %s',
     (maxDepth) => {

apps/sim/tools/firecrawl/crawl.ts

@@ -1,5 +1,6 @@
 import { createLogger } from '@sim/logger'
 import { sleep } from '@sim/utils/helpers'
+import { escapeRegExp } from '@sim/utils/string'
 import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits'
 import { firecrawlHosting } from '@/tools/firecrawl/hosting'
 import {
@@ -163,7 +164,28 @@ export const crawlTool: ToolConfig<FirecrawlCrawlParams, FirecrawlCrawlResponse>
         // v1 depth counted URL path segments; v2 discovery depth counts link hops.
         body.regexOnFullURL = false
         if (params.includePaths?.length) {
-          body.excludePaths = [...(params.excludePaths ?? []), `^(/+[^/]+){${maxDepth + 1}}`]
+          const indexFiles = ['index.html', 'index.php']
+          const nonIndexSegmentPatterns = new Set<string>()
+          // Rust regex has no lookarounds, so enumerate incomplete prefixes and first mismatches.
+          for (const file of indexFiles) {
+            for (let length = 0; length <= file.length; length++) {
+              const prefix = file.slice(0, length)
+              const nextCharacters = indexFiles
+                .filter((indexFile) => indexFile.startsWith(prefix))
+                .map((indexFile) => indexFile.charAt(length))
+                .join('')
+              const escapedPrefix = escapeRegExp(prefix)
+              if (prefix && !indexFiles.includes(prefix)) {
+                nonIndexSegmentPatterns.add(escapedPrefix)
+              }
+              nonIndexSegmentPatterns.add(`${escapedPrefix}[^/${nextCharacters}][^/]*`)
+            }
+          }
+          const nonIndexBoundary = `^(/+[^/]+){${maxDepth}}/+(${[...nonIndexSegmentPatterns].join('|')})/*$`
+          body.excludePaths = [
+            ...(params.excludePaths ?? []),
+            `^(/+[^/]+){${maxDepth + 2}}|${nonIndexBoundary}`,
+          ]
         } else {
           body.includePaths = [
             `^(/+[^/]+){0,${maxDepth}}/*$`,