fix(release): leave Firecrawl crawl depth to #9864

Author: waleedlatif1

apps/sim/tools/firecrawl/crawl.test.ts

@@ -1,11 +1,10 @@
 import { isPlainRecord } from '@sim/utils/object'
 import { describe, expect, it } from 'vitest'
 import { crawlTool } from '@/tools/firecrawl/crawl'
-import type { FirecrawlCrawlParams } from '@/tools/firecrawl/types'
 
 function crawlBody(options: {
-  url?: string
   maxDepth: number
+  url?: string
   maxDiscoveryDepth?: number
   includePaths?: string[]
   excludePaths?: string[]
@@ -68,40 +67,35 @@ describe('Firecrawl legacy crawl depth', () => {
     expect(patterns.some((pattern) => pattern.test(path))).toBe(excluded)
   })
 
-  it.each([-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])(
-    'rejects an invalid legacy depth: %s',
-    (maxDepth) => {
-      expect(() => crawlBody({ maxDepth })).toThrow(/maxDepth/)
-    }
-  )
-
-  it('measures depth below the start URL, as Firecrawl v1 did', () => {
-    const body = crawlBody({ url: 'https://example.com/guide/intro', maxDepth: 1 })
+  it.each<[string, string, boolean]>([
+    ['https://example.com/docs/guide', '/docs/guide', true],
+    ['https://example.com/docs/guide/', '/docs/guide/setup', true],
+    ['https://example.com/docs/guide/index.html', '/docs/guide/setup', true],
+    ['https://example.com/docs/guide', '/docs/guide/setup/advanced', false],
+    ['docs.example.com/guide', '/guide/setup', true],
+  ])('counts depth from the start URL %s for %s', (url, path, accepted) => {
+    const body = crawlBody({ maxDepth: 1, url })
     const patterns = (body.includePaths as string[]).map((pattern) => new RegExp(pattern))
-    const accepts = (path: string) => patterns.some((pattern) => pattern.test(path))
-
-    expect(accepts('/guide/intro')).toBe(true)
-    expect(accepts('/guide/intro/setup')).toBe(true)
-    expect(accepts('/guide/intro/setup/advanced')).toBe(false)
+    expect(patterns.some((pattern) => pattern.test(path))).toBe(accepted)
   })
 
-  it('ignores an index file in the start URL when measuring its depth', () => {
-    const body = crawlBody({ url: 'https://example.com/docs/index.html', maxDepth: 0 })
-    const patterns = (body.includePaths as string[]).map((pattern) => new RegExp(pattern))
-    expect(patterns.some((pattern) => pattern.test('/docs'))).toBe(true)
-    expect(patterns.some((pattern) => pattern.test('/docs/guide'))).toBe(false)
+  it('counts depth from the start URL alongside existing path filters', () => {
+    const body = crawlBody({
+      maxDepth: 1,
+      url: 'https://example.com/docs/guide',
+      includePaths: ['^/docs'],
+    })
+    const patterns = (body.excludePaths as string[]).map((pattern) => new RegExp(pattern))
+    const excludes = (path: string) => patterns.some((pattern) => pattern.test(path))
+    expect(excludes('/docs/guide')).toBe(false)
+    expect(excludes('/docs/guide/setup')).toBe(false)
+    expect(excludes('/docs/guide/setup/advanced')).toBe(true)
   })
 
-  it('treats a blank depth as unset instead of a zero-depth crawl', () => {
-    if (typeof crawlTool.request.body !== 'function') throw new Error('Missing request formatter')
-    const blankDepth: Record<string, unknown> = {
-      apiKey: 'test',
-      url: 'https://example.com',
-      maxDepth: '',
+  it.each([-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])(
+    'rejects an invalid legacy depth: %s',
+    (maxDepth) => {
+      expect(() => crawlBody({ maxDepth })).toThrow(/maxDepth/)
     }
-    const body = crawlTool.request.body(blankDepth as FirecrawlCrawlParams)
-    if (!isPlainRecord(body)) throw new Error('Expected a JSON request body')
-    expect(body.includePaths).toBeUndefined()
-    expect(body.regexOnFullURL).toBeUndefined()
-  })
+  )
 })

apps/sim/tools/firecrawl/crawl.ts

@@ -18,16 +18,17 @@ const logger = createLogger('FirecrawlCrawlTool')
 const POLL_INTERVAL_MS = 5000
 const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS
 
-/** Path segments in a URL, ignoring index files, as Firecrawl v1 measured crawl depth. */
-function startUrlPathDepth(url: string): number {
-  try {
-    return new URL(url).pathname
-      .split('/')
-      .filter((segment) => segment !== '' && segment !== 'index.html' && segment !== 'index.php')
-      .length
-  } catch {
-    return 0
-  }
+/**
+ * Path segments of a URL as Firecrawl v1 counted them: index files add no depth. A URL without
+ * a scheme is read as https, as Firecrawl reads it.
+ */
+function urlPathDepth(url: string): number {
+  const absolute = /^[a-z][a-z\d+.-]*:\/\//i.test(url) ? url : `https://${url}`
+  if (!URL.canParse(absolute)) return 0
+  return new URL(absolute).pathname
+    .split('/')
+    .filter((segment) => segment !== '' && segment !== 'index.html' && segment !== 'index.php')
+    .length
 }
 
 export const crawlTool: ToolConfig<FirecrawlCrawlParams, FirecrawlCrawlResponse> = {
@@ -54,7 +55,7 @@ export const crawlTool: ToolConfig<FirecrawlCrawlParams, FirecrawlCrawlResponse>
       required: false,
       visibility: 'user-or-llm',
       description:
-        'Maximum URL path depth (e.g., 1 allows /docs and /docs/index.html). Independent of link discovery depth',
+        'Maximum URL path depth below the starting URL (e.g., from /docs, 1 allows /docs/guide). Independent of link discovery depth',
     },
     formats: {
       type: 'json',
@@ -167,14 +168,15 @@ export const crawlTool: ToolConfig<FirecrawlCrawlParams, FirecrawlCrawlResponse>
       if (params.maxConcurrency) body.maxConcurrency = Number(params.maxConcurrency)
       if (params.excludePaths) body.excludePaths = params.excludePaths
       if (params.includePaths) body.includePaths = params.includePaths
-      if (params.maxDepth != null && String(params.maxDepth).trim() !== '') {
+      if (params.maxDepth != null) {
         const requestedDepth = Number(params.maxDepth)
         if (!Number.isSafeInteger(requestedDepth) || requestedDepth < 0) {
           throw new Error('maxDepth must be a non-negative integer')
         }
-        // v1 counted path segments below the start URL's own depth; v2 discovery depth counts link hops.
-        const maxDepth = requestedDepth + startUrlPathDepth(params.url)
+        // v1 counted depth from the start URL, so the start page itself always passes the filter.
+        const maxDepth = requestedDepth + urlPathDepth(params.url)
 
+        // v1 depth counted URL path segments; v2 discovery depth counts link hops.
         body.regexOnFullURL = false
         if (params.includePaths?.length) {
           const indexFiles = ['index.html', 'index.php']