From cd0e043860c63ae0d6c5f3a8c8822b972eced5f9 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 10 Nov 2026 15:16:07 -0800 Subject: [PATCH 1/4] fix(release): measure Firecrawl crawl depth from the start URL and reuse only full-access delegation keys --- apps/sim/lib/mothership/chat/delegation.ts | 2 ++ apps/sim/tools/firecrawl/crawl.test.ts | 32 ++++++++++++++++++++++ apps/sim/tools/firecrawl/crawl.ts | 21 +++++++++++--- 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/apps/sim/lib/mothership/chat/delegation.ts b/apps/sim/lib/mothership/chat/delegation.ts index 18b46064da2..80e313e3ab9 100644 --- a/apps/sim/lib/mothership/chat/delegation.ts +++ b/apps/sim/lib/mothership/chat/delegation.ts @@ -49,6 +49,7 @@ export async function mintDelegationToken(params: { eq(apiKey.userId, params.userId), eq(apiKey.name, name), eq(apiKey.type, 'personal'), + eq(apiKey.access, 'write'), gt(apiKey.expiresAt, new Date(Date.now() + MIN_REMAINING_MS)) ) ) @@ -83,6 +84,7 @@ export async function mintDelegationToken(params: { key: stored, keyHash: hashApiKey(plainKey), type: 'personal', + access: 'write', createdAt: now, updatedAt: now, expiresAt, diff --git a/apps/sim/tools/firecrawl/crawl.test.ts b/apps/sim/tools/firecrawl/crawl.test.ts index f14a4c70fc8..61f428b1e3f 100644 --- a/apps/sim/tools/firecrawl/crawl.test.ts +++ b/apps/sim/tools/firecrawl/crawl.test.ts @@ -1,8 +1,10 @@ import { isPlainRecord } from '@sim/utils/object' import { describe, expect, it } from 'vitest' import { crawlTool } from '@/tools/firecrawl/crawl' +import type { FirecrawlCrawlParams } from '@/tools/firecrawl/types' function crawlBody(options: { + url?: string maxDepth: number maxDiscoveryDepth?: number includePaths?: string[] @@ -72,4 +74,34 @@ describe('Firecrawl legacy crawl depth', () => { expect(() => crawlBody({ maxDepth })).toThrow(/maxDepth/) } ) + + it('measures depth below the start URL, as Firecrawl v1 did', () => { + const body = crawlBody({ url: 'https://example.com/guide/intro', maxDepth: 1 }) + const patterns = (body.includePaths as string[]).map((pattern) => new RegExp(pattern)) + const accepts = (path: string) => patterns.some((pattern) => pattern.test(path)) + + expect(accepts('/guide/intro')).toBe(true) + expect(accepts('/guide/intro/setup')).toBe(true) + expect(accepts('/guide/intro/setup/advanced')).toBe(false) + }) + + it('ignores an index file in the start URL when measuring its depth', () => { + const body = crawlBody({ url: 'https://example.com/docs/index.html', maxDepth: 0 }) + const patterns = (body.includePaths as string[]).map((pattern) => new RegExp(pattern)) + expect(patterns.some((pattern) => pattern.test('/docs'))).toBe(true) + expect(patterns.some((pattern) => pattern.test('/docs/guide'))).toBe(false) + }) + + it('treats a blank depth as unset instead of a zero-depth crawl', () => { + if (typeof crawlTool.request.body !== 'function') throw new Error('Missing request formatter') + const blankDepth: Record = { + apiKey: 'test', + url: 'https://example.com', + maxDepth: '', + } + const body = crawlTool.request.body(blankDepth as FirecrawlCrawlParams) + if (!isPlainRecord(body)) throw new Error('Expected a JSON request body') + expect(body.includePaths).toBeUndefined() + expect(body.regexOnFullURL).toBeUndefined() + }) }) diff --git a/apps/sim/tools/firecrawl/crawl.ts b/apps/sim/tools/firecrawl/crawl.ts index 15d16907bff..d2db24f90f3 100644 --- a/apps/sim/tools/firecrawl/crawl.ts +++ b/apps/sim/tools/firecrawl/crawl.ts @@ -18,6 +18,18 @@ const logger = createLogger('FirecrawlCrawlTool') const POLL_INTERVAL_MS = 5000 const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS +/** Path segments in a URL, ignoring index files, as Firecrawl v1 measured crawl depth. */ +function startUrlPathDepth(url: string): number { + try { + return new URL(url).pathname + .split('/') + .filter((segment) => segment !== '' && segment !== 'index.html' && segment !== 'index.php') + .length + } catch { + return 0 + } +} + export const crawlTool: ToolConfig = { id: 'firecrawl_crawl', name: 'Firecrawl Crawl', @@ -155,13 +167,14 @@ export const crawlTool: ToolConfig if (params.maxConcurrency) body.maxConcurrency = Number(params.maxConcurrency) if (params.excludePaths) body.excludePaths = params.excludePaths if (params.includePaths) body.includePaths = params.includePaths - if (params.maxDepth != null) { - const maxDepth = Number(params.maxDepth) - if (!Number.isSafeInteger(maxDepth) || maxDepth < 0) { + if (params.maxDepth != null && String(params.maxDepth).trim() !== '') { + const requestedDepth = Number(params.maxDepth) + if (!Number.isSafeInteger(requestedDepth) || requestedDepth < 0) { throw new Error('maxDepth must be a non-negative integer') } + // v1 counted path segments below the start URL's own depth; v2 discovery depth counts link hops. + const maxDepth = requestedDepth + startUrlPathDepth(params.url) - // v1 depth counted URL path segments; v2 discovery depth counts link hops. body.regexOnFullURL = false if (params.includePaths?.length) { const indexFiles = ['index.html', 'index.php'] From f5b5d0a7c0d767d7ad94d1696d3223e7abb0f1cb Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 10 Nov 2026 15:45:54 -0800 Subject: [PATCH 2/4] fix(release): leave Firecrawl crawl depth to #9864 --- apps/sim/tools/firecrawl/crawl.test.ts | 58 ++++++++++++-------------- apps/sim/tools/firecrawl/crawl.ts | 30 ++++++------- 2 files changed, 42 insertions(+), 46 deletions(-) diff --git a/apps/sim/tools/firecrawl/crawl.test.ts b/apps/sim/tools/firecrawl/crawl.test.ts index 61f428b1e3f..5c4119c60cc 100644 --- a/apps/sim/tools/firecrawl/crawl.test.ts +++ b/apps/sim/tools/firecrawl/crawl.test.ts @@ -1,11 +1,10 @@ import { isPlainRecord } from '@sim/utils/object' import { describe, expect, it } from 'vitest' import { crawlTool } from '@/tools/firecrawl/crawl' -import type { FirecrawlCrawlParams } from '@/tools/firecrawl/types' function crawlBody(options: { - url?: string maxDepth: number + url?: string maxDiscoveryDepth?: number includePaths?: string[] excludePaths?: string[] @@ -68,40 +67,35 @@ describe('Firecrawl legacy crawl depth', () => { expect(patterns.some((pattern) => pattern.test(path))).toBe(excluded) }) - it.each([-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])( - 'rejects an invalid legacy depth: %s', - (maxDepth) => { - expect(() => crawlBody({ maxDepth })).toThrow(/maxDepth/) - } - ) - - it('measures depth below the start URL, as Firecrawl v1 did', () => { - const body = crawlBody({ url: 'https://example.com/guide/intro', maxDepth: 1 }) + it.each<[string, string, boolean]>([ + ['https://example.com/docs/guide', '/docs/guide', true], + ['https://example.com/docs/guide/', '/docs/guide/setup', true], + ['https://example.com/docs/guide/index.html', '/docs/guide/setup', true], + ['https://example.com/docs/guide', '/docs/guide/setup/advanced', false], + ['docs.example.com/guide', '/guide/setup', true], + ])('counts depth from the start URL %s for %s', (url, path, accepted) => { + const body = crawlBody({ maxDepth: 1, url }) const patterns = (body.includePaths as string[]).map((pattern) => new RegExp(pattern)) - const accepts = (path: string) => patterns.some((pattern) => pattern.test(path)) - - expect(accepts('/guide/intro')).toBe(true) - expect(accepts('/guide/intro/setup')).toBe(true) - expect(accepts('/guide/intro/setup/advanced')).toBe(false) + expect(patterns.some((pattern) => pattern.test(path))).toBe(accepted) }) - it('ignores an index file in the start URL when measuring its depth', () => { - const body = crawlBody({ url: 'https://example.com/docs/index.html', maxDepth: 0 }) - const patterns = (body.includePaths as string[]).map((pattern) => new RegExp(pattern)) - expect(patterns.some((pattern) => pattern.test('/docs'))).toBe(true) - expect(patterns.some((pattern) => pattern.test('/docs/guide'))).toBe(false) + it('counts depth from the start URL alongside existing path filters', () => { + const body = crawlBody({ + maxDepth: 1, + url: 'https://example.com/docs/guide', + includePaths: ['^/docs'], + }) + const patterns = (body.excludePaths as string[]).map((pattern) => new RegExp(pattern)) + const excludes = (path: string) => patterns.some((pattern) => pattern.test(path)) + expect(excludes('/docs/guide')).toBe(false) + expect(excludes('/docs/guide/setup')).toBe(false) + expect(excludes('/docs/guide/setup/advanced')).toBe(true) }) - it('treats a blank depth as unset instead of a zero-depth crawl', () => { - if (typeof crawlTool.request.body !== 'function') throw new Error('Missing request formatter') - const blankDepth: Record = { - apiKey: 'test', - url: 'https://example.com', - maxDepth: '', + it.each([-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])( + 'rejects an invalid legacy depth: %s', + (maxDepth) => { + expect(() => crawlBody({ maxDepth })).toThrow(/maxDepth/) } - const body = crawlTool.request.body(blankDepth as FirecrawlCrawlParams) - if (!isPlainRecord(body)) throw new Error('Expected a JSON request body') - expect(body.includePaths).toBeUndefined() - expect(body.regexOnFullURL).toBeUndefined() - }) + ) }) diff --git a/apps/sim/tools/firecrawl/crawl.ts b/apps/sim/tools/firecrawl/crawl.ts index d2db24f90f3..3859fdb4975 100644 --- a/apps/sim/tools/firecrawl/crawl.ts +++ b/apps/sim/tools/firecrawl/crawl.ts @@ -18,16 +18,17 @@ const logger = createLogger('FirecrawlCrawlTool') const POLL_INTERVAL_MS = 5000 const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS -/** Path segments in a URL, ignoring index files, as Firecrawl v1 measured crawl depth. */ -function startUrlPathDepth(url: string): number { - try { - return new URL(url).pathname - .split('/') - .filter((segment) => segment !== '' && segment !== 'index.html' && segment !== 'index.php') - .length - } catch { - return 0 - } +/** + * Path segments of a URL as Firecrawl v1 counted them: index files add no depth. A URL without + * a scheme is read as https, as Firecrawl reads it. + */ +function urlPathDepth(url: string): number { + const absolute = /^[a-z][a-z\d+.-]*:\/\//i.test(url) ? url : `https://${url}` + if (!URL.canParse(absolute)) return 0 + return new URL(absolute).pathname + .split('/') + .filter((segment) => segment !== '' && segment !== 'index.html' && segment !== 'index.php') + .length } export const crawlTool: ToolConfig = { @@ -54,7 +55,7 @@ export const crawlTool: ToolConfig required: false, visibility: 'user-or-llm', description: - 'Maximum URL path depth (e.g., 1 allows /docs and /docs/index.html). Independent of link discovery depth', + 'Maximum URL path depth below the starting URL (e.g., from /docs, 1 allows /docs/guide). Independent of link discovery depth', }, formats: { type: 'json', @@ -167,14 +168,15 @@ export const crawlTool: ToolConfig if (params.maxConcurrency) body.maxConcurrency = Number(params.maxConcurrency) if (params.excludePaths) body.excludePaths = params.excludePaths if (params.includePaths) body.includePaths = params.includePaths - if (params.maxDepth != null && String(params.maxDepth).trim() !== '') { + if (params.maxDepth != null) { const requestedDepth = Number(params.maxDepth) if (!Number.isSafeInteger(requestedDepth) || requestedDepth < 0) { throw new Error('maxDepth must be a non-negative integer') } - // v1 counted path segments below the start URL's own depth; v2 discovery depth counts link hops. - const maxDepth = requestedDepth + startUrlPathDepth(params.url) + // v1 counted depth from the start URL, so the start page itself always passes the filter. + const maxDepth = requestedDepth + urlPathDepth(params.url) + // v1 depth counted URL path segments; v2 discovery depth counts link hops. body.regexOnFullURL = false if (params.includePaths?.length) { const indexFiles = ['index.html', 'index.php'] From 239caf55bef91bfd349a850f6b6983825340ffab Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 10 Nov 2026 15:48:24 -0800 Subject: [PATCH 3/4] fix(firecrawl): treat a blank crawl maxDepth as unset --- apps/sim/tools/firecrawl/crawl.test.ts | 14 ++++++++++++++ apps/sim/tools/firecrawl/crawl.ts | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/apps/sim/tools/firecrawl/crawl.test.ts b/apps/sim/tools/firecrawl/crawl.test.ts index 5c4119c60cc..2310840adf1 100644 --- a/apps/sim/tools/firecrawl/crawl.test.ts +++ b/apps/sim/tools/firecrawl/crawl.test.ts @@ -1,6 +1,7 @@ import { isPlainRecord } from '@sim/utils/object' import { describe, expect, it } from 'vitest' import { crawlTool } from '@/tools/firecrawl/crawl' +import type { FirecrawlCrawlParams } from '@/tools/firecrawl/types' function crawlBody(options: { maxDepth: number @@ -98,4 +99,17 @@ describe('Firecrawl legacy crawl depth', () => { expect(() => crawlBody({ maxDepth })).toThrow(/maxDepth/) } ) + + it('treats a blank depth as unset instead of a zero-depth crawl', () => { + if (typeof crawlTool.request.body !== 'function') throw new Error('Missing request formatter') + const blankDepth: Record = { + apiKey: 'test', + url: 'https://example.com/docs', + maxDepth: '', + } + const body = crawlTool.request.body(blankDepth as FirecrawlCrawlParams) + if (!isPlainRecord(body)) throw new Error('Expected a JSON request body') + expect(body.includePaths).toBeUndefined() + expect(body.regexOnFullURL).toBeUndefined() + }) }) diff --git a/apps/sim/tools/firecrawl/crawl.ts b/apps/sim/tools/firecrawl/crawl.ts index 3859fdb4975..110b4529b76 100644 --- a/apps/sim/tools/firecrawl/crawl.ts +++ b/apps/sim/tools/firecrawl/crawl.ts @@ -168,7 +168,7 @@ export const crawlTool: ToolConfig if (params.maxConcurrency) body.maxConcurrency = Number(params.maxConcurrency) if (params.excludePaths) body.excludePaths = params.excludePaths if (params.includePaths) body.includePaths = params.includePaths - if (params.maxDepth != null) { + if (params.maxDepth != null && String(params.maxDepth).trim() !== '') { const requestedDepth = Number(params.maxDepth) if (!Number.isSafeInteger(requestedDepth) || requestedDepth < 0) { throw new Error('maxDepth must be a non-negative integer') From 18585865422296bbb80cb3509049ed891c20dc97 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 10 Nov 2026 16:05:22 -0800 Subject: [PATCH 4/4] test(release): prove delegation refuses read-only keys and type a blank crawl depth --- .../lib/mothership/chat/delegation.test.ts | 35 +++++++++++++++---- apps/sim/tools/firecrawl/crawl.test.ts | 4 +-- apps/sim/tools/firecrawl/types.ts | 3 +- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/apps/sim/lib/mothership/chat/delegation.test.ts b/apps/sim/lib/mothership/chat/delegation.test.ts index 5096228b90f..b28278d50be 100644 --- a/apps/sim/lib/mothership/chat/delegation.test.ts +++ b/apps/sim/lib/mothership/chat/delegation.test.ts @@ -29,6 +29,7 @@ interface KeyRow { name: string key: string expiresAt: Date + access?: 'read' | 'write' } /** Models a transaction-scoped advisory lock and commits only after the entire callback succeeds. */ @@ -64,12 +65,18 @@ function database() { if (!locked) throw new Error('Read before acquiring the user lock') return { from: () => ({ - where: () => ({ - limit: async () => { - const row = rows.get(name) - return row && row.expiresAt.getTime() > Date.now() + 30 * 60_000 ? [row] : [] - }, - }), + where: (condition: SQL) => { + const query = dialect.sqlToQuery(condition) + // Only the access filter binds 'write'; the other predicates bind ids, names and dates. + const writeOnly = query.params.includes('write') + return { + limit: async () => { + const row = rows.get(name) + if (!row || row.expiresAt.getTime() <= Date.now() + 30 * 60_000) return [] + return writeOnly && row.access === 'read' ? [] : [row] + }, + } + }, }), } }, @@ -154,6 +161,22 @@ describe('delegation key concurrency', () => { expect(db.rows.get('mothership-delegation:alice')).toBe(old) }) + it('replaces a read-only key instead of handing it to the CLI', async () => { + const db = database() + db.rows.set('mothership-delegation:actor', { + id: 'read-key', + name: 'mothership-delegation:actor', + key: 'encrypted-read-only', + expiresAt: new Date(Date.now() + 6 * 60 * 60_000), + access: 'read', + }) + expect(await mintDelegationToken({ workspaceId: 'shared', userId: 'actor' })).toBe('plain-1') + expect(db.rows.get('mothership-delegation:actor')).toMatchObject({ + key: 'encrypted-1', + access: 'write', + }) + }) + it('does not log bound credential values from database errors', async () => { mocks.transaction.mockRejectedValue( new Error('Failed query params: plaintext-key, encrypted-key') diff --git a/apps/sim/tools/firecrawl/crawl.test.ts b/apps/sim/tools/firecrawl/crawl.test.ts index 2310840adf1..2b342396d14 100644 --- a/apps/sim/tools/firecrawl/crawl.test.ts +++ b/apps/sim/tools/firecrawl/crawl.test.ts @@ -102,12 +102,12 @@ describe('Firecrawl legacy crawl depth', () => { it('treats a blank depth as unset instead of a zero-depth crawl', () => { if (typeof crawlTool.request.body !== 'function') throw new Error('Missing request formatter') - const blankDepth: Record = { + const blankDepth: FirecrawlCrawlParams = { apiKey: 'test', url: 'https://example.com/docs', maxDepth: '', } - const body = crawlTool.request.body(blankDepth as FirecrawlCrawlParams) + const body = crawlTool.request.body(blankDepth) if (!isPlainRecord(body)) throw new Error('Expected a JSON request body') expect(body.includePaths).toBeUndefined() expect(body.regexOnFullURL).toBeUndefined() diff --git a/apps/sim/tools/firecrawl/types.ts b/apps/sim/tools/firecrawl/types.ts index 60dd60c3ef7..571909aadf9 100644 --- a/apps/sim/tools/firecrawl/types.ts +++ b/apps/sim/tools/firecrawl/types.ts @@ -285,7 +285,8 @@ export interface FirecrawlCrawlParams { apiKey: string url: string limit?: number - maxDepth?: number + /** Saved tool params can carry the raw field text, so a cleared field arrives as `''`. */ + maxDepth?: number | string formats?: FirecrawlFormat[] onlyMainContent?: boolean prompt?: string