fix(release): cap proxied plain-http responses, count Firecrawl depth from the start URL, and drop stale route cache on upgrade

.github/CONTRIBUTING.md

@@ -140,23 +140,19 @@ Using clear and consistent commit messages makes it easier for everyone to under
 
 To set up your local development environment:
 
-### Option 1: Using NPM Package (Simplest)
+### Option 1: Using the Setup Wizard (Simplest)
 
-The easiest way to run Sim locally is using our NPM package:
+The easiest way to run Sim locally is the interactive setup wizard:
 
 ```bash
-npx simstudio
+npx sim-setup
 ```
 
-After running this command, open [http://localhost:3000/](http://localhost:3000/) in your browser.
-
-#### Options
-
-- `-p, --port <port>`: Specify the port to run Sim on (default: 3000)
-- `--no-pull`: Skip pulling the latest Docker images
+When it finishes, open [http://localhost:3000/](http://localhost:3000/) in your browser. Inside a cloned Sim repository, run `bun run sim-setup --mode dev` for the local development mode.
 
 #### Requirements
 
+- Node.js 20+
 - Docker must be installed and running on your machine
 
 ### Option 2: Using Docker Compose

apps/docs/content/docs/integrations/firecrawl.mdx

@@ -184,7 +184,7 @@ Crawl entire websites and extract structured content from all accessible pages
 | --------- | ---- | -------- | ----------- |
 | `url` | string | Yes | The website URL to crawl \(e.g., "https://example.com" or "https://docs.example.com/guide") |
 | `limit` | number | No | Maximum number of pages to crawl \(e.g., 50, 100, 500\). Default: 100 |
-| `maxDepth` | number | No | Maximum URL path depth \(e.g., 1 allows /docs and /docs/index.html\). Independent of link discovery depth |
+| `maxDepth` | number | No | Maximum URL path depth below the starting URL \(e.g., from /docs, 1 allows /docs/guide\). Independent of link discovery depth |
 | `formats` | json | No | Output formats for scraped content \(e.g., \["markdown"\], \["markdown", "html"\], \["markdown", "links"\]\) |
 | `prompt` | string | No | Natural-language crawl guidance supplied by existing configurations |
 | `excludePaths` | json | No | URL paths to exclude from crawling \(e.g., \["/blog/*", "/admin/*", "/*.pdf"\]\) |

apps/sim/bootstrap.ts

@@ -1,11 +1,15 @@
 /**
- * Container entrypoint. Hydrates `process.env` from the runtime secret before
- * loading the Next.js standalone server, so application modules that read env at
- * import time see the full configuration. See `@sim/runtime-secrets`.
+ * Container entrypoint. Hydrates `process.env` from the runtime secret and drops Next's route
+ * cache when an earlier build wrote it, before loading the Next.js standalone server, so
+ * application modules that read env at import time see the full configuration. See
+ * `@sim/runtime-secrets`.
  */
+import { fileURLToPath } from 'node:url'
 import { loadRuntimeSecrets } from '@sim/runtime-secrets'
+import { clearRouteCacheFromOtherBuild } from '@/lib/core/utils/route-cache'
 
 await loadRuntimeSecrets()
+await clearRouteCacheFromOtherBuild(fileURLToPath(new URL('./.next', import.meta.url)))
 // `server.js` is the Next standalone build artifact, a sibling of this file in
 // the image; it does not exist at type-check time, so the specifier is held in a
 // variable to keep it out of static module resolution.

apps/sim/lib/core/network/environment-proxy.server.test.ts

@@ -206,6 +206,25 @@ describe('guarded requests under HTTPS_PROXY', () => {
     ])
   })
 
+  it('caps a forwarded plain-http response at maxResponseSize', async () => {
+    const guarded = createSsrfGuardedFetchWithDispatcher({
+      ...options,
+      profile: 'selfHostedService',
+      maxResponseSize: 4,
+    })
+    try {
+      const response = await guarded.fetch('http://origin.invalid/plain')
+      await expect(response.text()).rejects.toMatchObject({
+        code: 'UND_ERR_RES_EXCEEDED_MAX_SIZE',
+      })
+    } finally {
+      await guarded.dispatcher.destroy()
+    }
+    expect(forwarded).toEqual([
+      { url: 'http://origin.invalid/plain', authorization: PROXY_AUTHORIZATION },
+    ])
+  })
+
   it('connects directly, still pinned, when NO_PROXY names the host', async () => {
     vi.stubEnv('NO_PROXY', 'localhost,.invalid')
     const response = await secureFetchWithPinnedIP(

apps/sim/lib/core/security/input-validation.server.ts

@@ -893,8 +893,7 @@ async function undiciRequestAsResponse(
     return response
   }
 
-  // Decode Content-Encoding like `fetch` does (`request()` returns raw bytes). `maxResponseSize`
-  // still caps the compressed wire bytes on `body`.
+  // Decode Content-Encoding like `fetch` does (`request()` returns raw bytes).
   const contentEncoding = String(headers['content-encoding'] ?? '')
     .toLowerCase()
     .trim()
@@ -909,13 +908,16 @@ async function undiciRequestAsResponse(
   // AFTER it means a synchronous zlib error (e.g. a server mislabeling a non-gzip body as gzip)
   // is caught and rejects the reader instead of taking down the process.
   let webBody = nodeReadableToWebStream(decoder ?? body)
-  if (decoder && maxResponseSize !== undefined && maxResponseSize >= 0) {
-    let decodedBytes = 0
+  // Counted here rather than left to the dispatcher: undici's ProxyAgent forwards plain-http
+  // requests through a client that drops `maxResponseSize`, and a decoder's output is larger
+  // than the wire bytes the dispatcher counts.
+  if (maxResponseSize !== undefined && maxResponseSize >= 0) {
+    let bodyBytes = 0
     webBody = webBody.pipeThrough(
       new TransformStream<Uint8Array, Uint8Array>({
         transform(chunk, controller) {
-          decodedBytes += chunk.byteLength
-          if (decodedBytes > maxResponseSize) throw new errors.ResponseExceededMaxSizeError()
+          bodyBytes += chunk.byteLength
+          if (bodyBytes > maxResponseSize) throw new errors.ResponseExceededMaxSizeError()
           controller.enqueue(chunk)
         },
       })

apps/sim/lib/core/utils/route-cache.test.ts

@@ -0,0 +1,46 @@
+import { mkdir, mkdtemp, readdir, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import { clearRouteCacheFromOtherBuild } from '@/lib/core/utils/route-cache'
+
+let nextDirectory: string
+let routeCache: string
+
+beforeEach(async () => {
+  nextDirectory = await mkdtemp(join(tmpdir(), 'route-cache-'))
+  routeCache = join(nextDirectory, 'server', 'route-cache')
+  await mkdir(join(routeCache, 'app-page'), { recursive: true })
+  await writeFile(join(routeCache, 'app-page', 'contact.html'), '<html>old build</html>')
+  await writeFile(join(nextDirectory, 'BUILD_ID'), 'build-2\n')
+})
+
+afterEach(async () => {
+  await rm(nextDirectory, { recursive: true, force: true })
+})
+
+describe('clearRouteCacheFromOtherBuild', () => {
+  it('clears pages rendered by an earlier build', async () => {
+    await writeFile(join(routeCache, '.build-id'), 'build-1')
+    await clearRouteCacheFromOtherBuild(nextDirectory)
+    expect(await readdir(routeCache)).toEqual(['.build-id'])
+  })
+
+  it('clears a cache that records no build', async () => {
+    await clearRouteCacheFromOtherBuild(nextDirectory)
+    expect(await readdir(routeCache)).toEqual(['.build-id'])
+  })
+
+  it('keeps pages rendered by the current build across restarts', async () => {
+    await clearRouteCacheFromOtherBuild(nextDirectory)
+    await writeFile(join(routeCache, 'app-page.html'), '<html>current build</html>')
+    await clearRouteCacheFromOtherBuild(nextDirectory)
+    expect((await readdir(routeCache)).sort()).toEqual(['.build-id', 'app-page.html'])
+  })
+
+  it('leaves the cache alone when the build id is unreadable', async () => {
+    await rm(join(nextDirectory, 'BUILD_ID'))
+    await clearRouteCacheFromOtherBuild(nextDirectory)
+    expect(await readdir(routeCache)).toEqual(['app-page'])
+  })
+})

apps/sim/lib/core/utils/route-cache.ts

@@ -0,0 +1,25 @@
+import { readdir, readFile, rm, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+
+const BUILD_MARKER = '.build-id'
+
+/**
+ * Clears Next's route cache when it was written by a different build. Its keys carry no build
+ * id, and the image declares the directory a volume, so a container recreated from a newer
+ * image would otherwise serve pages the old build rendered, pointing at chunks it no longer
+ * ships. Never throws: a cache that cannot be read or cleared is left to Next.
+ */
+export async function clearRouteCacheFromOtherBuild(nextDirectory: string): Promise<void> {
+  const routeCache = join(nextDirectory, 'server', 'route-cache')
+  try {
+    const buildId = (await readFile(join(nextDirectory, 'BUILD_ID'), 'utf8')).trim()
+    if (!buildId) return
+    const cachedBuildId = await readFile(join(routeCache, BUILD_MARKER), 'utf8').catch(() => null)
+    if (cachedBuildId?.trim() === buildId) return
+    const entries = await readdir(routeCache)
+    await Promise.all(
+      entries.map((entry) => rm(join(routeCache, entry), { recursive: true, force: true }))
+    )
+    await writeFile(join(routeCache, BUILD_MARKER), buildId)
+  } catch {}
+}

apps/sim/tools/firecrawl/crawl.test.ts

@@ -4,6 +4,7 @@ import { crawlTool } from '@/tools/firecrawl/crawl'
 
 function crawlBody(options: {
   maxDepth: number
+  url?: string
   maxDiscoveryDepth?: number
   includePaths?: string[]
   excludePaths?: string[]
@@ -66,6 +67,31 @@ describe('Firecrawl legacy crawl depth', () => {
     expect(patterns.some((pattern) => pattern.test(path))).toBe(excluded)
   })
 
+  it.each<[string, string, boolean]>([
+    ['https://example.com/docs/guide', '/docs/guide', true],
+    ['https://example.com/docs/guide/', '/docs/guide/setup', true],
+    ['https://example.com/docs/guide/index.html', '/docs/guide/setup', true],
+    ['https://example.com/docs/guide', '/docs/guide/setup/advanced', false],
+    ['docs.example.com/guide', '/guide/setup', true],
+  ])('counts depth from the start URL %s for %s', (url, path, accepted) => {
+    const body = crawlBody({ maxDepth: 1, url })
+    const patterns = (body.includePaths as string[]).map((pattern) => new RegExp(pattern))
+    expect(patterns.some((pattern) => pattern.test(path))).toBe(accepted)
+  })
+
+  it('counts depth from the start URL alongside existing path filters', () => {
+    const body = crawlBody({
+      maxDepth: 1,
+      url: 'https://example.com/docs/guide',
+      includePaths: ['^/docs'],
+    })
+    const patterns = (body.excludePaths as string[]).map((pattern) => new RegExp(pattern))
+    const excludes = (path: string) => patterns.some((pattern) => pattern.test(path))
+    expect(excludes('/docs/guide')).toBe(false)
+    expect(excludes('/docs/guide/setup')).toBe(false)
+    expect(excludes('/docs/guide/setup/advanced')).toBe(true)
+  })
+
   it.each([-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])(
     'rejects an invalid legacy depth: %s',
     (maxDepth) => {

apps/sim/tools/firecrawl/crawl.ts

@@ -18,6 +18,19 @@ const logger = createLogger('FirecrawlCrawlTool')
 const POLL_INTERVAL_MS = 5000
 const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS
 
+/**
+ * Path segments of a URL as Firecrawl v1 counted them: index files add no depth. A URL without
+ * a scheme is read as https, as Firecrawl reads it.
+ */
+function urlPathDepth(url: string): number {
+  const absolute = /^[a-z][a-z\d+.-]*:\/\//i.test(url) ? url : `https://${url}`
+  if (!URL.canParse(absolute)) return 0
+  return new URL(absolute).pathname
+    .split('/')
+    .filter((segment) => segment !== '' && segment !== 'index.html' && segment !== 'index.php')
+    .length
+}
+
 export const crawlTool: ToolConfig<FirecrawlCrawlParams, FirecrawlCrawlResponse> = {
   id: 'firecrawl_crawl',
   name: 'Firecrawl Crawl',
@@ -42,7 +55,7 @@ export const crawlTool: ToolConfig<FirecrawlCrawlParams, FirecrawlCrawlResponse>
       required: false,
       visibility: 'user-or-llm',
       description:
-        'Maximum URL path depth (e.g., 1 allows /docs and /docs/index.html). Independent of link discovery depth',
+        'Maximum URL path depth below the starting URL (e.g., from /docs, 1 allows /docs/guide). Independent of link discovery depth',
     },
     formats: {
       type: 'json',
@@ -156,10 +169,12 @@ export const crawlTool: ToolConfig<FirecrawlCrawlParams, FirecrawlCrawlResponse>
       if (params.excludePaths) body.excludePaths = params.excludePaths
       if (params.includePaths) body.includePaths = params.includePaths
       if (params.maxDepth != null) {
-        const maxDepth = Number(params.maxDepth)
-        if (!Number.isSafeInteger(maxDepth) || maxDepth < 0) {
+        const requestedDepth = Number(params.maxDepth)
+        if (!Number.isSafeInteger(requestedDepth) || requestedDepth < 0) {
           throw new Error('maxDepth must be a non-negative integer')
         }
+        // v1 counted depth from the start URL, so the start page itself always passes the filter.
+        const maxDepth = requestedDepth + urlPathDepth(params.url)
 
         // v1 depth counted URL path segments; v2 discovery depth counts link hops.
         body.regexOnFullURL = false