Opt-in deep crawling via CRAWL4AI_ALLOW_DEEP_CRAWL parameter

crawl4ai/async_configs.py

@@ -199,6 +199,14 @@ class UntrustedConfigError(ValueError):
     "CacheMode", "MatchMode", "DisplayMode",
 }
 
+# Allowed only with CRAWL4AI_ALLOW_DEEP_CRAWL=true; SEOFilter/ContentRelevanceFilter excluded (unguarded httpx fetch).
+UNTRUSTED_DEEP_CRAWL_TYPES = {
+    "BFSDeepCrawlStrategy", "DFSDeepCrawlStrategy", "BestFirstCrawlingStrategy",
+    "FilterChain", "URLPatternFilter", "DomainFilter", "ContentTypeFilter",
+    "KeywordRelevanceScorer", "CompositeScorer", "DomainAuthorityScorer",
+    "FreshnessScorer", "PathDepthScorer",
+}
+
 # Fields that must NEVER come from an untrusted body. Presence => 400 (loud), so
 # a client smuggling these gets an explicit error rather than a silent drop.
 UNTRUSTED_FORBIDDEN_FIELDS = {
@@ -223,6 +231,11 @@ class UntrustedConfigError(ValueError):
     "PDFContentScrapingStrategy": {
         "image_save_dir", "save_images_locally",
     },
+    # resume_state.pending would queue URLs that skip seed validation.
+    **{
+        name: {"resume_state", "logger", "on_state_change", "should_cancel"}
+        for name in ("BFSDeepCrawlStrategy", "DFSDeepCrawlStrategy", "BestFirstCrawlingStrategy")
+    },
 }
 
 # Field names that must NEVER be set from an untrusted body on ANY allowed
@@ -302,6 +315,9 @@ class UntrustedConfigError(ValueError):
 _MAX_LINK_PREVIEW_LINKS = 100      # the class default
 _MAX_LINK_PREVIEW_CONCURRENCY = 10  # the class default
 _MAX_LINK_PREVIEW_TIMEOUT_S = 10    # seconds here, not ms
+_MAX_DEEP_CRAWL_LIST = 100          # patterns, keywords, filters, scorers per object
+_MAX_DEEP_CRAWL_OBJECTS = 20        # filters and scorers in one strategy
+_MAX_DEEP_CRAWL_ITEMS = 200         # list and dict items (patterns, keywords, ...) in one strategy
 
 
 def _max_timeout_ms() -> int:
@@ -339,10 +355,28 @@ def _max_timeout_ms() -> int:
     return ceiling
 
 
+def _deep_crawl_allowed() -> bool:
+    """Operator opt-in for deep crawling from untrusted requests."""
+    return os.getenv("CRAWL4AI_ALLOW_DEEP_CRAWL", "false").lower() == "true"
+
+
+def _all_nodes(node):
+    """Yield every value in a request tree. An object's params dict is entered but not yielded."""
+    yield node
+    if _is_typed_shape(node) and isinstance(node.get("params"), dict):
+        node = node["params"]
+    children = node.values() if isinstance(node, dict) else node if isinstance(node, list) else []
+    for child in children:
+        yield from _all_nodes(child)
+
+
 def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
     """Drop non-allowlisted fields and raise on forbidden (power) fields."""
     forbidden = UNTRUSTED_FORBIDDEN_FIELDS.get(type_name, set())
     allowlist = UNTRUSTED_FIELD_ALLOWLIST.get(type_name)  # None => keep all non-forbidden
+    if type_name == "CrawlerRunConfig" and _deep_crawl_allowed():
+        forbidden = forbidden - {"deep_crawl_strategy"}
+        allowlist = allowlist | {"deep_crawl_strategy"}
     out = {}
     for key, value in params.items():
         if key in UNTRUSTED_GLOBAL_FORBIDDEN_FIELDS or key in forbidden:
@@ -352,6 +386,47 @@ def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
         if allowlist is not None and key not in allowlist:
             continue  # silently drop unknown/unsafe fields (forward-compatible)
         out[key] = value
+    if type_name in UNTRUSTED_DEEP_CRAWL_TYPES and any(
+        isinstance(v, list) and len(v) > _MAX_DEEP_CRAWL_LIST for v in out.values()
+    ):
+        raise UntrustedConfigError(
+            f"{type_name} list parameters are limited to {_MAX_DEEP_CRAWL_LIST} items"
+        )
+    if type_name in ("BFSDeepCrawlStrategy", "DFSDeepCrawlStrategy", "BestFirstCrawlingStrategy"):
+        # Filters and scorers run synchronously on the event loop for every discovered link.
+        nodes = [n for value in out.values() for n in _all_nodes(value)]
+        filters_and_scorers = sum(1 for n in nodes if isinstance(n, dict) and n.get("type") in UNTRUSTED_DEEP_CRAWL_TYPES)
+        list_items = sum(len(n) for n in nodes if isinstance(n, (list, dict)) and not _is_typed_shape(n))
+        if filters_and_scorers > _MAX_DEEP_CRAWL_OBJECTS or list_items > _MAX_DEEP_CRAWL_ITEMS:
+            raise UntrustedConfigError(
+                f"{type_name} is limited to {_MAX_DEEP_CRAWL_OBJECTS} filters and scorers "
+                f"and {_MAX_DEEP_CRAWL_ITEMS} list items in total"
+            )
+    if type_name == "URLPatternFilter":
+        # Regex and *.domain:// patterns compile unescaped, and globs not led by * search in
+        # quadratic time; allow only *-led globs, /prefix/* and *.ext.
+        patterns = out.get("patterns")
+        patterns = [patterns] if isinstance(patterns, str) else patterns
+        if not isinstance(patterns, list) or not all(
+            isinstance(p, str)
+            and not (p.startswith("^") or p.endswith("$") or "\\d" in p or "**" in p)
+            and not ("://" in p and p.startswith("*."))
+            and (p.startswith("*") or (p.count("*") == 1 and p.endswith("/*")))
+            for p in patterns
+        ):
+            raise UntrustedConfigError(
+                "URLPatternFilter accepts only globs that start with '*' or end with a "
+                "single '/*' from an untrusted request"
+            )
+    if type_name == "CompositeScorer" and any(
+        isinstance(s, dict) and s.get("type") == "CompositeScorer" for s in out.get("scorers") or []
+    ):
+        raise UntrustedConfigError("CompositeScorer cannot be nested from an untrusted request")
+    if type_name == "CrawlerRunConfig" and out.get("deep_crawl_strategy") and out.get("fetch_ssl_certificate"):
+        # The certificate fetch dials each discovered link directly, outside the egress proxy.
+        raise UntrustedConfigError(
+            "fetch_ssl_certificate cannot be combined with deep_crawl_strategy from an untrusted request"
+        )
     return out
 
 
@@ -556,7 +631,11 @@ def from_serializable_dict(data: Any, provenance: "Provenance" = None) -> Any:
 
         # Untrusted bodies may only construct the strict subset of types and
         # may not set forbidden power-fields.
-        if provenance == Provenance.UNTRUSTED and type_name not in UNTRUSTED_ALLOWED_TYPES:
+        if (
+            provenance == Provenance.UNTRUSTED
+            and type_name not in UNTRUSTED_ALLOWED_TYPES
+            and not (type_name in UNTRUSTED_DEEP_CRAWL_TYPES and _deep_crawl_allowed())
+        ):
             raise UntrustedConfigError(
                 f"type '{type_name}' may not be constructed from an untrusted request"
             )

crawl4ai/deep_crawling/filters.py

@@ -209,7 +209,8 @@ def _add_pattern(self, pattern: str, pattern_type: int):
                         lambda m: f'({"|".join(m.group(1).split(","))})',
                         pattern,
                     )
-                pattern = fnmatch.translate(pattern)
+                # A leading * already matches any prefix; \A stops search() retrying every offset.
+                pattern = ("\\A" if pattern.startswith("*") else "") + fnmatch.translate(pattern)
             self._path_patterns.append(
                 pattern if isinstance(pattern, Pattern) else re.compile(pattern)
             )

deploy/docker/MIGRATION.md

@@ -68,6 +68,10 @@ control):
 `base_url`, `deep_crawl_strategy`, `simulate_user`, `magic`,
 `process_in_browser`, and nested LLM config objects.
 
+`deep_crawl_strategy` can be turned back on by the operator with
+`CRAWL4AI_ALLOW_DEEP_CRAWL=true`, with server-side limits. See "Deep Crawling
+(opt-in)" in the self-hosting guide.
+
 Unknown fields are dropped; timeouts, viewport and scroll counts are clamped to
 safe maximums.
 

deploy/docker/README.md

@@ -562,7 +562,62 @@ Crucially, when sending configurations directly via JSON, they **must** follow t
 ```
 
 **LLM Extraction Strategy** *(Keep example, ensure schema uses type/value wrapper)*
-*(Keep Deep Crawler Example)*
+#### Deep Crawling (opt-in)
+
+Deep crawling over the API is **disabled by default**: a request with
+`deep_crawl_strategy` returns HTTP 400. To allow it, start the container with
+`CRAWL4AI_ALLOW_DEEP_CRAWL=true`:
+
+```bash
+docker run -d \
+  -p 11235:11235 \
+  --name crawl4ai \
+  --shm-size=1g \
+  -e CRAWL4AI_API_TOKEN="$CRAWL4AI_API_TOKEN" \
+  -e CRAWL4AI_ALLOW_DEEP_CRAWL=true \
+  unclecode/crawl4ai:latest
+```
+
+With Docker Compose, put `CRAWL4AI_ALLOW_DEEP_CRAWL=true` in the `.env` file in
+the project root, or export it in your shell. The compose file sets this
+variable itself, so a value in `.llm.env` is ignored.
+
+```json
+{
+  "urls": ["https://docs.crawl4ai.com"],
+  "crawler_config": {
+    "type": "CrawlerRunConfig",
+    "params": {
+      "deep_crawl_strategy": {
+        "type": "BFSDeepCrawlStrategy",
+        "params": {
+          "max_depth": 2,
+          "max_pages": 20,
+          "filter_chain": {
+            "type": "FilterChain",
+            "params": {
+              "filters": [
+                {"type": "URLPatternFilter", "params": {"patterns": ["*/core/*"]}},
+                {"type": "DomainFilter", "params": {"allowed_domains": ["docs.crawl4ai.com"]}}
+              ]
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+Rules the server applies:
+
+- **One start URL** per request, and not inside `crawler_configs`.
+- `max_pages` and `max_depth` are clamped to `limits.max_pages` and `limits.max_depth` in `config.yml` (default 100 and 5; `max_pages: 0` means no limit). The crawl stops after `limits.wall_clock_s`, also on `/crawl/stream`.
+- **Allowed types:** `BFSDeepCrawlStrategy`, `DFSDeepCrawlStrategy`, `BestFirstCrawlingStrategy`; `FilterChain` with `URLPatternFilter`, `DomainFilter`, `ContentTypeFilter`; scorers `KeywordRelevanceScorer`, `CompositeScorer`, `DomainAuthorityScorer`, `FreshnessScorer`, `PathDepthScorer`. `SEOFilter` and `ContentRelevanceFilter` are refused (they fetch pages outside the egress controls).
+- `URLPatternFilter` accepts globs that start with `*` (`*/blog/*`), `/prefix/*` or `*.ext`. Regex patterns are refused.
+- `resume_state` is refused, and so is `fetch_ssl_certificate` together with a deep crawl. Lists (patterns, keywords, filters, scorers) are limited to 100 items. One strategy may have at most 20 filters and scorers and 200 list items in total, and a `CompositeScorer` cannot contain another `CompositeScorer`. Discovered links longer than 2048 characters are skipped.
+
+Turn the flag on only when you accept that one request can fetch many pages.
 
 ### REST API Examples
 

deploy/docker/api.py

@@ -641,6 +641,8 @@ async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator)
 
         yield json.dumps({"status": "completed"}).encode('utf-8')
 
+    except asyncio.TimeoutError:
+        yield json.dumps({"status": "error", "error": "Crawl exceeded the time limit"}).encode('utf-8')
     except asyncio.CancelledError:
         logger.warning("Client disconnected during streaming")
     finally:
@@ -663,6 +665,8 @@ def _load_crawler_configs(
         for cc in crawler_configs
     ]
     for cfg in config_list:
+        if cfg.deep_crawl_strategy is not None:
+            raise HTTPException(status_code=400, detail="Deep crawling supports exactly one URL per request.")
         for key, value in (base_config or {}).items():
             if hasattr(cfg, key):
                 current_value = getattr(cfg, key)
@@ -699,6 +703,20 @@ def _needs_pdf_crawler(
     return all(pdf)
 
 
+async def _with_deadline(gen: AsyncGenerator, seconds: float) -> AsyncGenerator:
+    """Stop a result stream once its wall-clock budget is spent (raises TimeoutError)."""
+    loop = asyncio.get_running_loop()
+    end = loop.time() + seconds
+    try:
+        while True:
+            try:
+                yield await asyncio.wait_for(gen.__anext__(), max(end - loop.time(), 0))
+            except StopAsyncIteration:
+                return
+    finally:
+        await gen.aclose()
+
+
 def _normalize_and_validate_seeds(urls: List[str]) -> Tuple[List[str], List[CrawlResult]]:
     """Prefix bare hosts with https:// and SSRF-validate every seed URL's
     destination. Shared by the streaming and non-streaming crawl handlers so a
@@ -753,8 +771,10 @@ async def handle_crawl_request(
         crawler_config = CrawlerRunConfig.load(crawler_config, provenance=Provenance.UNTRUSTED)
         from egress_broker import enforce_egress
         enforce_egress(browser_config)
-        from governor import clamp_deep_crawl
-        clamp_deep_crawl(crawler_config)
+        from governor import clamp_deep_crawl, deep_crawl_limits
+        clamp_deep_crawl(crawler_config, **deep_crawl_limits(config))
+        if crawler_config.deep_crawl_strategy is not None and len(urls) + len(refused) != 1:
+            raise HTTPException(status_code=400, detail="Deep crawling supports exactly one URL per request.")
 
         dispatcher = MemoryAdaptiveDispatcher(
             memory_threshold_percent=config["crawler"]["memory_threshold_percent"],
@@ -984,9 +1004,9 @@ async def handle_stream_crawl_request(
         crawler_config = CrawlerRunConfig.load(
             crawler_config, provenance=Provenance.UNTRUSTED
         )
-        from governor import clamp_deep_crawl
+        from governor import clamp_deep_crawl, deep_crawl_limits
 
-        clamp_deep_crawl(crawler_config)
+        clamp_deep_crawl(crawler_config, **deep_crawl_limits(config))
         crawler_config.stream = True
         config_list = (
             _load_crawler_configs(
@@ -1034,6 +1054,9 @@ async def handle_stream_crawl_request(
                 urls[0],
                 config=crawler_config,
             )
+            from governor import wall_clock_seconds
+            if wall_clock_seconds(config) > 0:
+                results_gen = _with_deadline(results_gen, wall_clock_seconds(config))
         else:
             # Default multi-URL streaming via arun_many
             dispatcher = MemoryAdaptiveDispatcher(

deploy/docker/config.yml

@@ -37,7 +37,7 @@ redis:
 # (the pre-hardening behavior), so you can relax or disable each one.
 limits:
   max_body_bytes: 10485760  # 10 MiB request body cap (413 if exceeded); 0 = unbounded
-  max_pages: 100            # deep-crawl page budget clamp (defense in depth)
+  max_pages: 100            # deep-crawl page budget clamp (deep crawl needs CRAWL4AI_ALLOW_DEEP_CRAWL=true); 0 = no limit
   max_depth: 5              # deep-crawl depth clamp
   wall_clock_s: 300         # per-crawl deadline in seconds (504 on timeout); 0 = no deadline
   queue:                    # background job queue for /crawl/job and /llm/job

deploy/docker/governor.py

@@ -1,15 +1,15 @@
 """
 governor.py - server-enforced resource governance (R5).
 
-The deep-crawl DoS via a client-supplied unbounded max_pages is already closed
-upstream: R2 forbids `deep_crawl_strategy` on an untrusted request body, and the
-`urls` list is capped at 100 by the request schema. This module adds the two
-remaining verifiable chokepoints:
+R2 forbids `deep_crawl_strategy` on an untrusted request body unless the
+operator sets CRAWL4AI_ALLOW_DEEP_CRAWL=true, and the `urls` list is capped at
+100 by the request schema. This module adds the two remaining verifiable
+chokepoints:
 
   * a request body-size limit (ASGI middleware) so a giant body / inline `raw:`
     HTML cannot be buffered and processed in-process -> 413;
-  * clamp_deep_crawl(): defense in depth for any *trusted* / server-built config
-    that still carries a deep_crawl strategy with an unbounded page/depth count.
+  * clamp_deep_crawl(): bounds the page/depth budget and URL length of any
+    deep-crawl strategy, whether opted in from a request or server-built.
 
 Heavier governance (bounded work queue replacing BackgroundTasks, per-principal
 Redis quotas, wall-clock deadlines, stream decoupling) is left for the
@@ -19,11 +19,26 @@
 from __future__ import annotations
 
 import json
-import math
+
+from crawl4ai.deep_crawling.filters import FilterChain, URLFilter
 
 DEFAULT_MAX_BODY_BYTES = 10 * 1024 * 1024  # 10 MiB
 DEFAULT_MAX_PAGES = 100
 DEFAULT_MAX_DEPTH = 5
+DEFAULT_MAX_URL_LENGTH = 2048
+
+
+class URLLengthFilter(URLFilter):
+    """Reject discovered URLs too long to filter, score and cache cheaply."""
+
+    def __init__(self, max_length: int = DEFAULT_MAX_URL_LENGTH):
+        super().__init__()
+        self.max_length = max_length
+
+    def apply(self, url: str) -> bool:
+        passed = len(url) <= self.max_length
+        self._update_stats(passed)
+        return passed
 
 
 class BodySizeLimitMiddleware:
@@ -65,27 +80,39 @@ async def _reject(self, send):
 
 def clamp_deep_crawl(crawler_config, *, max_pages: int = DEFAULT_MAX_PAGES,
                      max_depth: int = DEFAULT_MAX_DEPTH) -> None:
-    """Clamp an attached deep-crawl strategy's page/depth budget in place.
+    """Clamp an attached deep-crawl strategy's page/depth budget in place and
+    put a URL-length guard first in its filter chain.
 
-    Defense in depth: untrusted bodies cannot set deep_crawl_strategy at all
-    (R2), but a server/base config might, and the library default for max_pages
-    is infinity.
+    The library default for max_pages is infinity.
     """
     dc = getattr(crawler_config, "deep_crawl_strategy", None)
     if dc is None:
         return
+    # NaN fails every comparison, so test for "inside the range", not "above the cap".
     mp = getattr(dc, "max_pages", None)
-    if mp is None or (isinstance(mp, float) and math.isinf(mp)) or mp > max_pages:
+    # max_pages=0 means no limit, like the other limits in config.yml.
+    if max_pages and (not isinstance(mp, (int, float)) or not 0 <= mp <= max_pages):
         try:
             dc.max_pages = max_pages
         except Exception:
             pass
     md = getattr(dc, "max_depth", None)
-    if md is None or md > max_depth:
+    if not isinstance(md, (int, float)) or not 0 <= md <= max_depth:
         try:
             dc.max_depth = max_depth
         except Exception:
             pass
+    chain = getattr(dc, "filter_chain", None) or FilterChain()
+    dc.filter_chain = FilterChain([URLLengthFilter(), *chain.filters])
+
+
+def deep_crawl_limits(config: dict) -> dict:
+    """Deep-crawl page/depth clamps from config limits."""
+    limits = _limits(config)
+    return {
+        "max_pages": int(limits.get("max_pages", DEFAULT_MAX_PAGES)),
+        "max_depth": int(limits.get("max_depth", DEFAULT_MAX_DEPTH)),
+    }
 
 
 def max_body_bytes_from_config(config: dict) -> int: