From a4312acdf3bf648e5317894a68251d54bd6f1807 Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Mon, 5 Oct 2026 16:56:21 +0530 Subject: [PATCH 1/2] feat(docker): opt-in deep crawling via CRAWL4AI_ALLOW_DEEP_CRAWL (#2308) Requests may set deep_crawl_strategy when the flag is true, with safe types only, glob-only URL patterns, config page/depth limits and a deadline on streamed deep crawls. --- crawl4ai/async_configs.py | 55 +++++++++++++++++- crawl4ai/deep_crawling/filters.py | 3 +- deploy/docker/MIGRATION.md | 4 ++ deploy/docker/README.md | 57 ++++++++++++++++++- deploy/docker/api.py | 31 ++++++++-- deploy/docker/config.yml | 2 +- deploy/docker/governor.py | 52 ++++++++++++----- .../tests/test_security_resource_caps.py | 20 +++++++ docker-compose.yml | 2 + docs/md_v2/core/self-hosting.md | 56 +++++++++++++++++- tests/unit/test_config_provenance.py | 52 +++++++++++++++++ 11 files changed, 312 insertions(+), 22 deletions(-) diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index cc1efdb8e..c131ff39d 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -199,6 +199,14 @@ class UntrustedConfigError(ValueError): "CacheMode", "MatchMode", "DisplayMode", } +# Allowed only with CRAWL4AI_ALLOW_DEEP_CRAWL=true; SEOFilter/ContentRelevanceFilter excluded (unguarded httpx fetch). +UNTRUSTED_DEEP_CRAWL_TYPES = { + "BFSDeepCrawlStrategy", "DFSDeepCrawlStrategy", "BestFirstCrawlingStrategy", + "FilterChain", "URLPatternFilter", "DomainFilter", "ContentTypeFilter", + "KeywordRelevanceScorer", "CompositeScorer", "DomainAuthorityScorer", + "FreshnessScorer", "PathDepthScorer", +} + # Fields that must NEVER come from an untrusted body. Presence => 400 (loud), so # a client smuggling these gets an explicit error rather than a silent drop. UNTRUSTED_FORBIDDEN_FIELDS = { @@ -223,6 +231,11 @@ class UntrustedConfigError(ValueError): "PDFContentScrapingStrategy": { "image_save_dir", "save_images_locally", }, + # resume_state.pending would queue URLs that skip seed validation. + **{ + name: {"resume_state", "logger", "on_state_change", "should_cancel"} + for name in ("BFSDeepCrawlStrategy", "DFSDeepCrawlStrategy", "BestFirstCrawlingStrategy") + }, } # Field names that must NEVER be set from an untrusted body on ANY allowed @@ -302,6 +315,7 @@ class UntrustedConfigError(ValueError): _MAX_LINK_PREVIEW_LINKS = 100 # the class default _MAX_LINK_PREVIEW_CONCURRENCY = 10 # the class default _MAX_LINK_PREVIEW_TIMEOUT_S = 10 # seconds here, not ms +_MAX_DEEP_CRAWL_LIST = 100 # patterns, keywords, filters, scorers per object def _max_timeout_ms() -> int: @@ -339,10 +353,18 @@ def _max_timeout_ms() -> int: return ceiling +def _deep_crawl_allowed() -> bool: + """Operator opt-in for deep crawling from untrusted requests.""" + return os.getenv("CRAWL4AI_ALLOW_DEEP_CRAWL", "false").lower() == "true" + + def _filter_untrusted_fields(type_name: str, params: dict) -> dict: """Drop non-allowlisted fields and raise on forbidden (power) fields.""" forbidden = UNTRUSTED_FORBIDDEN_FIELDS.get(type_name, set()) allowlist = UNTRUSTED_FIELD_ALLOWLIST.get(type_name) # None => keep all non-forbidden + if type_name == "CrawlerRunConfig" and _deep_crawl_allowed(): + forbidden = forbidden - {"deep_crawl_strategy"} + allowlist = allowlist | {"deep_crawl_strategy"} out = {} for key, value in params.items(): if key in UNTRUSTED_GLOBAL_FORBIDDEN_FIELDS or key in forbidden: @@ -352,6 +374,33 @@ def _filter_untrusted_fields(type_name: str, params: dict) -> dict: if allowlist is not None and key not in allowlist: continue # silently drop unknown/unsafe fields (forward-compatible) out[key] = value + if type_name in UNTRUSTED_DEEP_CRAWL_TYPES and any( + isinstance(v, list) and len(v) > _MAX_DEEP_CRAWL_LIST for v in out.values() + ): + raise UntrustedConfigError( + f"{type_name} list parameters are limited to {_MAX_DEEP_CRAWL_LIST} items" + ) + if type_name == "URLPatternFilter": + # Regex and *.domain:// patterns compile unescaped, and globs not led by * search in + # quadratic time; allow only *-led globs, /prefix/* and *.ext. + patterns = out.get("patterns") + patterns = [patterns] if isinstance(patterns, str) else patterns + if not isinstance(patterns, list) or not all( + isinstance(p, str) + and not (p.startswith("^") or p.endswith("$") or "\\d" in p or "**" in p) + and not ("://" in p and p.startswith("*.")) + and (p.startswith("*") or (p.count("*") == 1 and p.endswith("/*"))) + for p in patterns + ): + raise UntrustedConfigError( + "URLPatternFilter accepts only globs that start with '*' or end with a " + "single '/*' from an untrusted request" + ) + if type_name == "CrawlerRunConfig" and out.get("deep_crawl_strategy") and out.get("fetch_ssl_certificate"): + # The certificate fetch dials each discovered link directly, outside the egress proxy. + raise UntrustedConfigError( + "fetch_ssl_certificate cannot be combined with deep_crawl_strategy from an untrusted request" + ) return out @@ -556,7 +605,11 @@ def from_serializable_dict(data: Any, provenance: "Provenance" = None) -> Any: # Untrusted bodies may only construct the strict subset of types and # may not set forbidden power-fields. - if provenance == Provenance.UNTRUSTED and type_name not in UNTRUSTED_ALLOWED_TYPES: + if ( + provenance == Provenance.UNTRUSTED + and type_name not in UNTRUSTED_ALLOWED_TYPES + and not (type_name in UNTRUSTED_DEEP_CRAWL_TYPES and _deep_crawl_allowed()) + ): raise UntrustedConfigError( f"type '{type_name}' may not be constructed from an untrusted request" ) diff --git a/crawl4ai/deep_crawling/filters.py b/crawl4ai/deep_crawling/filters.py index 2fb819ecc..85352def0 100644 --- a/crawl4ai/deep_crawling/filters.py +++ b/crawl4ai/deep_crawling/filters.py @@ -209,7 +209,8 @@ def _add_pattern(self, pattern: str, pattern_type: int): lambda m: f'({"|".join(m.group(1).split(","))})', pattern, ) - pattern = fnmatch.translate(pattern) + # A leading * already matches any prefix; \A stops search() retrying every offset. + pattern = ("\\A" if pattern.startswith("*") else "") + fnmatch.translate(pattern) self._path_patterns.append( pattern if isinstance(pattern, Pattern) else re.compile(pattern) ) diff --git a/deploy/docker/MIGRATION.md b/deploy/docker/MIGRATION.md index 5123c950f..c389a045e 100644 --- a/deploy/docker/MIGRATION.md +++ b/deploy/docker/MIGRATION.md @@ -68,6 +68,10 @@ control): `base_url`, `deep_crawl_strategy`, `simulate_user`, `magic`, `process_in_browser`, and nested LLM config objects. +`deep_crawl_strategy` can be turned back on by the operator with +`CRAWL4AI_ALLOW_DEEP_CRAWL=true`, with server-side limits. See "Deep Crawling +(opt-in)" in the self-hosting guide. + Unknown fields are dropped; timeouts, viewport and scroll counts are clamped to safe maximums. diff --git a/deploy/docker/README.md b/deploy/docker/README.md index 9216de6ce..1b8c57756 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -562,7 +562,62 @@ Crucially, when sending configurations directly via JSON, they **must** follow t ``` **LLM Extraction Strategy** *(Keep example, ensure schema uses type/value wrapper)* -*(Keep Deep Crawler Example)* +#### Deep Crawling (opt-in) + +Deep crawling over the API is **disabled by default**: a request with +`deep_crawl_strategy` returns HTTP 400. To allow it, start the container with +`CRAWL4AI_ALLOW_DEEP_CRAWL=true`: + +```bash +docker run -d \ + -p 11235:11235 \ + --name crawl4ai \ + --shm-size=1g \ + -e CRAWL4AI_API_TOKEN="$CRAWL4AI_API_TOKEN" \ + -e CRAWL4AI_ALLOW_DEEP_CRAWL=true \ + unclecode/crawl4ai:latest +``` + +With Docker Compose, put `CRAWL4AI_ALLOW_DEEP_CRAWL=true` in the `.env` file in +the project root, or export it in your shell. The compose file sets this +variable itself, so a value in `.llm.env` is ignored. + +```json +{ + "urls": ["https://docs.crawl4ai.com"], + "crawler_config": { + "type": "CrawlerRunConfig", + "params": { + "deep_crawl_strategy": { + "type": "BFSDeepCrawlStrategy", + "params": { + "max_depth": 2, + "max_pages": 20, + "filter_chain": { + "type": "FilterChain", + "params": { + "filters": [ + {"type": "URLPatternFilter", "params": {"patterns": ["*/core/*"]}}, + {"type": "DomainFilter", "params": {"allowed_domains": ["docs.crawl4ai.com"]}} + ] + } + } + } + } + } + } +} +``` + +Rules the server applies: + +- **One start URL** per request, and not inside `crawler_configs`. +- `max_pages` and `max_depth` are clamped to `limits.max_pages` and `limits.max_depth` in `config.yml` (default 100 and 5). The crawl stops after `limits.wall_clock_s`, also on `/crawl/stream`. +- **Allowed types:** `BFSDeepCrawlStrategy`, `DFSDeepCrawlStrategy`, `BestFirstCrawlingStrategy`; `FilterChain` with `URLPatternFilter`, `DomainFilter`, `ContentTypeFilter`; scorers `KeywordRelevanceScorer`, `CompositeScorer`, `DomainAuthorityScorer`, `FreshnessScorer`, `PathDepthScorer`. `SEOFilter` and `ContentRelevanceFilter` are refused (they fetch pages outside the egress controls). +- `URLPatternFilter` accepts globs that start with `*` (`*/blog/*`), `/prefix/*` or `*.ext`. Regex patterns are refused. +- `resume_state` is refused, and so is `fetch_ssl_certificate` together with a deep crawl. Lists (patterns, keywords, filters, scorers) are limited to 100 items. Discovered links longer than 2048 characters are skipped. + +Turn the flag on only when you accept that one request can fetch many pages. ### REST API Examples diff --git a/deploy/docker/api.py b/deploy/docker/api.py index 244b5046f..d4636c412 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -641,6 +641,8 @@ async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator) yield json.dumps({"status": "completed"}).encode('utf-8') + except asyncio.TimeoutError: + yield json.dumps({"status": "error", "error": "Crawl exceeded the time limit"}).encode('utf-8') except asyncio.CancelledError: logger.warning("Client disconnected during streaming") finally: @@ -663,6 +665,8 @@ def _load_crawler_configs( for cc in crawler_configs ] for cfg in config_list: + if cfg.deep_crawl_strategy is not None: + raise HTTPException(status_code=400, detail="Deep crawling supports exactly one URL per request.") for key, value in (base_config or {}).items(): if hasattr(cfg, key): current_value = getattr(cfg, key) @@ -699,6 +703,20 @@ def _needs_pdf_crawler( return all(pdf) +async def _with_deadline(gen: AsyncGenerator, seconds: float) -> AsyncGenerator: + """Stop a result stream once its wall-clock budget is spent (raises TimeoutError).""" + loop = asyncio.get_running_loop() + end = loop.time() + seconds + try: + while True: + try: + yield await asyncio.wait_for(gen.__anext__(), max(end - loop.time(), 0)) + except StopAsyncIteration: + return + finally: + await gen.aclose() + + def _normalize_and_validate_seeds(urls: List[str]) -> Tuple[List[str], List[CrawlResult]]: """Prefix bare hosts with https:// and SSRF-validate every seed URL's destination. Shared by the streaming and non-streaming crawl handlers so a @@ -753,8 +771,10 @@ async def handle_crawl_request( crawler_config = CrawlerRunConfig.load(crawler_config, provenance=Provenance.UNTRUSTED) from egress_broker import enforce_egress enforce_egress(browser_config) - from governor import clamp_deep_crawl - clamp_deep_crawl(crawler_config) + from governor import clamp_deep_crawl, deep_crawl_limits + clamp_deep_crawl(crawler_config, **deep_crawl_limits(config)) + if crawler_config.deep_crawl_strategy is not None and len(urls) + len(refused) != 1: + raise HTTPException(status_code=400, detail="Deep crawling supports exactly one URL per request.") dispatcher = MemoryAdaptiveDispatcher( memory_threshold_percent=config["crawler"]["memory_threshold_percent"], @@ -984,9 +1004,9 @@ async def handle_stream_crawl_request( crawler_config = CrawlerRunConfig.load( crawler_config, provenance=Provenance.UNTRUSTED ) - from governor import clamp_deep_crawl + from governor import clamp_deep_crawl, deep_crawl_limits - clamp_deep_crawl(crawler_config) + clamp_deep_crawl(crawler_config, **deep_crawl_limits(config)) crawler_config.stream = True config_list = ( _load_crawler_configs( @@ -1034,6 +1054,9 @@ async def handle_stream_crawl_request( urls[0], config=crawler_config, ) + from governor import wall_clock_seconds + if wall_clock_seconds(config) > 0: + results_gen = _with_deadline(results_gen, wall_clock_seconds(config)) else: # Default multi-URL streaming via arun_many dispatcher = MemoryAdaptiveDispatcher( diff --git a/deploy/docker/config.yml b/deploy/docker/config.yml index 094209899..c22bf3817 100644 --- a/deploy/docker/config.yml +++ b/deploy/docker/config.yml @@ -37,7 +37,7 @@ redis: # (the pre-hardening behavior), so you can relax or disable each one. limits: max_body_bytes: 10485760 # 10 MiB request body cap (413 if exceeded); 0 = unbounded - max_pages: 100 # deep-crawl page budget clamp (defense in depth) + max_pages: 100 # deep-crawl page budget clamp (deep crawl needs CRAWL4AI_ALLOW_DEEP_CRAWL=true) max_depth: 5 # deep-crawl depth clamp wall_clock_s: 300 # per-crawl deadline in seconds (504 on timeout); 0 = no deadline queue: # background job queue for /crawl/job and /llm/job diff --git a/deploy/docker/governor.py b/deploy/docker/governor.py index 73534e993..4cabe0abf 100644 --- a/deploy/docker/governor.py +++ b/deploy/docker/governor.py @@ -1,15 +1,15 @@ """ governor.py - server-enforced resource governance (R5). -The deep-crawl DoS via a client-supplied unbounded max_pages is already closed -upstream: R2 forbids `deep_crawl_strategy` on an untrusted request body, and the -`urls` list is capped at 100 by the request schema. This module adds the two -remaining verifiable chokepoints: +R2 forbids `deep_crawl_strategy` on an untrusted request body unless the +operator sets CRAWL4AI_ALLOW_DEEP_CRAWL=true, and the `urls` list is capped at +100 by the request schema. This module adds the two remaining verifiable +chokepoints: * a request body-size limit (ASGI middleware) so a giant body / inline `raw:` HTML cannot be buffered and processed in-process -> 413; - * clamp_deep_crawl(): defense in depth for any *trusted* / server-built config - that still carries a deep_crawl strategy with an unbounded page/depth count. + * clamp_deep_crawl(): bounds the page/depth budget and URL length of any + deep-crawl strategy, whether opted in from a request or server-built. Heavier governance (bounded work queue replacing BackgroundTasks, per-principal Redis quotas, wall-clock deadlines, stream decoupling) is left for the @@ -19,11 +19,26 @@ from __future__ import annotations import json -import math + +from crawl4ai.deep_crawling.filters import FilterChain, URLFilter DEFAULT_MAX_BODY_BYTES = 10 * 1024 * 1024 # 10 MiB DEFAULT_MAX_PAGES = 100 DEFAULT_MAX_DEPTH = 5 +DEFAULT_MAX_URL_LENGTH = 2048 + + +class URLLengthFilter(URLFilter): + """Reject discovered URLs too long to filter, score and cache cheaply.""" + + def __init__(self, max_length: int = DEFAULT_MAX_URL_LENGTH): + super().__init__() + self.max_length = max_length + + def apply(self, url: str) -> bool: + passed = len(url) <= self.max_length + self._update_stats(passed) + return passed class BodySizeLimitMiddleware: @@ -65,27 +80,38 @@ async def _reject(self, send): def clamp_deep_crawl(crawler_config, *, max_pages: int = DEFAULT_MAX_PAGES, max_depth: int = DEFAULT_MAX_DEPTH) -> None: - """Clamp an attached deep-crawl strategy's page/depth budget in place. + """Clamp an attached deep-crawl strategy's page/depth budget in place and + put a URL-length guard first in its filter chain. - Defense in depth: untrusted bodies cannot set deep_crawl_strategy at all - (R2), but a server/base config might, and the library default for max_pages - is infinity. + The library default for max_pages is infinity. """ dc = getattr(crawler_config, "deep_crawl_strategy", None) if dc is None: return + # NaN fails every comparison, so test for "inside the range", not "above the cap". mp = getattr(dc, "max_pages", None) - if mp is None or (isinstance(mp, float) and math.isinf(mp)) or mp > max_pages: + if not isinstance(mp, (int, float)) or not 0 <= mp <= max_pages: try: dc.max_pages = max_pages except Exception: pass md = getattr(dc, "max_depth", None) - if md is None or md > max_depth: + if not isinstance(md, (int, float)) or not 0 <= md <= max_depth: try: dc.max_depth = max_depth except Exception: pass + chain = getattr(dc, "filter_chain", None) or FilterChain() + dc.filter_chain = FilterChain([URLLengthFilter(), *chain.filters]) + + +def deep_crawl_limits(config: dict) -> dict: + """Deep-crawl page/depth clamps from config limits.""" + limits = _limits(config) + return { + "max_pages": int(limits.get("max_pages", DEFAULT_MAX_PAGES)), + "max_depth": int(limits.get("max_depth", DEFAULT_MAX_DEPTH)), + } def max_body_bytes_from_config(config: dict) -> int: diff --git a/deploy/docker/tests/test_security_resource_caps.py b/deploy/docker/tests/test_security_resource_caps.py index 6cf4f0385..a60a3cc27 100644 --- a/deploy/docker/tests/test_security_resource_caps.py +++ b/deploy/docker/tests/test_security_resource_caps.py @@ -49,6 +49,26 @@ def test_no_deep_crawl_is_noop(self): clamp_deep_crawl(cfg) # must not raise assert cfg.deep_crawl_strategy is None + def test_nan_limits_clamped(self): + from governor import clamp_deep_crawl + from crawl4ai.deep_crawling import BFSDeepCrawlStrategy + from crawl4ai import CrawlerRunConfig + nan = float("nan") + cfg = CrawlerRunConfig(deep_crawl_strategy=BFSDeepCrawlStrategy(max_depth=nan, max_pages=nan)) + clamp_deep_crawl(cfg) + assert (cfg.deep_crawl_strategy.max_pages, cfg.deep_crawl_strategy.max_depth) == (100, 5) + + def test_url_length_guard_added_even_without_filter_chain(self): + from governor import URLLengthFilter, clamp_deep_crawl + from crawl4ai.deep_crawling import BFSDeepCrawlStrategy + from crawl4ai import CrawlerRunConfig + cfg = CrawlerRunConfig(deep_crawl_strategy=BFSDeepCrawlStrategy(max_depth=1, filter_chain=None)) + clamp_deep_crawl(cfg) + guard = cfg.deep_crawl_strategy.filter_chain.filters[0] + assert isinstance(guard, URLLengthFilter) + assert not guard.apply("https://a.com/" + "x" * 3000) + assert guard.apply("https://a.com/blog/post") + class TestUntrustedDeepCrawlStillForbidden: def test_request_cannot_set_deep_crawl(self): diff --git a/docker-compose.yml b/docker-compose.yml index 02cbe0584..a0a7aa9af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,8 @@ x-base-config: &base-config environment: # Auth token passthrough from the host shell (overwrites .llm.env). - CRAWL4AI_API_TOKEN=${CRAWL4AI_API_TOKEN:-} + # Deep crawling over the API (off unless set to true in the shell or .env). + - CRAWL4AI_ALLOW_DEEP_CRAWL=${CRAWL4AI_ALLOW_DEEP_CRAWL:-false} # Uncomment to set default environment variables (will overwrite .llm.env) # - OPENAI_API_KEY=${OPENAI_API_KEY:-} # - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY:-} diff --git a/docs/md_v2/core/self-hosting.md b/docs/md_v2/core/self-hosting.md index 8d0e54f2b..f5445c988 100644 --- a/docs/md_v2/core/self-hosting.md +++ b/docs/md_v2/core/self-hosting.md @@ -1290,7 +1290,61 @@ Crucially, when sending configurations directly via JSON, they **must** follow t ``` **LLM Extraction Strategy** *(Keep example, ensure schema uses type/value wrapper)* -*(Keep Deep Crawler Example)* +#### Deep Crawling (opt-in) + +Deep crawling over the API is **disabled by default**: a request with +`deep_crawl_strategy` returns HTTP 400. To allow it, start the container with +`CRAWL4AI_ALLOW_DEEP_CRAWL=true`: + +```bash +docker run -d \ + -p 11235:11235 \ + --name crawl4ai \ + --shm-size=1g \ + -e CRAWL4AI_API_TOKEN="$CRAWL4AI_API_TOKEN" \ + -e CRAWL4AI_ALLOW_DEEP_CRAWL=true \ + unclecode/crawl4ai:latest +``` + +With Docker Compose, put `CRAWL4AI_ALLOW_DEEP_CRAWL=true` in the `.env` file in +the project root, or export it in your shell. + +```json +{ + "urls": ["https://docs.crawl4ai.com"], + "crawler_config": { + "type": "CrawlerRunConfig", + "params": { + "deep_crawl_strategy": { + "type": "BFSDeepCrawlStrategy", + "params": { + "max_depth": 2, + "max_pages": 20, + "filter_chain": { + "type": "FilterChain", + "params": { + "filters": [ + {"type": "URLPatternFilter", "params": {"patterns": ["*/core/*"]}}, + {"type": "DomainFilter", "params": {"allowed_domains": ["docs.crawl4ai.com"]}} + ] + } + } + } + } + } + } +} +``` + +Rules the server applies: + +- **One start URL** per request, and not inside `crawler_configs`. +- `max_pages` and `max_depth` are clamped to `limits.max_pages` and `limits.max_depth` in `config.yml` (default 100 and 5). The crawl stops after `limits.wall_clock_s`, also on `/crawl/stream`. +- **Allowed types:** `BFSDeepCrawlStrategy`, `DFSDeepCrawlStrategy`, `BestFirstCrawlingStrategy`; `FilterChain` with `URLPatternFilter`, `DomainFilter`, `ContentTypeFilter`; scorers `KeywordRelevanceScorer`, `CompositeScorer`, `DomainAuthorityScorer`, `FreshnessScorer`, `PathDepthScorer`. `SEOFilter` and `ContentRelevanceFilter` are refused (they fetch pages outside the egress controls). +- `URLPatternFilter` accepts globs that start with `*` (`*/blog/*`), `/prefix/*` or `*.ext`. Regex patterns are refused. +- `resume_state` is refused, and so is `fetch_ssl_certificate` together with a deep crawl. Lists (patterns, keywords, filters, scorers) are limited to 100 items. Discovered links longer than 2048 characters are skipped. + +Turn the flag on only when you accept that one request can fetch many pages. ### LLM Configuration Examples diff --git a/tests/unit/test_config_provenance.py b/tests/unit/test_config_provenance.py index 49246a8e6..883b71702 100644 --- a/tests/unit/test_config_provenance.py +++ b/tests/unit/test_config_provenance.py @@ -116,3 +116,55 @@ def test_plain_business_dict_with_type_key_still_passes(): provenance=UNTRUSTED, ) assert cfg.extraction_strategy.schema["baseSelector"] == ".item" + + +def _deep_crawl(**params): + return { + "type": "CrawlerRunConfig", + "params": {"deep_crawl_strategy": {"type": "BFSDeepCrawlStrategy", "params": {"max_depth": 2, **params}}}, + } + + +def test_deep_crawl_allowed_with_opt_in(monkeypatch): + monkeypatch.setenv("CRAWL4AI_ALLOW_DEEP_CRAWL", "true") + chain = {"type": "FilterChain", "params": {"filters": [ + {"type": "URLPatternFilter", "params": {"patterns": ["*/blog/*"]}}]}} + cfg = CrawlerRunConfig.load(_deep_crawl(filter_chain=chain, max_pages=50), provenance=UNTRUSTED) + assert type(cfg.deep_crawl_strategy).__name__ == "BFSDeepCrawlStrategy" + assert cfg.deep_crawl_strategy.max_pages == 50 + + +@pytest.mark.parametrize("pattern", ["^(a+)+$", "tag*x"]) +def test_deep_crawl_unsafe_pattern_refused(monkeypatch, pattern): + monkeypatch.setenv("CRAWL4AI_ALLOW_DEEP_CRAWL", "true") + chain = {"type": "FilterChain", "params": {"filters": [ + {"type": "URLPatternFilter", "params": {"patterns": [pattern]}}]}} + with pytest.raises(UntrustedConfigError, match="URLPatternFilter"): + CrawlerRunConfig.load(_deep_crawl(filter_chain=chain), provenance=UNTRUSTED) + + +def test_deep_crawl_resume_state_refused(monkeypatch): + monkeypatch.setenv("CRAWL4AI_ALLOW_DEEP_CRAWL", "true") + data = _deep_crawl(resume_state={"pending": [{"url": "http://169.254.169.254/", "depth": 1}]}) + with pytest.raises(UntrustedConfigError, match="resume_state"): + CrawlerRunConfig.load(data, provenance=UNTRUSTED) + + +def test_deep_crawl_ssl_certificate_refused(monkeypatch): + monkeypatch.setenv("CRAWL4AI_ALLOW_DEEP_CRAWL", "true") + data = _deep_crawl() + data["params"]["fetch_ssl_certificate"] = True + with pytest.raises(UntrustedConfigError, match="fetch_ssl_certificate"): + CrawlerRunConfig.load(data, provenance=UNTRUSTED) + + +def test_url_pattern_star_led_glob_is_linear_and_unchanged(): + import time + from crawl4ai.deep_crawling.filters import URLPatternFilter + + f = URLPatternFilter(["*/blog/*"]) + assert f.apply("https://x.com/blog/post") + assert not f.apply("https://x.com/news/post") + start = time.time() + assert not f.apply("https://x.com/" + "a" * 50000) + assert time.time() - start < 0.5 From d6f8a755d5a8648f6e55f2e63e8731afe572b554 Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Tue, 13 Oct 2026 13:14:03 +0530 Subject: [PATCH 2/2] feat(docker): total size budget for untrusted deep-crawl strategies Refuse nested CompositeScorer, limit one strategy to 20 filters and scorers and 200 list or dict items, and treat limits.max_pages: 0 as no limit.x --- crawl4ai/async_configs.py | 26 ++++++++++++++++++ deploy/docker/README.md | 4 +-- deploy/docker/config.yml | 2 +- deploy/docker/governor.py | 3 ++- .../tests/test_security_resource_caps.py | 8 ++++++ docs/md_v2/core/self-hosting.md | 4 +-- tests/unit/test_config_provenance.py | 27 +++++++++++++++++++ 7 files changed, 68 insertions(+), 6 deletions(-) diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index c131ff39d..ad51cb178 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -316,6 +316,8 @@ class UntrustedConfigError(ValueError): _MAX_LINK_PREVIEW_CONCURRENCY = 10 # the class default _MAX_LINK_PREVIEW_TIMEOUT_S = 10 # seconds here, not ms _MAX_DEEP_CRAWL_LIST = 100 # patterns, keywords, filters, scorers per object +_MAX_DEEP_CRAWL_OBJECTS = 20 # filters and scorers in one strategy +_MAX_DEEP_CRAWL_ITEMS = 200 # list and dict items (patterns, keywords, ...) in one strategy def _max_timeout_ms() -> int: @@ -358,6 +360,16 @@ def _deep_crawl_allowed() -> bool: return os.getenv("CRAWL4AI_ALLOW_DEEP_CRAWL", "false").lower() == "true" +def _all_nodes(node): + """Yield every value in a request tree. An object's params dict is entered but not yielded.""" + yield node + if _is_typed_shape(node) and isinstance(node.get("params"), dict): + node = node["params"] + children = node.values() if isinstance(node, dict) else node if isinstance(node, list) else [] + for child in children: + yield from _all_nodes(child) + + def _filter_untrusted_fields(type_name: str, params: dict) -> dict: """Drop non-allowlisted fields and raise on forbidden (power) fields.""" forbidden = UNTRUSTED_FORBIDDEN_FIELDS.get(type_name, set()) @@ -380,6 +392,16 @@ def _filter_untrusted_fields(type_name: str, params: dict) -> dict: raise UntrustedConfigError( f"{type_name} list parameters are limited to {_MAX_DEEP_CRAWL_LIST} items" ) + if type_name in ("BFSDeepCrawlStrategy", "DFSDeepCrawlStrategy", "BestFirstCrawlingStrategy"): + # Filters and scorers run synchronously on the event loop for every discovered link. + nodes = [n for value in out.values() for n in _all_nodes(value)] + filters_and_scorers = sum(1 for n in nodes if isinstance(n, dict) and n.get("type") in UNTRUSTED_DEEP_CRAWL_TYPES) + list_items = sum(len(n) for n in nodes if isinstance(n, (list, dict)) and not _is_typed_shape(n)) + if filters_and_scorers > _MAX_DEEP_CRAWL_OBJECTS or list_items > _MAX_DEEP_CRAWL_ITEMS: + raise UntrustedConfigError( + f"{type_name} is limited to {_MAX_DEEP_CRAWL_OBJECTS} filters and scorers " + f"and {_MAX_DEEP_CRAWL_ITEMS} list items in total" + ) if type_name == "URLPatternFilter": # Regex and *.domain:// patterns compile unescaped, and globs not led by * search in # quadratic time; allow only *-led globs, /prefix/* and *.ext. @@ -396,6 +418,10 @@ def _filter_untrusted_fields(type_name: str, params: dict) -> dict: "URLPatternFilter accepts only globs that start with '*' or end with a " "single '/*' from an untrusted request" ) + if type_name == "CompositeScorer" and any( + isinstance(s, dict) and s.get("type") == "CompositeScorer" for s in out.get("scorers") or [] + ): + raise UntrustedConfigError("CompositeScorer cannot be nested from an untrusted request") if type_name == "CrawlerRunConfig" and out.get("deep_crawl_strategy") and out.get("fetch_ssl_certificate"): # The certificate fetch dials each discovered link directly, outside the egress proxy. raise UntrustedConfigError( diff --git a/deploy/docker/README.md b/deploy/docker/README.md index 1b8c57756..a52816018 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -612,10 +612,10 @@ variable itself, so a value in `.llm.env` is ignored. Rules the server applies: - **One start URL** per request, and not inside `crawler_configs`. -- `max_pages` and `max_depth` are clamped to `limits.max_pages` and `limits.max_depth` in `config.yml` (default 100 and 5). The crawl stops after `limits.wall_clock_s`, also on `/crawl/stream`. +- `max_pages` and `max_depth` are clamped to `limits.max_pages` and `limits.max_depth` in `config.yml` (default 100 and 5; `max_pages: 0` means no limit). The crawl stops after `limits.wall_clock_s`, also on `/crawl/stream`. - **Allowed types:** `BFSDeepCrawlStrategy`, `DFSDeepCrawlStrategy`, `BestFirstCrawlingStrategy`; `FilterChain` with `URLPatternFilter`, `DomainFilter`, `ContentTypeFilter`; scorers `KeywordRelevanceScorer`, `CompositeScorer`, `DomainAuthorityScorer`, `FreshnessScorer`, `PathDepthScorer`. `SEOFilter` and `ContentRelevanceFilter` are refused (they fetch pages outside the egress controls). - `URLPatternFilter` accepts globs that start with `*` (`*/blog/*`), `/prefix/*` or `*.ext`. Regex patterns are refused. -- `resume_state` is refused, and so is `fetch_ssl_certificate` together with a deep crawl. Lists (patterns, keywords, filters, scorers) are limited to 100 items. Discovered links longer than 2048 characters are skipped. +- `resume_state` is refused, and so is `fetch_ssl_certificate` together with a deep crawl. Lists (patterns, keywords, filters, scorers) are limited to 100 items. One strategy may have at most 20 filters and scorers and 200 list items in total, and a `CompositeScorer` cannot contain another `CompositeScorer`. Discovered links longer than 2048 characters are skipped. Turn the flag on only when you accept that one request can fetch many pages. diff --git a/deploy/docker/config.yml b/deploy/docker/config.yml index c22bf3817..8bf16a118 100644 --- a/deploy/docker/config.yml +++ b/deploy/docker/config.yml @@ -37,7 +37,7 @@ redis: # (the pre-hardening behavior), so you can relax or disable each one. limits: max_body_bytes: 10485760 # 10 MiB request body cap (413 if exceeded); 0 = unbounded - max_pages: 100 # deep-crawl page budget clamp (deep crawl needs CRAWL4AI_ALLOW_DEEP_CRAWL=true) + max_pages: 100 # deep-crawl page budget clamp (deep crawl needs CRAWL4AI_ALLOW_DEEP_CRAWL=true); 0 = no limit max_depth: 5 # deep-crawl depth clamp wall_clock_s: 300 # per-crawl deadline in seconds (504 on timeout); 0 = no deadline queue: # background job queue for /crawl/job and /llm/job diff --git a/deploy/docker/governor.py b/deploy/docker/governor.py index 4cabe0abf..0c1f8d6a6 100644 --- a/deploy/docker/governor.py +++ b/deploy/docker/governor.py @@ -90,7 +90,8 @@ def clamp_deep_crawl(crawler_config, *, max_pages: int = DEFAULT_MAX_PAGES, return # NaN fails every comparison, so test for "inside the range", not "above the cap". mp = getattr(dc, "max_pages", None) - if not isinstance(mp, (int, float)) or not 0 <= mp <= max_pages: + # max_pages=0 means no limit, like the other limits in config.yml. + if max_pages and (not isinstance(mp, (int, float)) or not 0 <= mp <= max_pages): try: dc.max_pages = max_pages except Exception: diff --git a/deploy/docker/tests/test_security_resource_caps.py b/deploy/docker/tests/test_security_resource_caps.py index a60a3cc27..f20d32157 100644 --- a/deploy/docker/tests/test_security_resource_caps.py +++ b/deploy/docker/tests/test_security_resource_caps.py @@ -69,6 +69,14 @@ def test_url_length_guard_added_even_without_filter_chain(self): assert not guard.apply("https://a.com/" + "x" * 3000) assert guard.apply("https://a.com/blog/post") + def test_zero_max_pages_means_no_limit(self): + from governor import clamp_deep_crawl + from crawl4ai.deep_crawling import BFSDeepCrawlStrategy + from crawl4ai import CrawlerRunConfig + cfg = CrawlerRunConfig(deep_crawl_strategy=BFSDeepCrawlStrategy(max_depth=2)) + clamp_deep_crawl(cfg, max_pages=0) + assert cfg.deep_crawl_strategy.max_pages == float("inf") + class TestUntrustedDeepCrawlStillForbidden: def test_request_cannot_set_deep_crawl(self): diff --git a/docs/md_v2/core/self-hosting.md b/docs/md_v2/core/self-hosting.md index f5445c988..5f6ae1c41 100644 --- a/docs/md_v2/core/self-hosting.md +++ b/docs/md_v2/core/self-hosting.md @@ -1339,10 +1339,10 @@ the project root, or export it in your shell. Rules the server applies: - **One start URL** per request, and not inside `crawler_configs`. -- `max_pages` and `max_depth` are clamped to `limits.max_pages` and `limits.max_depth` in `config.yml` (default 100 and 5). The crawl stops after `limits.wall_clock_s`, also on `/crawl/stream`. +- `max_pages` and `max_depth` are clamped to `limits.max_pages` and `limits.max_depth` in `config.yml` (default 100 and 5; `max_pages: 0` means no limit). The crawl stops after `limits.wall_clock_s`, also on `/crawl/stream`. - **Allowed types:** `BFSDeepCrawlStrategy`, `DFSDeepCrawlStrategy`, `BestFirstCrawlingStrategy`; `FilterChain` with `URLPatternFilter`, `DomainFilter`, `ContentTypeFilter`; scorers `KeywordRelevanceScorer`, `CompositeScorer`, `DomainAuthorityScorer`, `FreshnessScorer`, `PathDepthScorer`. `SEOFilter` and `ContentRelevanceFilter` are refused (they fetch pages outside the egress controls). - `URLPatternFilter` accepts globs that start with `*` (`*/blog/*`), `/prefix/*` or `*.ext`. Regex patterns are refused. -- `resume_state` is refused, and so is `fetch_ssl_certificate` together with a deep crawl. Lists (patterns, keywords, filters, scorers) are limited to 100 items. Discovered links longer than 2048 characters are skipped. +- `resume_state` is refused, and so is `fetch_ssl_certificate` together with a deep crawl. Lists (patterns, keywords, filters, scorers) are limited to 100 items. One strategy may have at most 20 filters and scorers and 200 list items in total, and a `CompositeScorer` cannot contain another `CompositeScorer`. Discovered links longer than 2048 characters are skipped. Turn the flag on only when you accept that one request can fetch many pages. diff --git a/tests/unit/test_config_provenance.py b/tests/unit/test_config_provenance.py index 883b71702..62f167a6f 100644 --- a/tests/unit/test_config_provenance.py +++ b/tests/unit/test_config_provenance.py @@ -158,6 +158,33 @@ def test_deep_crawl_ssl_certificate_refused(monkeypatch): CrawlerRunConfig.load(data, provenance=UNTRUSTED) +def _pattern_chain(*sizes): + return {"type": "FilterChain", "params": {"filters": [ + {"type": "URLPatternFilter", "params": {"patterns": [f"*/p{i}/*" for i in range(n)]}} + for n in sizes]}} + + +@pytest.mark.parametrize("strategy, match", [ + ({"filter_chain": _pattern_chain(101)}, "limited to 100 items"), + ({"filter_chain": _pattern_chain(*(1,) * 20)}, "in total"), + ({"filter_chain": _pattern_chain(100, 100, 1)}, "in total"), + ({"url_scorer": {"type": "KeywordRelevanceScorer", "params": { + "keywords": {"type": "dict", "value": {f"k{i}": 1 for i in range(201)}}}}}, "in total"), +]) +def test_deep_crawl_size_budget_refused(monkeypatch, strategy, match): + monkeypatch.setenv("CRAWL4AI_ALLOW_DEEP_CRAWL", "true") + with pytest.raises(UntrustedConfigError, match=match): + CrawlerRunConfig.load(_deep_crawl(**strategy), provenance=UNTRUSTED) + + +def test_deep_crawl_nested_composite_scorer_refused(monkeypatch): + monkeypatch.setenv("CRAWL4AI_ALLOW_DEEP_CRAWL", "true") + inner = {"type": "CompositeScorer", "params": {"scorers": [{"type": "PathDepthScorer", "params": {}}]}} + data = _deep_crawl(url_scorer={"type": "CompositeScorer", "params": {"scorers": [inner]}}) + with pytest.raises(UntrustedConfigError, match="nested"): + CrawlerRunConfig.load(data, provenance=UNTRUSTED) + + def test_url_pattern_star_led_glob_is_linear_and_unchanged(): import time from crawl4ai.deep_crawling.filters import URLPatternFilter