From a4c4b2d105015ff62a1d2e703cc39272090ba75d Mon Sep 17 00:00:00 2001
From: Quinn Xu <satonaoyukii@gmail.com>
Date: Thu, 15 Oct 2026 10:19:56 -0700
Subject: [PATCH] fix: keep fallback_fetch_function and other fields in
 CrawlerRunConfig.clone()

CrawlerRunConfig.clone() rebuilds the config from to_dict(), which left out
fallback_fetch_function, check_cache_freshness, cache_validation_timeout and
virtual_scroll_config. Deep crawl strategies clone the run config for every
batch, so these settings were silently reset and the fallback fetcher was
never called during deep crawls.

Add the missing keys to to_dict() and cover them with unit tests.

Fixes #2379
---
 crawl4ai/async_configs.py                  |  4 +++
 tests/unit/test_run_config_clone_fields.py | 38 ++++++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 tests/unit/test_run_config_clone_fields.py

diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py
index 15789be1e..d3eb96035 100644
--- a/crawl4ai/async_configs.py
+++ b/crawl4ai/async_configs.py
@@ -2398,6 +2398,8 @@ def to_dict(self):
             "disable_cache": self.disable_cache,
             "no_cache_read": self.no_cache_read,
             "no_cache_write": self.no_cache_write,
+            "check_cache_freshness": self.check_cache_freshness,
+            "cache_validation_timeout": self.cache_validation_timeout,
             "shared_data": self.shared_data,
             "wait_until": self.wait_until,
             "page_timeout": self.page_timeout,
@@ -2424,6 +2426,7 @@ def to_dict(self):
             "override_navigator": self.override_navigator,
             "magic": self.magic,
             "adjust_viewport_to_content": self.adjust_viewport_to_content,
+            "virtual_scroll_config": self.virtual_scroll_config,
             "screenshot": self.screenshot,
             "screenshot_wait_for": self.screenshot_wait_for,
             "screenshot_height_threshold": self.screenshot_height_threshold,
@@ -2461,6 +2464,7 @@ def to_dict(self):
             "match_mode": self.match_mode,
             "experimental": self.experimental,
             "max_retries": self.max_retries,
+            "fallback_fetch_function": self.fallback_fetch_function,
         }
 
     def clone(self, **kwargs):
diff --git a/tests/unit/test_run_config_clone_fields.py b/tests/unit/test_run_config_clone_fields.py
new file mode 100644
index 000000000..fe583c4fe
--- /dev/null
+++ b/tests/unit/test_run_config_clone_fields.py
@@ -0,0 +1,38 @@
+"""CrawlerRunConfig.clone() must carry over fields that to_dict() used to drop."""
+
+from crawl4ai import CrawlerRunConfig
+from crawl4ai.async_configs import VirtualScrollConfig
+
+
+async def _fallback(url: str) -> str:
+    return "<html><body>fallback</body></html>"
+
+
+def test_clone_preserves_fallback_fetch_function():
+    config = CrawlerRunConfig(fallback_fetch_function=_fallback)
+
+    cloned = config.clone(deep_crawl_strategy=None, stream=True)
+
+    assert cloned.fallback_fetch_function is _fallback
+    assert cloned.stream is True
+
+
+def test_clone_can_override_fallback_fetch_function():
+    config = CrawlerRunConfig(fallback_fetch_function=_fallback)
+
+    assert config.clone(fallback_fetch_function=None).fallback_fetch_function is None
+
+
+def test_clone_preserves_cache_validation_and_virtual_scroll():
+    scroll = VirtualScrollConfig(container_selector="#feed")
+    config = CrawlerRunConfig(
+        check_cache_freshness=True,
+        cache_validation_timeout=3.5,
+        virtual_scroll_config=scroll,
+    )
+
+    cloned = config.clone(stream=True)
+
+    assert cloned.check_cache_freshness is True
+    assert cloned.cache_validation_timeout == 3.5
+    assert cloned.virtual_scroll_config is scroll