diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index 15789be1e..d3eb96035 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -2398,6 +2398,8 @@ def to_dict(self): "disable_cache": self.disable_cache, "no_cache_read": self.no_cache_read, "no_cache_write": self.no_cache_write, + "check_cache_freshness": self.check_cache_freshness, + "cache_validation_timeout": self.cache_validation_timeout, "shared_data": self.shared_data, "wait_until": self.wait_until, "page_timeout": self.page_timeout, @@ -2424,6 +2426,7 @@ def to_dict(self): "override_navigator": self.override_navigator, "magic": self.magic, "adjust_viewport_to_content": self.adjust_viewport_to_content, + "virtual_scroll_config": self.virtual_scroll_config, "screenshot": self.screenshot, "screenshot_wait_for": self.screenshot_wait_for, "screenshot_height_threshold": self.screenshot_height_threshold, @@ -2461,6 +2464,7 @@ def to_dict(self): "match_mode": self.match_mode, "experimental": self.experimental, "max_retries": self.max_retries, + "fallback_fetch_function": self.fallback_fetch_function, } def clone(self, **kwargs): diff --git a/tests/unit/test_run_config_clone_fields.py b/tests/unit/test_run_config_clone_fields.py new file mode 100644 index 000000000..fe583c4fe --- /dev/null +++ b/tests/unit/test_run_config_clone_fields.py @@ -0,0 +1,38 @@ +"""CrawlerRunConfig.clone() must carry over fields that to_dict() used to drop.""" + +from crawl4ai import CrawlerRunConfig +from crawl4ai.async_configs import VirtualScrollConfig + + +async def _fallback(url: str) -> str: + return "fallback" + + +def test_clone_preserves_fallback_fetch_function(): + config = CrawlerRunConfig(fallback_fetch_function=_fallback) + + cloned = config.clone(deep_crawl_strategy=None, stream=True) + + assert cloned.fallback_fetch_function is _fallback + assert cloned.stream is True + + +def test_clone_can_override_fallback_fetch_function(): + config = CrawlerRunConfig(fallback_fetch_function=_fallback) + + assert config.clone(fallback_fetch_function=None).fallback_fetch_function is None + + +def test_clone_preserves_cache_validation_and_virtual_scroll(): + scroll = VirtualScrollConfig(container_selector="#feed") + config = CrawlerRunConfig( + check_cache_freshness=True, + cache_validation_timeout=3.5, + virtual_scroll_config=scroll, + ) + + cloned = config.clone(stream=True) + + assert cloned.check_cache_freshness is True + assert cloned.cache_validation_timeout == 3.5 + assert cloned.virtual_scroll_config is scroll