Fix: preserve screenshot and raw-HTML settings in config clones

crawl4ai/async_configs.py

@@ -2427,6 +2427,7 @@ def to_dict(self):
             "screenshot": self.screenshot,
             "screenshot_wait_for": self.screenshot_wait_for,
             "screenshot_height_threshold": self.screenshot_height_threshold,
+            "force_viewport_screenshot": self.force_viewport_screenshot,
             "pdf": self.pdf,
             "capture_mhtml": self.capture_mhtml,
             "image_description_min_word_threshold": self.image_description_min_word_threshold,
@@ -2457,6 +2458,7 @@ def to_dict(self):
             "deep_crawl_strategy": self.deep_crawl_strategy,
             "link_preview_config": self.link_preview_config.to_dict() if self.link_preview_config else None,
             "url": self.url,
+            "base_url": self.base_url,
             "url_matcher": self.url_matcher,
             "match_mode": self.match_mode,
             "experimental": self.experimental,

tests/unit/test_base_url_config.py

@@ -0,0 +1,90 @@
+"""Raw-HTML base URLs must survive configuration cloning."""
+
+from unittest.mock import Mock
+
+import pytest
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_crawler_strategy import AsyncCrawlerStrategy
+
+BASE_URL = "https://example.test/docs/"
+
+
+@pytest.mark.parametrize("base_url", [None, BASE_URL])
+def test_base_url_from_kwargs(base_url):
+    config = CrawlerRunConfig.from_kwargs({"base_url": base_url})
+
+    assert config.base_url == base_url
+
+
+@pytest.mark.parametrize("base_url", [None, BASE_URL])
+def test_base_url_dict_roundtrip(base_url):
+    config = CrawlerRunConfig(base_url=base_url)
+    config_dict = config.to_dict()
+
+    assert config_dict["base_url"] == base_url
+    assert CrawlerRunConfig.from_kwargs(config_dict).base_url == base_url
+
+
+@pytest.mark.parametrize("base_url", [None, BASE_URL])
+@pytest.mark.parametrize("updates", [{}, {"stream": True}])
+def test_clone_preserves_base_url(base_url, updates):
+    config = CrawlerRunConfig(base_url=base_url)
+    cloned = config.clone(**updates)
+
+    assert cloned is not config
+    assert cloned.base_url == base_url
+    assert cloned.stream is updates.get("stream", False)
+    assert config.base_url == base_url
+    assert config.stream is False
+
+
+@pytest.mark.parametrize("base_url", [None, "https://other.test/content/"])
+def test_clone_can_update_base_url(base_url):
+    config = CrawlerRunConfig(base_url=BASE_URL)
+    cloned = config.clone(base_url=base_url)
+
+    assert cloned.base_url == base_url
+    assert config.base_url == BASE_URL
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("prefetch", [True, False])
+async def test_cloned_config_preserves_raw_html_link_resolution(tmp_path, prefetch):
+    html = '<html><body><p>Read the <a href="guide.html">guide</a>.</p></body></html>'
+    expected = "https://example.test/docs/guide.html"
+    # Process supplied HTML directly without starting or calling the strategy.
+    strategy = Mock(spec=AsyncCrawlerStrategy)
+    crawler = AsyncWebCrawler(
+        crawler_strategy=strategy,
+        config=BrowserConfig(verbose=False),
+        base_directory=str(tmp_path),
+    )
+    config = CrawlerRunConfig(
+        base_url=BASE_URL, prefetch=prefetch, word_count_threshold=0
+    )
+
+    async def process(run_config):
+        return await crawler.aprocess_html(
+            url="raw:" + html,
+            html=html,
+            extracted_content=None,
+            config=run_config,
+            screenshot_data=None,
+            pdf_data=None,
+            verbose=False,
+        )
+
+    control = await process(config)
+    result = await process(config.clone(stream=True))
+
+    assert strategy.mock_calls == []
+    for processed in (control, result):
+        assert processed.success
+        if prefetch:
+            hrefs = [
+                link["href"] for links in processed.links.values() for link in links
+            ]
+            assert expected in hrefs
+        else:
+            assert expected in processed.markdown.raw_markdown

tests/unit/test_force_viewport_screenshot_config.py

@@ -0,0 +1,84 @@
+"""Viewport-only screenshot settings must survive configuration cloning."""
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from crawl4ai import CrawlerRunConfig
+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+
+
+def test_force_viewport_screenshot_defaults_to_false():
+    config = CrawlerRunConfig()
+
+    assert config.force_viewport_screenshot is False
+    assert config.clone().force_viewport_screenshot is False
+
+
+@pytest.mark.parametrize("force_viewport", [False, True])
+def test_force_viewport_screenshot_to_dict(force_viewport):
+    config = CrawlerRunConfig(force_viewport_screenshot=force_viewport)
+
+    assert config.to_dict()["force_viewport_screenshot"] is force_viewport
+
+
+@pytest.mark.parametrize("force_viewport", [False, True])
+def test_force_viewport_screenshot_from_kwargs(force_viewport):
+    config = CrawlerRunConfig.from_kwargs({"force_viewport_screenshot": force_viewport})
+
+    assert config.force_viewport_screenshot is force_viewport
+
+
+@pytest.mark.parametrize("force_viewport", [False, True])
+def test_force_viewport_screenshot_dict_roundtrip(force_viewport):
+    config = CrawlerRunConfig(force_viewport_screenshot=force_viewport)
+    restored = CrawlerRunConfig.from_kwargs(config.to_dict())
+
+    assert restored.force_viewport_screenshot is force_viewport
+
+
+@pytest.mark.parametrize("updates", [{}, {"stream": True}])
+def test_clone_preserves_force_viewport_screenshot(updates):
+    config = CrawlerRunConfig(force_viewport_screenshot=True)
+    cloned = config.clone(**updates)
+
+    assert cloned is not config
+    assert cloned.force_viewport_screenshot is True
+    assert cloned.stream is updates.get("stream", False)
+    assert config.force_viewport_screenshot is True
+    assert config.stream is False
+
+
+def test_clone_can_disable_force_viewport_screenshot():
+    config = CrawlerRunConfig(force_viewport_screenshot=True)
+    cloned = config.clone(force_viewport_screenshot=False)
+
+    assert cloned.force_viewport_screenshot is False
+    assert config.force_viewport_screenshot is True
+
+
+@pytest.mark.asyncio
+async def test_cloned_config_keeps_viewport_screenshot_when_scanning_full_page():
+    config = CrawlerRunConfig(
+        screenshot=True,
+        scan_full_page=True,
+        force_viewport_screenshot=True,
+    ).clone(stream=True)
+    strategy = object.__new__(AsyncPlaywrightCrawlerStrategy)
+    strategy.take_screenshot_naive = AsyncMock(return_value="viewport")
+    strategy.take_screenshot_scroller = AsyncMock(return_value="full-page")
+    strategy.page_need_scroll = AsyncMock(return_value=True)
+    page = MagicMock()
+
+    result = await strategy.take_screenshot(
+        page,
+        screenshot_height_threshold=config.screenshot_height_threshold,
+        force_viewport_screenshot=config.force_viewport_screenshot,
+        scan_full_page=config.scan_full_page,
+        scroll_delay=config.scroll_delay,
+    )
+
+    assert result == "viewport"
+    strategy.take_screenshot_naive.assert_awaited_once_with(page)
+    strategy.take_screenshot_scroller.assert_not_awaited()
+    strategy.page_need_scroll.assert_not_awaited()