From 280043ee0ec241b7d950f278df0056c104883af8 Mon Sep 17 00:00:00 2001 From: "Gareth.Y" <91742523+linjiaye929@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:49:36 -0700 Subject: [PATCH 1/2] Fix: preserve viewport screenshot preference in config clones --- crawl4ai/async_configs.py | 1 + .../test_force_viewport_screenshot_config.py | 84 +++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 tests/unit/test_force_viewport_screenshot_config.py diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index 15789be1e..18f616a34 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -2427,6 +2427,7 @@ def to_dict(self): "screenshot": self.screenshot, "screenshot_wait_for": self.screenshot_wait_for, "screenshot_height_threshold": self.screenshot_height_threshold, + "force_viewport_screenshot": self.force_viewport_screenshot, "pdf": self.pdf, "capture_mhtml": self.capture_mhtml, "image_description_min_word_threshold": self.image_description_min_word_threshold, diff --git a/tests/unit/test_force_viewport_screenshot_config.py b/tests/unit/test_force_viewport_screenshot_config.py new file mode 100644 index 000000000..9ed7b4c0e --- /dev/null +++ b/tests/unit/test_force_viewport_screenshot_config.py @@ -0,0 +1,84 @@ +"""Viewport-only screenshot settings must survive configuration cloning.""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from crawl4ai import CrawlerRunConfig +from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy + + +def test_force_viewport_screenshot_defaults_to_false(): + config = CrawlerRunConfig() + + assert config.force_viewport_screenshot is False + assert config.clone().force_viewport_screenshot is False + + +@pytest.mark.parametrize("force_viewport", [False, True]) +def test_force_viewport_screenshot_to_dict(force_viewport): + config = CrawlerRunConfig(force_viewport_screenshot=force_viewport) + + assert config.to_dict()["force_viewport_screenshot"] is force_viewport + + +@pytest.mark.parametrize("force_viewport", [False, True]) +def test_force_viewport_screenshot_from_kwargs(force_viewport): + config = CrawlerRunConfig.from_kwargs({"force_viewport_screenshot": force_viewport}) + + assert config.force_viewport_screenshot is force_viewport + + +@pytest.mark.parametrize("force_viewport", [False, True]) +def test_force_viewport_screenshot_dict_roundtrip(force_viewport): + config = CrawlerRunConfig(force_viewport_screenshot=force_viewport) + restored = CrawlerRunConfig.from_kwargs(config.to_dict()) + + assert restored.force_viewport_screenshot is force_viewport + + +@pytest.mark.parametrize("updates", [{}, {"stream": True}]) +def test_clone_preserves_force_viewport_screenshot(updates): + config = CrawlerRunConfig(force_viewport_screenshot=True) + cloned = config.clone(**updates) + + assert cloned is not config + assert cloned.force_viewport_screenshot is True + assert cloned.stream is updates.get("stream", False) + assert config.force_viewport_screenshot is True + assert config.stream is False + + +def test_clone_can_disable_force_viewport_screenshot(): + config = CrawlerRunConfig(force_viewport_screenshot=True) + cloned = config.clone(force_viewport_screenshot=False) + + assert cloned.force_viewport_screenshot is False + assert config.force_viewport_screenshot is True + + +@pytest.mark.asyncio +async def test_cloned_config_keeps_viewport_screenshot_when_scanning_full_page(): + config = CrawlerRunConfig( + screenshot=True, + scan_full_page=True, + force_viewport_screenshot=True, + ).clone(stream=True) + strategy = object.__new__(AsyncPlaywrightCrawlerStrategy) + strategy.take_screenshot_naive = AsyncMock(return_value="viewport") + strategy.take_screenshot_scroller = AsyncMock(return_value="full-page") + strategy.page_need_scroll = AsyncMock(return_value=True) + page = MagicMock() + + result = await strategy.take_screenshot( + page, + screenshot_height_threshold=config.screenshot_height_threshold, + force_viewport_screenshot=config.force_viewport_screenshot, + scan_full_page=config.scan_full_page, + scroll_delay=config.scroll_delay, + ) + + assert result == "viewport" + strategy.take_screenshot_naive.assert_awaited_once_with(page) + strategy.take_screenshot_scroller.assert_not_awaited() + strategy.page_need_scroll.assert_not_awaited() From 36c564821bb09fe294bf10d63f49affcd433eb9e Mon Sep 17 00:00:00 2001 From: "Gareth.Y" <91742523+linjiaye929@users.noreply.github.com> Date: Thu, 15 Oct 2026 04:59:20 -0700 Subject: [PATCH 2/2] Fix preserve raw HTML base URL in config clones --- crawl4ai/async_configs.py | 1 + tests/unit/test_base_url_config.py | 90 ++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 tests/unit/test_base_url_config.py diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index 18f616a34..a85495d1f 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -2458,6 +2458,7 @@ def to_dict(self): "deep_crawl_strategy": self.deep_crawl_strategy, "link_preview_config": self.link_preview_config.to_dict() if self.link_preview_config else None, "url": self.url, + "base_url": self.base_url, "url_matcher": self.url_matcher, "match_mode": self.match_mode, "experimental": self.experimental, diff --git a/tests/unit/test_base_url_config.py b/tests/unit/test_base_url_config.py new file mode 100644 index 000000000..0cccd088a --- /dev/null +++ b/tests/unit/test_base_url_config.py @@ -0,0 +1,90 @@ +"""Raw-HTML base URLs must survive configuration cloning.""" + +from unittest.mock import Mock + +import pytest + +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig +from crawl4ai.async_crawler_strategy import AsyncCrawlerStrategy + +BASE_URL = "https://example.test/docs/" + + +@pytest.mark.parametrize("base_url", [None, BASE_URL]) +def test_base_url_from_kwargs(base_url): + config = CrawlerRunConfig.from_kwargs({"base_url": base_url}) + + assert config.base_url == base_url + + +@pytest.mark.parametrize("base_url", [None, BASE_URL]) +def test_base_url_dict_roundtrip(base_url): + config = CrawlerRunConfig(base_url=base_url) + config_dict = config.to_dict() + + assert config_dict["base_url"] == base_url + assert CrawlerRunConfig.from_kwargs(config_dict).base_url == base_url + + +@pytest.mark.parametrize("base_url", [None, BASE_URL]) +@pytest.mark.parametrize("updates", [{}, {"stream": True}]) +def test_clone_preserves_base_url(base_url, updates): + config = CrawlerRunConfig(base_url=base_url) + cloned = config.clone(**updates) + + assert cloned is not config + assert cloned.base_url == base_url + assert cloned.stream is updates.get("stream", False) + assert config.base_url == base_url + assert config.stream is False + + +@pytest.mark.parametrize("base_url", [None, "https://other.test/content/"]) +def test_clone_can_update_base_url(base_url): + config = CrawlerRunConfig(base_url=BASE_URL) + cloned = config.clone(base_url=base_url) + + assert cloned.base_url == base_url + assert config.base_url == BASE_URL + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prefetch", [True, False]) +async def test_cloned_config_preserves_raw_html_link_resolution(tmp_path, prefetch): + html = '

Read the guide.

' + expected = "https://example.test/docs/guide.html" + # Process supplied HTML directly without starting or calling the strategy. + strategy = Mock(spec=AsyncCrawlerStrategy) + crawler = AsyncWebCrawler( + crawler_strategy=strategy, + config=BrowserConfig(verbose=False), + base_directory=str(tmp_path), + ) + config = CrawlerRunConfig( + base_url=BASE_URL, prefetch=prefetch, word_count_threshold=0 + ) + + async def process(run_config): + return await crawler.aprocess_html( + url="raw:" + html, + html=html, + extracted_content=None, + config=run_config, + screenshot_data=None, + pdf_data=None, + verbose=False, + ) + + control = await process(config) + result = await process(config.clone(stream=True)) + + assert strategy.mock_calls == [] + for processed in (control, result): + assert processed.success + if prefetch: + hrefs = [ + link["href"] for links in processed.links.values() for link in links + ] + assert expected in hrefs + else: + assert expected in processed.markdown.raw_markdown