From 5789ccb95a880bccc431e5ef569b9ec83b510894 Mon Sep 17 00:00:00 2001 From: JackyHsu-X Date: Thu, 15 Oct 2026 23:42:18 +0300 Subject: [PATCH] Allow fallback fetch after a failed attempt without retries --- crawl4ai/async_webcrawler.py | 14 ++- docs/md_v2/advanced/anti-bot-and-fallback.md | 2 + .../test_fallback_fetch_without_retries.py | 113 ++++++++++++++++++ 3 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 tests/unit/test_fallback_fetch_without_retries.py diff --git a/crawl4ai/async_webcrawler.py b/crawl4ai/async_webcrawler.py index 110773ca1..7dadca3fa 100644 --- a/crawl4ai/async_webcrawler.py +++ b/crawl4ai/async_webcrawler.py @@ -543,10 +543,16 @@ async def arun( tag="ANTIBOT", ) _block_reason = str(_crawl_err) - # If this is the only proxy and only attempt, re-raise - # so the caller gets the real error (not a silent swallow). - # But if there are more proxies or retries to try, continue. - if len(_proxy_list) <= 1 and _max_attempts <= 1: + # Preserve the error when neither another attempt + # nor a fallback is available. + if ( + len(_proxy_list) <= 1 + and _max_attempts <= 1 + and ( + _is_raw_url + or not config.fallback_fetch_function + ) + ): raise # Restore original proxy_config diff --git a/docs/md_v2/advanced/anti-bot-and-fallback.md b/docs/md_v2/advanced/anti-bot-and-fallback.md index 68ed03de8..b02292f37 100644 --- a/docs/md_v2/advanced/anti-bot-and-fallback.md +++ b/docs/md_v2/advanced/anti-bot-and-fallback.md @@ -148,6 +148,8 @@ With this setup, each round tries the datacenter proxy first, then the residenti When all browser-based attempts fail, call a custom async function as a last resort. This function receives the URL and must return raw HTML as a string. The returned HTML is processed through the normal pipeline (markdown generation, extraction, etc.). +The fallback also runs after a fetch exception when `max_retries=0`; no extra browser retry is required. Caller-provided `raw:` HTML does not invoke the fetch fallback. + This is useful when you have access to a scraping API, a pre-fetched cache, or any other source of HTML. ```python diff --git a/tests/unit/test_fallback_fetch_without_retries.py b/tests/unit/test_fallback_fetch_without_retries.py new file mode 100644 index 000000000..1b7f0222c --- /dev/null +++ b/tests/unit/test_fallback_fetch_without_retries.py @@ -0,0 +1,113 @@ +"""A configured fallback must run after the only fetch attempt fails.""" + +from unittest.mock import AsyncMock, Mock + +import pytest + +from crawl4ai import AsyncWebCrawler, CacheMode, CrawlerRunConfig +from crawl4ai.async_crawler_strategy import AsyncCrawlerStrategy +from crawl4ai.models import AsyncCrawlResponse + +URL = "https://example.com/" +HTML = "

" + "Recovered article content. " * 50 + "

" + + +@pytest.fixture +def crawler(tmp_path): + strategy = Mock(spec=AsyncCrawlerStrategy) + strategy.crawl = AsyncMock(side_effect=RuntimeError("fetch failed")) + instance = AsyncWebCrawler(crawler_strategy=strategy, base_directory=str(tmp_path)) + instance.ready = True # The fetch is mocked; no browser needs to start. + return instance, strategy + + +def config(**kwargs): + return CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False, **kwargs) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("max_retries", [0, 1]) +@pytest.mark.parametrize("failure", ["exception", "blocked"]) +async def test_fallback_recovers_after_attempts_fail(crawler, max_retries, failure): + instance, strategy = crawler + callback = AsyncMock(return_value=HTML) + if failure == "blocked": + strategy.crawl.side_effect = None + strategy.crawl.return_value = AsyncCrawlResponse( + html="Access denied", + status_code=403, + response_headers={}, + ) + + result = await instance.arun( + URL, config=config(max_retries=max_retries, fallback_fetch_function=callback) + ) + + assert result.success, result.error_message + assert "Recovered article content." in result.markdown.raw_markdown + assert result.crawl_stats["resolved_by"] == "fallback_fetch" + assert result.crawl_stats["fallback_fetch_used"] is True + assert result.crawl_stats["attempts"] == max_retries + 1 + assert strategy.crawl.await_count == max_retries + 1 + callback.assert_awaited_once_with(URL) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("outcome", ["empty", "exception"]) +async def test_unsuccessful_fallback_keeps_the_fetch_failed(crawler, outcome): + instance, _ = crawler + callback = AsyncMock(return_value="") + if outcome == "exception": + callback.side_effect = RuntimeError("fallback failed") + + result = await instance.arun(URL, config=config(fallback_fetch_function=callback)) + + assert not result.success + assert "fetch failed" in result.error_message + callback.assert_awaited_once_with(URL) + assert result.crawl_stats["fallback_fetch_used"] is True + assert result.crawl_stats["resolved_by"] is None + + +@pytest.mark.asyncio +async def test_no_fallback_preserves_the_original_fetch_error(crawler): + instance, strategy = crawler + + result = await instance.arun(URL, config=config()) + + assert not result.success + assert "fetch failed" in result.error_message + strategy.crawl.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_raw_html_does_not_use_a_fetch_fallback(crawler): + instance, strategy = crawler + callback = AsyncMock(return_value=HTML) + + result = await instance.arun( + "raw:provided content", + config=config(fallback_fetch_function=callback), + ) + + assert not result.success + assert "fetch failed" in result.error_message + strategy.crawl.assert_awaited_once() + callback.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_successful_fetch_does_not_use_a_fallback(crawler): + instance, strategy = crawler + strategy.crawl.side_effect = None + strategy.crawl.return_value = AsyncCrawlResponse( + html=HTML, status_code=200, response_headers={} + ) + callback = AsyncMock(return_value=HTML) + + result = await instance.arun(URL, config=config(fallback_fetch_function=callback)) + + assert result.success, result.error_message + assert result.crawl_stats["resolved_by"] == "direct" + strategy.crawl.assert_awaited_once() + callback.assert_not_awaited()