From b9feb3346bfec8348cd491d6d6f722679e5b36e8 Mon Sep 17 00:00:00 2001 From: "aiste.grigaliunaite@decodo.com" Date: Fri, 2 Oct 2026 13:19:53 +0300 Subject: [PATCH 1/4] fix(decodo): return markdown, parse search results, fix README and links Co-Authored-By: Claude Sonnet 5.5 --- libs/partners/decodo/README.md | 67 +++++++++++-------- .../langchain_decodo/document_loaders.py | 6 +- .../partners/decodo/langchain_decodo/tools.py | 24 ++++++- libs/partners/decodo/pyproject.toml | 4 +- .../tests/unit_tests/test_document_loaders.py | 20 ++++++ .../decodo/tests/unit_tests/test_tools.py | 23 +++++++ 6 files changed, 112 insertions(+), 32 deletions(-) create mode 100644 libs/partners/decodo/tests/unit_tests/test_document_loaders.py diff --git a/libs/partners/decodo/README.md b/libs/partners/decodo/README.md index 41cf74672..edf029d78 100644 --- a/libs/partners/decodo/README.md +++ b/libs/partners/decodo/README.md @@ -105,53 +105,64 @@ Each `Document` has: ## LangChain agent example +```bash +pip install langchain langchain-openai langchain-decodo +``` + ```python -from langchain import hub -from langchain.agents import AgentExecutor, create_react_agent -from langchain_openai import ChatOpenAI -from langchain_decodo import DecodoWebScrapeTool, DecodoSearchTool - -tools = [DecodoWebScrapeTool(), DecodoSearchTool()] -llm = ChatOpenAI(model="gpt-4o-mini", temperature=0) -prompt = hub.pull("hwchase17/react") - -agent = create_react_agent(llm=llm, tools=tools, prompt=prompt) -executor = AgentExecutor(agent=agent, tools=tools, verbose=True) - -result = executor.invoke({ - "input": "What is the latest stable version of Python? Check python.org." -}) -print(result["output"]) +from langchain.agents import create_agent +from langchain_decodo import DecodoSearchTool, DecodoWebScrapeTool + +agent = create_agent( + model="openai:gpt-4o-mini", + tools=[DecodoWebScrapeTool(), DecodoSearchTool()], +) + +result = agent.invoke( + { + "messages": [ + { + "role": "user", + "content": "What is the latest stable version of Python? Check python.org.", + } + ] + } +) +print(result["messages"][-1].content) ``` ## RAG pipeline example +```bash +pip install langchain-openai langchain-text-splitters langchain-decodo +``` + ```python -from langchain.text_splitter import RecursiveCharacterTextSplitter -from langchain_community.vectorstores import FAISS +from langchain_core.vectorstores import InMemoryVectorStore from langchain_openai import ChatOpenAI, OpenAIEmbeddings -from langchain.chains import RetrievalQA +from langchain_text_splitters import RecursiveCharacterTextSplitter from langchain_decodo import DecodoLoader -loader = DecodoLoader(urls=["https://python.org/about/"]) -docs = loader.load() +docs = DecodoLoader(urls=["https://python.org/about/"]).load() splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200) chunks = splitter.split_documents(docs) -store = FAISS.from_documents(chunks, OpenAIEmbeddings()) -chain = RetrievalQA.from_chain_type( - llm=ChatOpenAI(model="gpt-4o-mini"), - retriever=store.as_retriever(search_kwargs={"k": 4}), +store = InMemoryVectorStore.from_documents(chunks, OpenAIEmbeddings()) +context = "\n\n".join( + doc.page_content for doc in store.similarity_search("What is Python used for?", k=4) ) -result = chain.invoke({"query": "What is Python used for?"}) -print(result["result"]) +llm = ChatOpenAI(model="gpt-4o-mini") +answer = llm.invoke( + f"Answer using only this context:\n\n{context}\n\nQuestion: What is Python used for?" +) +print(answer.content) ``` ## Links - [Decodo website](https://decodo.com) -- [Decodo API documentation](https://developers.decodo.com) +- [Decodo API documentation](https://help.decodo.com) - [Decodo Dashboard](https://app.decodo.com) - [LangChain documentation](https://python.langchain.com) diff --git a/libs/partners/decodo/langchain_decodo/document_loaders.py b/libs/partners/decodo/langchain_decodo/document_loaders.py index b9b608ad8..0bb0c1a97 100644 --- a/libs/partners/decodo/langchain_decodo/document_loaders.py +++ b/libs/partners/decodo/langchain_decodo/document_loaders.py @@ -103,7 +103,11 @@ def _scrape_url( RuntimeError: On timeout, network error, or non-2xx HTTP response. """ endpoint = f"{_API_BASE}{_scrape_path(auth_mode)}" - payload: dict[str, Any] = {"target": "universal", "url": url} + payload: dict[str, Any] = { + "target": "universal", + "url": url, + "markdown": True, + } try: response = httpx.post( diff --git a/libs/partners/decodo/langchain_decodo/tools.py b/libs/partners/decodo/langchain_decodo/tools.py index 488f0908c..14cc24e52 100644 --- a/libs/partners/decodo/langchain_decodo/tools.py +++ b/libs/partners/decodo/langchain_decodo/tools.py @@ -289,7 +289,11 @@ def _run( "field or the ``DECODO_API_TOKEN`` environment variable." ) - payload: dict[str, Any] = {"target": "universal", "url": url} + payload: dict[str, Any] = { + "target": "universal", + "url": url, + "markdown": True, + } response = _do_scrape(token, self.base_url, payload, auth_mode=self.auth_mode) content = _extract_content(response) return content if content else "(No content returned by Decodo API)" @@ -425,6 +429,7 @@ def _run( "target": target, "query": effective_query, "limit": num_results, + "parse": True, } response = _do_scrape( @@ -432,6 +437,23 @@ def _run( ) results = response.get("results", []) + # The API answers HTTP 200 with a per-result failure status (e.g. 613 + # "not able to scrape the target"); surface it so agents can retry + # instead of reading it as "no results". + failed = [ + entry + for entry in results + if isinstance(entry.get("status_code"), int) + and entry["status_code"] >= 400 + ] + if results and len(failed) == len(results): + first = failed[0] + detail = first.get("content") if isinstance(first.get("content"), str) else "" + raise RuntimeError( + f"Decodo search failed with status {first['status_code']}" + + (f": {detail}" if detail else "") + ) + serialisable = [] for entry in results: content = entry.get("content", "") diff --git a/libs/partners/decodo/pyproject.toml b/libs/partners/decodo/pyproject.toml index ee18dbf37..31422b47c 100644 --- a/libs/partners/decodo/pyproject.toml +++ b/libs/partners/decodo/pyproject.toml @@ -41,8 +41,8 @@ test_integration = [] [project.urls] Homepage = "https://decodo.com" -Documentation = "https://developers.decodo.com" -Repository = "https://github.com/langchain-ai/langchain/tree/master/libs/partners/decodo" +Documentation = "https://help.decodo.com" +Repository = "https://github.com/Decodo/langchain/tree/master/libs/partners/decodo" [tool.hatch.build.targets.wheel] packages = ["langchain_decodo"] diff --git a/libs/partners/decodo/tests/unit_tests/test_document_loaders.py b/libs/partners/decodo/tests/unit_tests/test_document_loaders.py new file mode 100644 index 000000000..88f8d2e31 --- /dev/null +++ b/libs/partners/decodo/tests/unit_tests/test_document_loaders.py @@ -0,0 +1,20 @@ +"""Unit tests for DecodoLoader.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from langchain_decodo import DecodoLoader + + +@patch("langchain_decodo.document_loaders.httpx.post") +def test_loader_requests_markdown(mock_post: MagicMock) -> None: + response = MagicMock() + response.is_success = True + response.json.return_value = {"results": [{"content": "# Title", "status_code": 200}]} + mock_post.return_value = response + + docs = DecodoLoader(urls="https://example.com", api_token="tok").load() + + assert mock_post.call_args[1]["json"]["markdown"] is True + assert docs[0].page_content == "# Title" diff --git a/libs/partners/decodo/tests/unit_tests/test_tools.py b/libs/partners/decodo/tests/unit_tests/test_tools.py index b073ce07d..9dd11f3a8 100644 --- a/libs/partners/decodo/tests/unit_tests/test_tools.py +++ b/libs/partners/decodo/tests/unit_tests/test_tools.py @@ -112,6 +112,7 @@ def test_request_includes_correct_payload(self, mock_post: MagicMock) -> None: payload = kwargs["json"] assert payload["target"] == "universal" assert payload["url"] == "https://target.com/page" + assert payload["markdown"] is True @patch("langchain_decodo.tools.httpx.post") def test_authorization_header(self, mock_post: MagicMock) -> None: @@ -261,6 +262,28 @@ def test_num_results_forwarded(self, mock_post: MagicMock) -> None: _, kwargs = mock_post.call_args assert kwargs["json"]["limit"] == 5 + @patch("langchain_decodo.tools.httpx.post") + def test_requests_parsed_results(self, mock_post: MagicMock) -> None: + mock_post.return_value = _make_mock_httpx_response(_make_api_response("{}")) + DecodoSearchTool(decodo_api_token=SecretStr("tok"))._run("q") + assert mock_post.call_args[1]["json"]["parse"] is True + + @patch("langchain_decodo.tools.httpx.post") + def test_failed_scrape_status_raises(self, mock_post: MagicMock) -> None: + mock_post.return_value = _make_mock_httpx_response( + { + "results": [ + { + "content": "We were not able to scrape the target", + "status_code": 613, + } + ] + } + ) + tool = DecodoSearchTool(decodo_api_token=SecretStr("tok")) + with pytest.raises(RuntimeError, match="613"): + tool._run("q") + @patch("langchain_decodo.tools.httpx.post") def test_empty_results_returns_empty_json_list(self, mock_post: MagicMock) -> None: mock_post.return_value = _make_mock_httpx_response({"results": []}) From 6643f71765ffe95c885d0659b50fa53caedb8a74 Mon Sep 17 00:00:00 2001 From: "aiste.grigaliunaite@decodo.com" Date: Fri, 2 Oct 2026 13:25:30 +0300 Subject: [PATCH 2/4] fix(decodo): align Makefile and tests with monorepo CI Co-Authored-By: Claude Sonnet 5.5 --- libs/partners/decodo/Makefile | 25 ++++++++++++------- .../langchain_decodo/document_loaders.py | 11 +++----- .../partners/decodo/langchain_decodo/tools.py | 25 +++++++------------ libs/partners/decodo/pyproject.toml | 1 - .../tests/integration_tests/test_compile.py | 6 +++++ .../tests/integration_tests/test_tools.py | 17 +++++++------ .../decodo/tests/unit_tests/test_tools.py | 14 +++++------ 7 files changed, 50 insertions(+), 49 deletions(-) create mode 100644 libs/partners/decodo/tests/integration_tests/test_compile.py diff --git a/libs/partners/decodo/Makefile b/libs/partners/decodo/Makefile index 6af70cec5..87a15cc58 100644 --- a/libs/partners/decodo/Makefile +++ b/libs/partners/decodo/Makefile @@ -1,5 +1,7 @@ -.PHONY: all lint format test tests integration_tests help +.PHONY: all lint lint_package lint_tests format test tests integration_tests help + +PYTEST_EXTRA ?= all: help @@ -8,24 +10,29 @@ all: help ###################### lint format: PYTHON_FILES=. -lint: - ruff check $(PYTHON_FILES) - ruff format $(PYTHON_FILES) --diff - mypy $(PYTHON_FILES) +lint_package: PYTHON_FILES=langchain_decodo +lint_tests: PYTHON_FILES=tests +lint_tests: MYPY_CACHE=.mypy_cache_test +MYPY_CACHE=.mypy_cache + +lint lint_package lint_tests: + uv run --group lint --group typing ruff check $(PYTHON_FILES) + uv run --group lint --group typing ruff format $(PYTHON_FILES) --diff + mkdir -p $(MYPY_CACHE) && uv run --group lint --group typing mypy $(PYTHON_FILES) --cache-dir $(MYPY_CACHE) format: - ruff format $(PYTHON_FILES) - ruff check --fix $(PYTHON_FILES) + uv run --group lint ruff format $(PYTHON_FILES) + uv run --group lint ruff check --fix $(PYTHON_FILES) ###################### # TESTING ###################### tests test: - pytest tests/unit_tests + uv run --group test pytest $(PYTEST_EXTRA) tests/unit_tests integration_tests: - pytest tests/integration_tests + uv run --group test --group test_integration pytest tests/integration_tests ###################### # HELP diff --git a/libs/partners/decodo/langchain_decodo/document_loaders.py b/libs/partners/decodo/langchain_decodo/document_loaders.py index 0bb0c1a97..51ad982b0 100644 --- a/libs/partners/decodo/langchain_decodo/document_loaders.py +++ b/libs/partners/decodo/langchain_decodo/document_loaders.py @@ -37,7 +37,8 @@ import json import os -from typing import Any, Iterator, Literal +from collections.abc import Iterator +from typing import Any, Literal import httpx from langchain_core.document_loaders import BaseLoader @@ -117,13 +118,9 @@ def _scrape_url( timeout=timeout, ) except httpx.TimeoutException as exc: - raise RuntimeError( - f"Decodo API request for '{url}' timed out after {timeout}s" - ) from exc + raise RuntimeError(f"Decodo API request for '{url}' timed out after {timeout}s") from exc except httpx.RequestError as exc: - raise RuntimeError( - f"Decodo API network error while fetching '{url}': {exc}" - ) from exc + raise RuntimeError(f"Decodo API network error while fetching '{url}': {exc}") from exc if not response.is_success: try: diff --git a/libs/partners/decodo/langchain_decodo/tools.py b/libs/partners/decodo/langchain_decodo/tools.py index 14cc24e52..0675b5e67 100644 --- a/libs/partners/decodo/langchain_decodo/tools.py +++ b/libs/partners/decodo/langchain_decodo/tools.py @@ -21,7 +21,7 @@ import json import os -from typing import Any, Literal, Optional, Type +from typing import Any, Literal import httpx from langchain_core.callbacks import CallbackManagerForToolRun @@ -105,9 +105,7 @@ def _do_scrape( try: response = httpx.post(url, headers=headers, json=payload, timeout=timeout) except httpx.TimeoutException as exc: - raise RuntimeError( - f"Decodo API request timed out after {timeout}s: {exc}" - ) from exc + raise RuntimeError(f"Decodo API request timed out after {timeout}s: {exc}") from exc except httpx.RequestError as exc: raise RuntimeError(f"Decodo API network error: {exc}") from exc @@ -225,7 +223,7 @@ class DecodoWebScrapeTool(BaseTool): "automatically. Use this when you need the complete text of a specific URL. " "Input: a valid URL string (must include http:// or https://)." ) - args_schema: Type[BaseModel] = _WebScrapeInput + args_schema: type[BaseModel] = _WebScrapeInput decodo_api_token: SecretStr = Field( default=SecretStr(""), @@ -267,7 +265,7 @@ def validate_api_token(cls, values: dict[str, Any]) -> dict[str, Any]: def _run( self, url: str, - run_manager: Optional[CallbackManagerForToolRun] = None, + run_manager: CallbackManagerForToolRun | None = None, ) -> str: """Scrape the given URL and return its content. @@ -348,7 +346,7 @@ class DecodoSearchTool(BaseTool): "'num_results' (optional: integer 1-100; default 10). " "Use 'amazon' to search for products, 'reddit' for community discussions." ) - args_schema: Type[BaseModel] = _SearchInput + args_schema: type[BaseModel] = _SearchInput decodo_api_token: SecretStr = Field( default=SecretStr(""), @@ -392,7 +390,7 @@ def _run( query: str, engine: str = "google", num_results: int = 10, - run_manager: Optional[CallbackManagerForToolRun] = None, + run_manager: CallbackManagerForToolRun | None = None, ) -> str: """Execute a search and return results as a JSON string. @@ -421,9 +419,7 @@ def _run( target = _ENGINE_TARGET_MAP.get(engine, "google_search") # Prepend Reddit site filter when using the reddit pseudo-engine. - effective_query = ( - f"{_REDDIT_SITE_FILTER} {query}" if engine == "reddit" else query - ) + effective_query = f"{_REDDIT_SITE_FILTER} {query}" if engine == "reddit" else query payload: dict[str, Any] = { "target": target, @@ -432,9 +428,7 @@ def _run( "parse": True, } - response = _do_scrape( - token, _DEFAULT_BASE_URL, payload, auth_mode=self.auth_mode - ) + response = _do_scrape(token, _DEFAULT_BASE_URL, payload, auth_mode=self.auth_mode) results = response.get("results", []) # The API answers HTTP 200 with a per-result failure status (e.g. 613 @@ -443,8 +437,7 @@ def _run( failed = [ entry for entry in results - if isinstance(entry.get("status_code"), int) - and entry["status_code"] >= 400 + if isinstance(entry.get("status_code"), int) and entry["status_code"] >= 400 ] if results and len(failed) == len(results): first = failed[0] diff --git a/libs/partners/decodo/pyproject.toml b/libs/partners/decodo/pyproject.toml index 31422b47c..39dcc2878 100644 --- a/libs/partners/decodo/pyproject.toml +++ b/libs/partners/decodo/pyproject.toml @@ -55,7 +55,6 @@ target-version = "py310" select = ["E", "F", "I", "UP"] [tool.mypy] -python_version = "3.10" strict = true [tool.pytest.ini_options] diff --git a/libs/partners/decodo/tests/integration_tests/test_compile.py b/libs/partners/decodo/tests/integration_tests/test_compile.py new file mode 100644 index 000000000..264006671 --- /dev/null +++ b/libs/partners/decodo/tests/integration_tests/test_compile.py @@ -0,0 +1,6 @@ +import pytest + + +@pytest.mark.compile +def test_placeholder() -> None: + """Allows CI to verify that the integration tests import and compile.""" diff --git a/libs/partners/decodo/tests/integration_tests/test_tools.py b/libs/partners/decodo/tests/integration_tests/test_tools.py index 9e2d7b96a..073007d79 100644 --- a/libs/partners/decodo/tests/integration_tests/test_tools.py +++ b/libs/partners/decodo/tests/integration_tests/test_tools.py @@ -12,6 +12,7 @@ import json import os +from typing import Any import pytest @@ -23,21 +24,21 @@ @pytest.fixture() -def scrape_tool(): # type: ignore[return] +def scrape_tool() -> Any: from langchain_decodo.tools import DecodoWebScrapeTool return DecodoWebScrapeTool() @pytest.fixture() -def search_tool(): # type: ignore[return] +def search_tool() -> Any: from langchain_decodo.tools import DecodoSearchTool return DecodoSearchTool() @pytest.fixture() -def loader(): # type: ignore[return] +def loader() -> Any: from langchain_decodo import DecodoLoader return DecodoLoader(urls=["https://example.com"]) @@ -53,7 +54,7 @@ def test_scrape_example_dot_com(self, scrape_tool: object) -> None: from langchain_decodo.tools import DecodoWebScrapeTool assert isinstance(scrape_tool, DecodoWebScrapeTool) - result = scrape_tool._run("https://example.com") # type: ignore[union-attr] + result = scrape_tool._run("https://example.com") assert isinstance(result, str) assert len(result) > 0 # example.com always contains "Example Domain" @@ -63,7 +64,7 @@ def test_scrape_returns_string(self, scrape_tool: object) -> None: from langchain_decodo.tools import DecodoWebScrapeTool assert isinstance(scrape_tool, DecodoWebScrapeTool) - result = scrape_tool._run("https://httpbin.org/html") # type: ignore[union-attr] + result = scrape_tool._run("https://httpbin.org/html") assert isinstance(result, str) @@ -77,7 +78,7 @@ def test_google_search_returns_results(self, search_tool: object) -> None: from langchain_decodo.tools import DecodoSearchTool assert isinstance(search_tool, DecodoSearchTool) - raw = search_tool._run("Python programming language", engine="google", num_results=3) # type: ignore[union-attr] + raw = search_tool._run("Python programming language", engine="google", num_results=3) results = json.loads(raw) assert isinstance(results, list) assert len(results) > 0 @@ -89,7 +90,7 @@ def test_amazon_search_returns_results(self, search_tool: object) -> None: from langchain_decodo.tools import DecodoSearchTool assert isinstance(search_tool, DecodoSearchTool) - raw = search_tool._run("Python book", engine="amazon", num_results=3) # type: ignore[union-attr] + raw = search_tool._run("Python book", engine="amazon", num_results=3) results = json.loads(raw) assert isinstance(results, list) @@ -98,7 +99,7 @@ def test_reddit_search_prepends_filter(self, search_tool: object) -> None: assert isinstance(search_tool, DecodoSearchTool) # Should not raise; results may be empty depending on query - raw = search_tool._run("best Python libraries", engine="reddit", num_results=3) # type: ignore[union-attr] + raw = search_tool._run("best Python libraries", engine="reddit", num_results=3) results = json.loads(raw) assert isinstance(results, list) diff --git a/libs/partners/decodo/tests/unit_tests/test_tools.py b/libs/partners/decodo/tests/unit_tests/test_tools.py index 9dd11f3a8..d3db47dc3 100644 --- a/libs/partners/decodo/tests/unit_tests/test_tools.py +++ b/libs/partners/decodo/tests/unit_tests/test_tools.py @@ -14,12 +14,11 @@ from pydantic import SecretStr from langchain_decodo.tools import ( + _ENGINE_TARGET_MAP, DecodoSearchTool, DecodoWebScrapeTool, - _ENGINE_TARGET_MAP, ) - # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -38,9 +37,7 @@ def _make_api_response(content: str = "Hello world") -> dict[str, Any]: } -def _make_mock_httpx_response( - json_data: dict[str, Any], status_code: int = 200 -) -> MagicMock: +def _make_mock_httpx_response(json_data: dict[str, Any], status_code: int = 200) -> MagicMock: """Build a mock that behaves like an httpx.Response.""" mock_resp = MagicMock() mock_resp.is_success = status_code < 400 @@ -325,14 +322,14 @@ def test_scrape_schema_accepts_url(self) -> None: tool = DecodoWebScrapeTool(decodo_api_token=SecretStr("x")) schema = tool.args_schema assert schema is not None - instance = schema(url="https://example.com") + instance: Any = schema(url="https://example.com") assert instance.url == "https://example.com" def test_search_schema_defaults(self) -> None: tool = DecodoSearchTool(decodo_api_token=SecretStr("x")) schema = tool.args_schema assert schema is not None - instance = schema(query="hello") + instance: Any = schema(query="hello") assert instance.engine == "google" assert instance.num_results == 10 @@ -396,7 +393,8 @@ def test_auth_header_same_regardless_of_mode(self, mock_post: MagicMock) -> None mock_post.return_value = _make_mock_httpx_response(_make_api_response()) for mode in ("basic", "token"): tool = DecodoWebScrapeTool( - decodo_api_token=SecretStr("mytoken"), auth_mode=mode # type: ignore[arg-type] + decodo_api_token=SecretStr("mytoken"), + auth_mode=mode, ) tool._run("https://example.com") _, kwargs = mock_post.call_args From 910bc85a0b5df83c238a81ee7ac7e57d43be1092 Mon Sep 17 00:00:00 2001 From: "aiste.grigaliunaite@decodo.com" Date: Fri, 2 Oct 2026 13:33:15 +0300 Subject: [PATCH 3/4] readme update --- libs/partners/decodo/README.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/libs/partners/decodo/README.md b/libs/partners/decodo/README.md index edf029d78..c9c11fb95 100644 --- a/libs/partners/decodo/README.md +++ b/libs/partners/decodo/README.md @@ -23,6 +23,14 @@ export DECODO_API_TOKEN="your-decodo-api-token" Get a token from the [Decodo Dashboard](https://app.decodo.com). +By default every class expects basic credentials (base64-encoded +`username:password`). If your token is a plain API token, pass +`auth_mode="token"`.: + +```python +tool = DecodoWebScrapeTool(auth_mode="token") +``` + ## Components ### `DecodoWebScrapeTool` @@ -71,11 +79,11 @@ Returns a JSON string — a list of objects with `content`, `url`, and Supported engines: -| `engine` | Decodo target | Description | -|---|---|---| -| `google` | `google_search` | Google SERP | -| `amazon` | `amazon_search` | Amazon product search | -| `reddit` | `google_search` + `site:reddit.com` | Reddit via Google | +| `engine` | Decodo target | Description | +| -------- | ----------------------------------- | --------------------- | +| `google` | `google_search` | Google SERP | +| `amazon` | `amazon_search` | Amazon product search | +| `reddit` | `google_search` + `site:reddit.com` | Reddit via Google | ### `DecodoLoader` From 4423ae6d004f7ff12609f83ea35a84f630aa5eac Mon Sep 17 00:00:00 2001 From: "aiste.grigaliunaite@decodo.com" Date: Fri, 2 Oct 2026 16:20:57 +0300 Subject: [PATCH 4/4] more QA fixes --- libs/partners/decodo/README.md | 2 +- .../decodo/langchain_decodo/_version.py | 2 +- .../partners/decodo/langchain_decodo/tools.py | 28 ++++++++--------- libs/partners/decodo/pyproject.toml | 2 +- .../decodo/tests/unit_tests/test_tools.py | 31 ++++++++++++------- libs/partners/decodo/uv.lock | 2 +- 6 files changed, 36 insertions(+), 31 deletions(-) diff --git a/libs/partners/decodo/README.md b/libs/partners/decodo/README.md index c9c11fb95..915192a20 100644 --- a/libs/partners/decodo/README.md +++ b/libs/partners/decodo/README.md @@ -142,7 +142,7 @@ print(result["messages"][-1].content) ## RAG pipeline example ```bash -pip install langchain-openai langchain-text-splitters langchain-decodo +pip install langchain-openai langchain-text-splitters langchain-decodo numpy ``` ```python diff --git a/libs/partners/decodo/langchain_decodo/_version.py b/libs/partners/decodo/langchain_decodo/_version.py index 3dc1f76bc..485f44ac2 100644 --- a/libs/partners/decodo/langchain_decodo/_version.py +++ b/libs/partners/decodo/langchain_decodo/_version.py @@ -1 +1 @@ -__version__ = "0.1.0" +__version__ = "0.1.1" diff --git a/libs/partners/decodo/langchain_decodo/tools.py b/libs/partners/decodo/langchain_decodo/tools.py index 0675b5e67..5aa858c98 100644 --- a/libs/partners/decodo/langchain_decodo/tools.py +++ b/libs/partners/decodo/langchain_decodo/tools.py @@ -121,6 +121,12 @@ def _do_scrape( return response.json() # type: ignore[no-any-return] +def _is_failed(entry: dict[str, Any]) -> bool: + """Return whether a result entry carries a failed (4xx/5xx or 6xx) status.""" + status = entry.get("status_code") + return isinstance(status, int) and status >= 400 + + def _extract_content(response: dict[str, Any]) -> str: """Pull the first result's content string from a Decodo API response. @@ -431,21 +437,13 @@ def _run( response = _do_scrape(token, _DEFAULT_BASE_URL, payload, auth_mode=self.auth_mode) results = response.get("results", []) - # The API answers HTTP 200 with a per-result failure status (e.g. 613 - # "not able to scrape the target"); surface it so agents can retry - # instead of reading it as "no results". - failed = [ - entry - for entry in results - if isinstance(entry.get("status_code"), int) and entry["status_code"] >= 400 - ] - if results and len(failed) == len(results): - first = failed[0] - detail = first.get("content") if isinstance(first.get("content"), str) else "" - raise RuntimeError( - f"Decodo search failed with status {first['status_code']}" - + (f": {detail}" if detail else "") - ) + # A failed scrape comes back as HTTP 200, either with a failed status + # inside `results` or with no `results` at all (e.g. status 613). Raise + # so agents can retry instead of reading it as "no results". + if not results or all(_is_failed(entry) for entry in results): + status = results[0].get("status_code") if results else response.get("status_code") + message = response.get("message") or "no results returned" + raise RuntimeError(f"Decodo search failed (status {status}): {message}") serialisable = [] for entry in results: diff --git a/libs/partners/decodo/pyproject.toml b/libs/partners/decodo/pyproject.toml index 39dcc2878..c2e159693 100644 --- a/libs/partners/decodo/pyproject.toml +++ b/libs/partners/decodo/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "langchain-decodo" -version = "0.1.0" +version = "0.1.1" description = "LangChain integration for the Decodo web scraping API" readme = "README.md" license = { text = "MIT" } diff --git a/libs/partners/decodo/tests/unit_tests/test_tools.py b/libs/partners/decodo/tests/unit_tests/test_tools.py index d3db47dc3..03f3108d4 100644 --- a/libs/partners/decodo/tests/unit_tests/test_tools.py +++ b/libs/partners/decodo/tests/unit_tests/test_tools.py @@ -215,7 +215,7 @@ def test_google_search_returns_json(self, mock_post: MagicMock) -> None: @patch("langchain_decodo.tools.httpx.post") def test_google_engine_maps_to_correct_target(self, mock_post: MagicMock) -> None: - mock_post.return_value = _make_mock_httpx_response({"results": []}) + mock_post.return_value = _make_mock_httpx_response(_make_api_response()) tool = DecodoSearchTool(decodo_api_token=SecretStr("tok")) tool._run("something", engine="google") @@ -224,7 +224,7 @@ def test_google_engine_maps_to_correct_target(self, mock_post: MagicMock) -> Non @patch("langchain_decodo.tools.httpx.post") def test_amazon_engine_maps_to_correct_target(self, mock_post: MagicMock) -> None: - mock_post.return_value = _make_mock_httpx_response({"results": []}) + mock_post.return_value = _make_mock_httpx_response(_make_api_response()) tool = DecodoSearchTool(decodo_api_token=SecretStr("tok")) tool._run("laptop", engine="amazon") @@ -233,7 +233,7 @@ def test_amazon_engine_maps_to_correct_target(self, mock_post: MagicMock) -> Non @patch("langchain_decodo.tools.httpx.post") def test_reddit_engine_prepends_site_filter(self, mock_post: MagicMock) -> None: - mock_post.return_value = _make_mock_httpx_response({"results": []}) + mock_post.return_value = _make_mock_httpx_response(_make_api_response()) tool = DecodoSearchTool(decodo_api_token=SecretStr("tok")) tool._run("best books", engine="reddit") @@ -243,7 +243,7 @@ def test_reddit_engine_prepends_site_filter(self, mock_post: MagicMock) -> None: @patch("langchain_decodo.tools.httpx.post") def test_unknown_engine_defaults_to_google(self, mock_post: MagicMock) -> None: - mock_post.return_value = _make_mock_httpx_response({"results": []}) + mock_post.return_value = _make_mock_httpx_response(_make_api_response()) tool = DecodoSearchTool(decodo_api_token=SecretStr("tok")) tool._run("query", engine="bing") @@ -252,7 +252,7 @@ def test_unknown_engine_defaults_to_google(self, mock_post: MagicMock) -> None: @patch("langchain_decodo.tools.httpx.post") def test_num_results_forwarded(self, mock_post: MagicMock) -> None: - mock_post.return_value = _make_mock_httpx_response({"results": []}) + mock_post.return_value = _make_mock_httpx_response(_make_api_response()) tool = DecodoSearchTool(decodo_api_token=SecretStr("tok")) tool._run("query", num_results=5) @@ -282,11 +282,18 @@ def test_failed_scrape_status_raises(self, mock_post: MagicMock) -> None: tool._run("q") @patch("langchain_decodo.tools.httpx.post") - def test_empty_results_returns_empty_json_list(self, mock_post: MagicMock) -> None: - mock_post.return_value = _make_mock_httpx_response({"results": []}) + @pytest.mark.parametrize( + "response", + [ + {"results": []}, + {"status": "failed", "status_code": 613, "message": "We were not able to scrape"}, + ], + ) + def test_no_results_raises(self, mock_post: MagicMock, response: dict[str, Any]) -> None: + mock_post.return_value = _make_mock_httpx_response(response) tool = DecodoSearchTool(decodo_api_token=SecretStr("tok")) - result = tool._run("nothing") - assert json.loads(result) == [] + with pytest.raises(RuntimeError, match="Decodo search failed"): + tool._run("nothing") @patch("langchain_decodo.tools.httpx.post") def test_api_error_raises_runtime_error(self, mock_post: MagicMock) -> None: @@ -299,7 +306,7 @@ def test_api_error_raises_runtime_error(self, mock_post: MagicMock) -> None: @patch("langchain_decodo.tools.httpx.post") def test_authorization_header(self, mock_post: MagicMock) -> None: - mock_post.return_value = _make_mock_httpx_response({"results": []}) + mock_post.return_value = _make_mock_httpx_response(_make_api_response()) tool = DecodoSearchTool(decodo_api_token=SecretStr("secret")) tool._run("query") @@ -363,7 +370,7 @@ def test_scrape_token_mode_uses_unified_endpoint(self, mock_post: MagicMock) -> @patch("langchain_decodo.tools.httpx.post") def test_search_basic_mode_uses_v2_endpoint(self, mock_post: MagicMock) -> None: - mock_post.return_value = _make_mock_httpx_response({"results": []}) + mock_post.return_value = _make_mock_httpx_response(_make_api_response()) tool = DecodoSearchTool(decodo_api_token=SecretStr("tok"), auth_mode="basic") tool._run("query") @@ -372,7 +379,7 @@ def test_search_basic_mode_uses_v2_endpoint(self, mock_post: MagicMock) -> None: @patch("langchain_decodo.tools.httpx.post") def test_search_token_mode_uses_unified_endpoint(self, mock_post: MagicMock) -> None: - mock_post.return_value = _make_mock_httpx_response({"results": []}) + mock_post.return_value = _make_mock_httpx_response(_make_api_response()) tool = DecodoSearchTool(decodo_api_token=SecretStr("tok"), auth_mode="token") tool._run("query") diff --git a/libs/partners/decodo/uv.lock b/libs/partners/decodo/uv.lock index b3368919c..cd07669a3 100644 --- a/libs/partners/decodo/uv.lock +++ b/libs/partners/decodo/uv.lock @@ -342,7 +342,7 @@ wheels = [ [[package]] name = "langchain-decodo" -version = "0.1.0" +version = "0.1.1" source = { editable = "." } dependencies = [ { name = "httpx" },