From aa7a78412f9afc0ded0780a04f9fa0fb28aac439 Mon Sep 17 00:00:00 2001 From: vchamarthi Date: Mon, 19 Oct 2026 11:28:42 -0500 Subject: [PATCH 1/2] build(deps): move ragas and langchain to a dokimi[ragas] extra pip install dokimi no longer installs ragas, langchain-community, langchain-anthropic, langchain-openai or langchain-core. eval ragas and answers generate --ragas need the extra and exit with the install command when it is missing. The dev extra includes it. Signed-off-by: vchamarthi --- dokimi/CONTRIBUTING.md | 5 +++-- .../20261019_120000_vchamarthi_ragas-extra.md | 8 ++++++++ dokimi/docs/quickstart.md | 2 +- dokimi/dokimi/commands/evaluate.py | 2 ++ dokimi/dokimi/eval/ragas_eval.py | 4 ++-- dokimi/pyproject.toml | 19 ++++++++++++------- dokimi/tests/test_ragas_runtime.py | 13 +++++++++++++ 7 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 dokimi/changelog.d/20261019_120000_vchamarthi_ragas-extra.md diff --git a/dokimi/CONTRIBUTING.md b/dokimi/CONTRIBUTING.md index 34a7d903..f963605b 100644 --- a/dokimi/CONTRIBUTING.md +++ b/dokimi/CONTRIBUTING.md @@ -49,8 +49,9 @@ The package exposes a console entry point: dokimi --help # equivalent to `python -m dokimi` ``` -Optional feature sets are available as extras: `ocr` (Docling ingestion), -`mcp`, and `embeddings`. Install what you need, e.g. `pip install -e ".[dev,ocr]"`. +Optional feature sets are available as extras: `ragas` (`eval ragas`, which +`dev` includes), `ocr` (Docling ingestion), `mcp`, and `embeddings`. Install +what you need, e.g. `pip install -e ".[dev,ocr]"`. ## Before you open a PR diff --git a/dokimi/changelog.d/20261019_120000_vchamarthi_ragas-extra.md b/dokimi/changelog.d/20261019_120000_vchamarthi_ragas-extra.md new file mode 100644 index 00000000..01889f00 --- /dev/null +++ b/dokimi/changelog.d/20261019_120000_vchamarthi_ragas-extra.md @@ -0,0 +1,8 @@ +### Changed + +- **RAGAS is an optional extra**: `pip install dokimi` no longer installs + `ragas`, `langchain-community`, `langchain-anthropic`, `langchain-openai` or + `langchain-core`. `eval ragas` and `answers generate --ragas` need + `pip install 'dokimi[ragas]'` (the `dev` extra includes it). Without the + extra, both exit with an error that names the missing package and the + install command, instead of printing a warning and exiting 0. diff --git a/dokimi/docs/quickstart.md b/dokimi/docs/quickstart.md index e08eeb30..b7006243 100644 --- a/dokimi/docs/quickstart.md +++ b/dokimi/docs/quickstart.md @@ -239,7 +239,7 @@ per answer in `answers/.json`. | Symptom | Fix | | --- | --- | -| LLM calls fail / wrong provider | All providers go through LiteLLM (`dokimi/llm.py`); set the matching `*_API_KEY` and pass `--provider`/`--model`. LangChain is used only by `eval ragas`, which wraps its judge in `ChatOpenAI`/`ChatAnthropic` (`langchain-openai` comes with ragas; `langchain-anthropic` is a direct dependency). | +| LLM calls fail / wrong provider | All providers go through LiteLLM (`dokimi/llm.py`); set the matching `*_API_KEY` and pass `--provider`/`--model`. LangChain is used only by `eval ragas`, which wraps its judge in `ChatOpenAI`/`ChatAnthropic`; it and ragas come with the `ragas` extra (`pip install 'dokimi[ragas]'`). | | `No module named numpy` | Pipeline falls back to exact-text dedup; install numpy for embedding-based dedup | | `GitHub client not available` | Pipeline uses `gh` CLI fallback; `pip install PyGithub` to silence | | Context7 timeout | Raise `context7.timeout` in `dokimi/config/products.yaml` (or in a config/products.yaml of your own in the working directory) | diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index 431c0de9..1424a8e0 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -73,6 +73,8 @@ def _run_ragas_eval(answers, output_path, args) -> None: except EmptyQuestionError: raise # a bad input record is a refusal, not a RAGAS failure: cli.main reports it (exit 2) + except ImportError as exc: + sys.exit(f"RAGAS meta-evaluation was requested but cannot run: {exc}") except Exception as exc: print(f"\n⚠️ RAGAS evaluation failed: {exc}") diff --git a/dokimi/dokimi/eval/ragas_eval.py b/dokimi/dokimi/eval/ragas_eval.py index 86e493c0..27dee309 100644 --- a/dokimi/dokimi/eval/ragas_eval.py +++ b/dokimi/dokimi/eval/ragas_eval.py @@ -104,7 +104,7 @@ def evaluate( Dataset, ragas_evaluate = _import_ragas_runtime() except ImportError as exc: raise ImportError("ragas and datasets packages are required. " - "Install: pip install ragas datasets") from exc + "Install the ragas extra: pip install 'dokimi[ragas]'") from exc rows_with = [] rows_without = [] @@ -254,7 +254,7 @@ def _build_ragas_llm(self): # import (e.g. a langchain_community module that a newer release removed). raise ImportError( f"RAGAS evaluation could not import {exc.name or 'a dependency'}: {exc}. " - "Install the locked dependencies: pip install -r requirements.lock.txt" + "Install the ragas extra: pip install 'dokimi[ragas]'" ) from exc def _build_metrics(self) -> list: diff --git a/dokimi/pyproject.toml b/dokimi/pyproject.toml index 1a169bce..3d4ecb6e 100644 --- a/dokimi/pyproject.toml +++ b/dokimi/pyproject.toml @@ -33,13 +33,6 @@ dependencies = [ "numpy>=1.24.0", "scipy>=1.11.0", "PyGithub>=2.1.0", - "ragas>=0.3.0,<0.4", - # Every ragas release imports langchain_community.chat_models.vertexai, which 0.4.2 removed. - "langchain-community<0.4.2", - # eval ragas wraps the judge in ChatOpenAI or ChatAnthropic, typed as langchain-core's BaseChatModel. - "langchain-anthropic>=1.0,<2", - "langchain-openai>=1.0,<2", - "langchain-core>=1.0,<2", "datasets>=2.20,<6", "openai>=1.51,<3", ] @@ -59,6 +52,18 @@ dev = [ "types-PyYAML", "types-jsonschema", "scriv>=1.5", + # The test suite covers eval ragas. + "dokimi[ragas]", +] +# `eval ragas` and `answers generate --ragas`. +ragas = [ + "ragas>=0.3.0,<0.4", + # Every ragas release imports langchain_community.chat_models.vertexai, which 0.4.2 removed. + "langchain-community<0.4.2", + # eval ragas wraps the judge in ChatOpenAI or ChatAnthropic, typed as langchain-core's BaseChatModel. + "langchain-anthropic>=1.0,<2", + "langchain-openai>=1.0,<2", + "langchain-core>=1.0,<2", ] ocr = ["docling>=2.0"] mcp = ["mcp>=1.0"] diff --git a/dokimi/tests/test_ragas_runtime.py b/dokimi/tests/test_ragas_runtime.py index 7aeaba37..6adf6a05 100644 --- a/dokimi/tests/test_ragas_runtime.py +++ b/dokimi/tests/test_ragas_runtime.py @@ -17,6 +17,8 @@ import sys from pathlib import Path +import pytest + REPO = Path(__file__).resolve().parents[1] _PROBE = """ @@ -42,3 +44,14 @@ def test_ragas_imports_and_evaluator_builds_from_installed_deps(): timeout=120, ) assert proc.returncode == 0, proc.stderr[-3000:] + + +def test_ragas_without_the_extra_exits_with_the_install_command(monkeypatch, tmp_path): + from dokimi.commands.evaluate import _run_ragas_eval + + monkeypatch.setitem(sys.modules, "ragas", None) + monkeypatch.setitem(sys.modules, "ragas.llms", None) + with pytest.raises(SystemExit) as exc: + _run_ragas_eval([], tmp_path / "answers.json", object()) + assert "pip install 'dokimi[ragas]'" in str(exc.value.code) + assert not (tmp_path / "answers_ragas.json").exists() From 2a41c2c5f226d3757c50394897c4e4febaea7e4f Mon Sep 17 00:00:00 2001 From: vchamarthi Date: Mon, 19 Oct 2026 12:08:08 -0500 Subject: [PATCH 2/2] fix(ragas): check for the extra before any work; review follow-ups - eval ragas and answers generate --ragas exit before any model call when ragas is not installed - import-failure hints say when the extra is the fix - README lists the ragas extra; deptry comment points at it Signed-off-by: vchamarthi --- dokimi/README.md | 3 ++- .../20261019_120000_vchamarthi_ragas-extra.md | 3 +-- dokimi/dokimi/commands/answers.py | 4 +++- dokimi/dokimi/commands/evaluate.py | 15 +++++++++------ dokimi/dokimi/eval/ragas_eval.py | 4 ++-- dokimi/pyproject.toml | 2 +- dokimi/tests/test_ragas_runtime.py | 15 +++++++++++++++ 7 files changed, 33 insertions(+), 13 deletions(-) diff --git a/dokimi/README.md b/dokimi/README.md index 15eba980..0a116469 100644 --- a/dokimi/README.md +++ b/dokimi/README.md @@ -67,7 +67,8 @@ The project is pre-1.0; interfaces and artifact schemas may still change. `AWS_REGION` + `AWS_BEARER_TOKEN_BEDROCK` for Bedrock). `GITHUB_TOKEN` is optional and raises the persona-discovery rate limit. - Docker, only for the executable-task, harness and repository tracks. -- Optional extras: `ocr` (Docling ingestion) and `mcp` (real MCP servers). +- Optional extras: `ragas` (`eval ragas`, `answers generate --ragas`), `ocr` + (Docling ingestion) and `mcp` (real MCP servers). ```bash pip install -e ".[dev]" # installs the `dokimi` command plus test tooling diff --git a/dokimi/changelog.d/20261019_120000_vchamarthi_ragas-extra.md b/dokimi/changelog.d/20261019_120000_vchamarthi_ragas-extra.md index 01889f00..df32f1ff 100644 --- a/dokimi/changelog.d/20261019_120000_vchamarthi_ragas-extra.md +++ b/dokimi/changelog.d/20261019_120000_vchamarthi_ragas-extra.md @@ -4,5 +4,4 @@ `ragas`, `langchain-community`, `langchain-anthropic`, `langchain-openai` or `langchain-core`. `eval ragas` and `answers generate --ragas` need `pip install 'dokimi[ragas]'` (the `dev` extra includes it). Without the - extra, both exit with an error that names the missing package and the - install command, instead of printing a warning and exiting 0. + extra, both exit 1 with the install command before any model call. diff --git a/dokimi/dokimi/commands/answers.py b/dokimi/dokimi/commands/answers.py index d01c27f3..0e3b17ff 100644 --- a/dokimi/dokimi/commands/answers.py +++ b/dokimi/dokimi/commands/answers.py @@ -9,7 +9,7 @@ import sys from pathlib import Path -from dokimi.commands.evaluate import _run_ragas_eval +from dokimi.commands.evaluate import _require_ragas, _run_ragas_eval from dokimi.defaults import ( DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, @@ -23,6 +23,8 @@ def cmd_answers_generate(args: argparse.Namespace) -> None: """Generate context-arm and baseline answers for questions.""" + if getattr(args, "ragas", False): + _require_ragas() import yaml from dokimi.eval import Answerer from dokimi.mcp.factory import ( diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index 1424a8e0..075cfdfb 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse +import importlib.util import json import sys from pathlib import Path @@ -41,15 +42,16 @@ def _warn_judge_independence( return same +def _require_ragas() -> None: + """Exit before any model call when the ragas extra is not installed.""" + if importlib.util.find_spec("ragas") is None: + sys.exit("RAGAS meta-evaluation needs the ragas extra: pip install 'dokimi[ragas]'") + + def _run_ragas_eval(answers, output_path, args) -> None: """Run RAGAS meta-evaluation and save results alongside answers.""" from dokimi.eval.question_text import EmptyQuestionError - - try: - from dokimi.eval.ragas_eval import RagasEvaluator - except ImportError as e: - print(f"\n⚠️ RAGAS not available ({e}). Skipping meta-evaluation.") - return + from dokimi.eval.ragas_eval import RagasEvaluator ragas_model = getattr(args, "ragas_model", DEFAULT_JUDGE_MODEL) ragas_provider = getattr(args, "ragas_provider", DEFAULT_JUDGE_PROVIDER) @@ -225,6 +227,7 @@ def cmd_eval_score(args: argparse.Namespace) -> None: def cmd_eval_ragas(args: argparse.Namespace) -> None: """Run standalone RAGAS meta-evaluation on an answers JSON.""" + _require_ragas() answers_data = json.loads(Path(args.answers).read_text(encoding="utf-8")) answers = answers_data.get("answers", answers_data) print(f"Loaded {len(answers)} answers from {args.answers}") diff --git a/dokimi/dokimi/eval/ragas_eval.py b/dokimi/dokimi/eval/ragas_eval.py index 27dee309..311e7a43 100644 --- a/dokimi/dokimi/eval/ragas_eval.py +++ b/dokimi/dokimi/eval/ragas_eval.py @@ -104,7 +104,7 @@ def evaluate( Dataset, ragas_evaluate = _import_ragas_runtime() except ImportError as exc: raise ImportError("ragas and datasets packages are required. " - "Install the ragas extra: pip install 'dokimi[ragas]'") from exc + "If the ragas extra is not installed: pip install 'dokimi[ragas]'") from exc rows_with = [] rows_without = [] @@ -254,7 +254,7 @@ def _build_ragas_llm(self): # import (e.g. a langchain_community module that a newer release removed). raise ImportError( f"RAGAS evaluation could not import {exc.name or 'a dependency'}: {exc}. " - "Install the ragas extra: pip install 'dokimi[ragas]'" + "If the ragas extra is not installed: pip install 'dokimi[ragas]'" ) from exc def _build_metrics(self) -> list: diff --git a/dokimi/pyproject.toml b/dokimi/pyproject.toml index 3d4ecb6e..1c7478bb 100644 --- a/dokimi/pyproject.toml +++ b/dokimi/pyproject.toml @@ -130,7 +130,7 @@ extend_exclude = [ optional_dependencies_dev_groups = ["dev"] [tool.deptry.per_rule_ignores] -# Declared only to cap the version ragas resolves (see the comment in [project]). +# Declared only to cap the version ragas resolves (see the comment in the ragas extra). DEP002 = ["langchain-community"] [tool.ruff] diff --git a/dokimi/tests/test_ragas_runtime.py b/dokimi/tests/test_ragas_runtime.py index 6adf6a05..ba0eb4d2 100644 --- a/dokimi/tests/test_ragas_runtime.py +++ b/dokimi/tests/test_ragas_runtime.py @@ -12,6 +12,7 @@ from __future__ import annotations +import argparse import os import subprocess import sys @@ -55,3 +56,17 @@ def test_ragas_without_the_extra_exits_with_the_install_command(monkeypatch, tmp _run_ragas_eval([], tmp_path / "answers.json", object()) assert "pip install 'dokimi[ragas]'" in str(exc.value.code) assert not (tmp_path / "answers_ragas.json").exists() + + +@pytest.mark.parametrize("command", ["eval_ragas", "answers_generate"]) +def test_ragas_commands_exit_before_any_work_without_the_extra(monkeypatch, tmp_path, command): + from dokimi.commands.answers import cmd_answers_generate + from dokimi.commands.evaluate import cmd_eval_ragas + + monkeypatch.setitem(sys.modules, "ragas", None) + missing = tmp_path / "missing.json" # reading it would raise FileNotFoundError + args = argparse.Namespace(answers=str(missing), questions=str(missing), ragas=True) + run = cmd_eval_ragas if command == "eval_ragas" else cmd_answers_generate + with pytest.raises(SystemExit) as exc: + run(args) + assert "pip install 'dokimi[ragas]'" in str(exc.value.code)