From a5f2ea02ddc656916ac26f872a5ac41f228cc8dc Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Wed, 2 Sep 2026 19:53:55 -0400 Subject: [PATCH 1/5] build: update directive drafter to 0.2.0dev5 --- pyproject.toml | 6 +++--- .../examples/prompt_construction/litellm/README.md | 2 +- .../reference_integrations/litellm_proxy/README.md | 2 +- .../reference_integrations/openwebui_pipe/README.md | 6 +++--- .../open_webui_pipe_with_directive_drafter.py | 2 +- uv.lock | 12 ++++++------ 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2d9fbe0..7ac9f1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,12 +47,12 @@ Issues = "https://github.com/rlippmann/context-compiler-example-integrations/iss [project.optional-dependencies] all = [ "chromadb", - "context-compiler-directive-drafter==0.2.0dev4", + "context-compiler-directive-drafter==0.2.0dev5", "fastapi", "litellm", ] drafter = [ - "context-compiler-directive-drafter==0.2.0dev4", + "context-compiler-directive-drafter==0.2.0dev5", ] fastapi = [ "fastapi", @@ -67,7 +67,7 @@ retrieval = [ [dependency-groups] dev = [ "chromadb", - "context-compiler-directive-drafter==0.2.0dev4", + "context-compiler-directive-drafter==0.2.0dev5", "fastapi", "httpx2>=2.5.0", "httpx>=0.28.1", diff --git a/python/examples/prompt_construction/litellm/README.md b/python/examples/prompt_construction/litellm/README.md index b323bad..8829479 100644 --- a/python/examples/prompt_construction/litellm/README.md +++ b/python/examples/prompt_construction/litellm/README.md @@ -68,7 +68,7 @@ For `with_directive_drafter.py`: pip install "context-compiler-example-integrations[all]" ``` -That variant requires `context-compiler-directive-drafter>=0.2.0dev4`. +That variant requires `context-compiler-directive-drafter>=0.2.0dev5`. ## Quickstart (copy/paste) diff --git a/python/reference_integrations/litellm_proxy/README.md b/python/reference_integrations/litellm_proxy/README.md index 4143e39..3e4d8e0 100644 --- a/python/reference_integrations/litellm_proxy/README.md +++ b/python/reference_integrations/litellm_proxy/README.md @@ -108,7 +108,7 @@ For `context_compiler_precall_hook_with_directive_drafter.py`: pip install "context-compiler-example-integrations[all]" ``` -That variant requires `context-compiler-directive-drafter>=0.2.0dev4`. +That variant requires `context-compiler-directive-drafter>=0.2.0dev5`. For the opt-in runtime smoke test, install the proxy runtime extras: diff --git a/python/reference_integrations/openwebui_pipe/README.md b/python/reference_integrations/openwebui_pipe/README.md index 33b7390..41323cd 100644 --- a/python/reference_integrations/openwebui_pipe/README.md +++ b/python/reference_integrations/openwebui_pipe/README.md @@ -68,7 +68,7 @@ These examples require `context-compiler>=0.9.0dev13`. If using `open_webui_pipe_with_directive_drafter.py`: - Install directive-drafter support if needed: - `pip install "context-compiler>=0.9.0dev13" "context-compiler-directive-drafter>=0.2.0dev4"` + `pip install "context-compiler>=0.9.0dev13" "context-compiler-directive-drafter>=0.2.0dev5"` - Set `PREPROCESSOR_PROMPT_PROFILE=default` for heuristic-first behavior - Optionally set `PREPROCESSOR_MODEL_ID` to use a separate fallback drafting model - If `PREPROCESSOR_MODEL_ID` is unset, fallback uses `BASE_MODEL_ID` @@ -86,7 +86,7 @@ If frontmatter dependency installs are disabled, offline, or unavailable: 1. Install the package manually: - Minimal pipe: `pip install "context-compiler>=0.9.0dev13"` -- Directive Drafter pipe: `pip install "context-compiler>=0.9.0dev13" "context-compiler-directive-drafter>=0.2.0dev4"` +- Directive Drafter pipe: `pip install "context-compiler>=0.9.0dev13" "context-compiler-directive-drafter>=0.2.0dev5"` 1. Import and enable the function in Open WebUI, then configure valves. @@ -246,7 +246,7 @@ rejection flows. - `PREPROCESSOR_MODEL_ID must not match the selected pipe model id`: choose a real backend model id, not the pipe model id itself. - `PREPROCESSOR_MODEL_ID is invalid or not configured in Open WebUI`: the fallback route hit a missing model; fix the configured fallback model or unset it to reuse `BASE_MODEL_ID`. - `ALLOW_MISSING_BASE_MODEL_FOR_DEBUG=true`: directive-only updates still run locally, but passthrough returns a deterministic debug message instead of calling a downstream model. -- imports fail after function upload: install `context-compiler>=0.9.0dev13` in the Open WebUI runtime, and add `context-compiler-directive-drafter>=0.2.0dev4` only for the Directive Drafter pipe, because the copied function runs from a temp/cached location. +- imports fail after function upload: install `context-compiler>=0.9.0dev13` in the Open WebUI runtime, and add `context-compiler-directive-drafter>=0.2.0dev5` only for the Directive Drafter pipe, because the copied function runs from a temp/cached location. ## Fallback notes diff --git a/python/reference_integrations/openwebui_pipe/open_webui_pipe_with_directive_drafter.py b/python/reference_integrations/openwebui_pipe/open_webui_pipe_with_directive_drafter.py index 12b636d..a3f49e6 100644 --- a/python/reference_integrations/openwebui_pipe/open_webui_pipe_with_directive_drafter.py +++ b/python/reference_integrations/openwebui_pipe/open_webui_pipe_with_directive_drafter.py @@ -3,7 +3,7 @@ author: rlippmann author_url: https://github.com/rlippmann/context-compiler-example-integrations version: 0.10.1 -requirements: context-compiler>=0.9.0dev13, context-compiler-directive-drafter>=0.2.0dev4 +requirements: context-compiler>=0.9.0dev13, context-compiler-directive-drafter>=0.2.0dev5 Open WebUI integration with Context Compiler directive drafter. diff --git a/uv.lock b/uv.lock index 7f496b6..4c3e850 100644 --- a/uv.lock +++ b/uv.lock @@ -667,14 +667,14 @@ wheels = [ [[package]] name = "context-compiler-directive-drafter" -version = "0.2.0.dev4" +version = "0.2.0.dev5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "context-compiler" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/95/33/797dd6d527d6f0513f6cff613ec8bae3b9332e62f6b772863a5aeb16ebd4/context_compiler_directive_drafter-0.2.0.dev4.tar.gz", hash = "sha256:80d89fc004966282e592fa757e17e5bdd1171440f6f5d8606766dead8eaa74da", size = 159341, upload-time = "2026-09-01T15:41:57.258Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3d/92/2b5487df80e887c27002f97c117edf2dfbde2022d67cd24e86c95961a5a0/context_compiler_directive_drafter-0.2.0.dev5.tar.gz", hash = "sha256:020fae29026fee0dd7f38c6a3d58d972c288ddd200df72ff7d2473fb64dd4f75", size = 280663, upload-time = "2026-09-02T07:24:26.502Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/8e/5800ed38b3924dfa08e81cec5470c7e88bb376fa04c50bfaf0d21df0f340/context_compiler_directive_drafter-0.2.0.dev4-py3-none-any.whl", hash = "sha256:9ea45cf88b08763039a229a938285c6e6d5f38bbe2df573d1181ddd3570208f2", size = 29735, upload-time = "2026-09-01T15:41:55.794Z" }, + { url = "https://files.pythonhosted.org/packages/b5/3a/11d2ef36aecb7df8c2388b3f6015958ceefc2971b802137188e02b2c8383/context_compiler_directive_drafter-0.2.0.dev5-py3-none-any.whl", hash = "sha256:158d2305c479fc25f572dd72a4eefd298d702b796db4fb62f48611a773ba8c58", size = 35113, upload-time = "2026-09-02T07:24:25.307Z" }, ] [[package]] @@ -727,8 +727,8 @@ requires-dist = [ { name = "chromadb", marker = "extra == 'all'" }, { name = "chromadb", marker = "extra == 'retrieval'" }, { name = "context-compiler", specifier = "==0.9.0.dev13" }, - { name = "context-compiler-directive-drafter", marker = "extra == 'all'", specifier = "==0.2.0.dev4" }, - { name = "context-compiler-directive-drafter", marker = "extra == 'drafter'", specifier = "==0.2.0.dev4" }, + { name = "context-compiler-directive-drafter", marker = "extra == 'all'", specifier = "==0.2.0.dev5" }, + { name = "context-compiler-directive-drafter", marker = "extra == 'drafter'", specifier = "==0.2.0.dev5" }, { name = "fastapi", marker = "extra == 'all'" }, { name = "fastapi", marker = "extra == 'fastapi'" }, { name = "litellm", marker = "extra == 'all'" }, @@ -739,7 +739,7 @@ provides-extras = ["all", "drafter", "fastapi", "litellm", "retrieval"] [package.metadata.requires-dev] dev = [ { name = "chromadb" }, - { name = "context-compiler-directive-drafter", specifier = "==0.2.0.dev4" }, + { name = "context-compiler-directive-drafter", specifier = "==0.2.0.dev5" }, { name = "fastapi" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "httpx2", specifier = ">=2.5.0" }, From f9500ba2b932cc146c5a0e6ccbbdd76b52f5b09a Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Wed, 2 Sep 2026 19:57:27 -0400 Subject: [PATCH 2/5] refactor: adopt dev5 fallback profiles and LiteLLM adapter --- .../litellm/with_directive_drafter.py | 62 +++++-------- ...ler_precall_hook_with_directive_drafter.py | 87 ++++++------------- .../open_webui_pipe_with_directive_drafter.py | 6 +- 3 files changed, 54 insertions(+), 101 deletions(-) diff --git a/python/examples/prompt_construction/litellm/with_directive_drafter.py b/python/examples/prompt_construction/litellm/with_directive_drafter.py index 59b79ec..b648a72 100644 --- a/python/examples/prompt_construction/litellm/with_directive_drafter.py +++ b/python/examples/prompt_construction/litellm/with_directive_drafter.py @@ -18,6 +18,7 @@ import logging import os from collections.abc import Callable, Mapping, Sequence +from functools import lru_cache from importlib import import_module from typing import TypedDict, cast @@ -37,7 +38,9 @@ RejectedDirective, UnknownDirective, ) -from context_compiler_directive_drafter.fallbacks import get_converter_prompt +from context_compiler_directive_drafter.fallbacks.litellm import ( + create_litellm_fallback, +) from context_compiler_example_integrations.examples._shared.provider_mode import ( print_startup_config, @@ -82,12 +85,6 @@ def _extract_response_content(response: object) -> str | None: return None -_DIRECTIVE_DRAFTER = DirectiveDrafter( - fallback=lambda message: _llm_fallback_candidate(message), - fallback_source="litellm_fallback", -) - - def _render_state_lines( premise: str | None, policies: Mapping[str, PolicyValue] ) -> list[str]: @@ -211,44 +208,31 @@ def _call_litellm(messages: list[dict[str, str]]) -> str: return content -def _llm_fallback_candidate(message: str) -> str | None: - try: - completion = _get_litellm_completion() - except ModuleNotFoundError: - return None +@lru_cache(maxsize=8) +def _create_directive_drafter( + model: str, api_key: str | None, api_base: str +) -> DirectiveDrafter: + return DirectiveDrafter( + fallback=create_litellm_fallback( + model=model, + api_key=api_key, + api_base=api_base, + ), + fallback_source="litellm_fallback", + ) - try: - config = resolve_provider_config(default_model="openai/gpt-4o-mini") - except RuntimeError: - return None - if config.mode == "openai" and not config.api_key: - return None - preprocessor_model = os.getenv("PREPROCESSOR_MODEL", "").strip() - if not preprocessor_model: - preprocessor_model = os.getenv("MODEL", "openai/gpt-4o-mini") - kwargs: _LiteLLMCallKwargs = { - "model": preprocessor_model, - "messages": [ - {"role": "system", "content": get_converter_prompt()}, - {"role": "user", "content": message}, - ], - "temperature": 0, - "api_base": config.base_url, - } - if config.api_key: - kwargs["api_key"] = config.api_key - - try: - response = completion(**kwargs) - return _extract_response_content(response) - except Exception: - return None +def _get_directive_drafter() -> DirectiveDrafter: + config = resolve_provider_config(default_model="openai/gpt-4o-mini") + preprocessor_model = os.getenv("PREPROCESSOR_MODEL", "").strip() or config.model + return _create_directive_drafter( + preprocessor_model, config.api_key, config.base_url + ) def _preprocess_user_input(message: str) -> str | None: try: - drafted_result = _DIRECTIVE_DRAFTER.draft_directive(message) + drafted_result = _get_directive_drafter().draft_directive(message) logger.debug("preprocessor: drafted_result=%r", drafted_result) return _extract_drafted_text(drafted_result) except Exception: diff --git a/python/reference_integrations/litellm_proxy/context_compiler_precall_hook_with_directive_drafter.py b/python/reference_integrations/litellm_proxy/context_compiler_precall_hook_with_directive_drafter.py index 9a6af30..2a46670 100644 --- a/python/reference_integrations/litellm_proxy/context_compiler_precall_hook_with_directive_drafter.py +++ b/python/reference_integrations/litellm_proxy/context_compiler_precall_hook_with_directive_drafter.py @@ -13,9 +13,8 @@ import logging import os -from collections.abc import Callable, Mapping, Sequence -from importlib import import_module -from typing import Any, cast +from functools import lru_cache +from typing import Any try: from litellm.integrations.custom_logger import CustomLogger @@ -33,7 +32,9 @@ class CustomLogger: # type: ignore[no-redef] DirectiveDrafter, DraftResult, ) -from context_compiler_directive_drafter.fallbacks import get_converter_prompt +from context_compiler_directive_drafter.fallbacks.litellm import ( + create_litellm_fallback, +) from context_compiler_example_integrations.reference_integrations.litellm_proxy._checkpoint_support import ( MODE_PERSISTENT, CheckpointStore, @@ -60,75 +61,41 @@ class CustomLogger: # type: ignore[no-redef] CHECKPOINT_STORE: CheckpointStore = InMemoryCheckpointStore() -def _extract_response_content(response: object) -> str | None: - if isinstance(response, Mapping): - choices = response.get("choices") - if isinstance(choices, Sequence) and choices: - first = choices[0] - if isinstance(first, Mapping): - message = first.get("message") - if isinstance(message, Mapping): - content = message.get("content") - if isinstance(content, str): - return content - - choices_attr = getattr(response, "choices", None) - if isinstance(choices_attr, Sequence) and choices_attr: - first = choices_attr[0] - message_attr = getattr(first, "message", None) - content_attr = getattr(message_attr, "content", None) - if isinstance(content_attr, str): - return content_attr - - return None - - -def _get_litellm_completion() -> Callable[..., object]: - litellm_module = import_module("litellm") - return cast(Callable[..., object], litellm_module.completion) +@lru_cache(maxsize=8) +def _create_directive_drafter( + model: str, api_key: str, api_base: str | None +) -> DirectiveDrafter: + return DirectiveDrafter( + fallback=create_litellm_fallback( + model=model, + api_key=api_key, + api_base=api_base, + ), + fallback_source="litellm_fallback", + ) -def _llm_fallback_candidate(message: str) -> str | None: +def _get_directive_drafter() -> DirectiveDrafter: preprocessor_model = os.getenv("PREPROCESSOR_MODEL", "").strip() if not preprocessor_model: preprocessor_model = os.getenv("MODEL", "").strip() if not preprocessor_model: - return None + return DirectiveDrafter() api_key = os.getenv("OPENAI_API_KEY") if not api_key: - return None + return DirectiveDrafter() + return _create_directive_drafter( + preprocessor_model, api_key, os.getenv("OPENAI_BASE_URL") or None + ) - try: - completion = _get_litellm_completion() - except ModuleNotFoundError: - return None - - kwargs: dict[str, object] = { - "model": preprocessor_model, - "messages": [ - {"role": "system", "content": get_converter_prompt()}, - {"role": "user", "content": message}, - ], - "api_key": api_key, - "temperature": 0, - } - api_base = os.getenv("OPENAI_BASE_URL") - if api_base: - kwargs["api_base"] = api_base +def _draft_last_user_message(message: str) -> DraftResult: try: - response = completion(**kwargs) - return _extract_response_content(response) + return _get_directive_drafter().draft_directive(message) except Exception: - return None - - -def _draft_last_user_message(message: str) -> DraftResult: - drafter = DirectiveDrafter( - fallback=_llm_fallback_candidate, fallback_source="litellm_fallback" - ) - return drafter.draft_directive(message) + logger.debug("litellm_proxy: drafter_exception", exc_info=True) + return DirectiveDrafter().draft_directive(message) class ContextCompilerPreCallHookWithPreprocessor(CustomLogger): diff --git a/python/reference_integrations/openwebui_pipe/open_webui_pipe_with_directive_drafter.py b/python/reference_integrations/openwebui_pipe/open_webui_pipe_with_directive_drafter.py index a3f49e6..21c76e1 100644 --- a/python/reference_integrations/openwebui_pipe/open_webui_pipe_with_directive_drafter.py +++ b/python/reference_integrations/openwebui_pipe/open_webui_pipe_with_directive_drafter.py @@ -62,10 +62,12 @@ def Field(*, default: Any, description: str = "") -> Any: # type: ignore[no-red RejectedDirective, UnknownDirective, ) -from context_compiler_directive_drafter.fallbacks import get_converter_prompt +from context_compiler_directive_drafter.fallbacks import get_fallback_profile logger = logging.getLogger(__name__) +_FALLBACK_PROFILE = get_fallback_profile() + _CC_MARKER = "[[cc_state]]" _ENGINES_BY_CHAT_KEY: dict[str, Engine] = {} @@ -634,7 +636,7 @@ async def _llm_fallback_candidate( "model": model_id, "stream": False, "messages": [ - {"role": "system", "content": get_converter_prompt()}, + {"role": "system", "content": _FALLBACK_PROFILE.system_prompt}, {"role": "user", "content": message}, ], } From 1b478ba58a7a40c7d0aa5a7f37761c723e434ecd Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Wed, 2 Sep 2026 20:00:57 -0400 Subject: [PATCH 3/5] test: validate dev5 LiteLLM fallback integration --- python/tests/test_litellm_drafter_adapter.py | 56 ++++++++++++ ...st_litellm_proxy_with_directive_drafter.py | 46 +++++----- .../test_litellm_with_directive_drafter.py | 88 ++++++++++--------- 3 files changed, 128 insertions(+), 62 deletions(-) create mode 100644 python/tests/test_litellm_drafter_adapter.py diff --git a/python/tests/test_litellm_drafter_adapter.py b/python/tests/test_litellm_drafter_adapter.py new file mode 100644 index 0000000..16c12f1 --- /dev/null +++ b/python/tests/test_litellm_drafter_adapter.py @@ -0,0 +1,56 @@ +import json +import sys +import types +from types import SimpleNamespace + +from context_compiler_directive_drafter.fallbacks.litellm import ( + create_litellm_fallback, +) + + +def _response(content: str) -> SimpleNamespace: + return SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content=content))] + ) + + +def test_structured_litellm_fallback_sends_schema_and_parses_response(monkeypatch): + calls: list[dict[str, object]] = [] + litellm = types.ModuleType("litellm") + litellm.supports_response_schema = lambda **_: True # type: ignore[attr-defined] + + def completion(**kwargs: object) -> SimpleNamespace: + calls.append(kwargs) + return _response( + json.dumps({"classification": "directive", "output": "use docker"}) + ) + + litellm.completion = completion # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "litellm", litellm) + + fallback = create_litellm_fallback("openai/demo-model") + + assert fallback("please use docker") == "use docker" + assert calls[0]["model"] == "openai/demo-model" + assert calls[0]["response_format"]["type"] == "json_schema" # type: ignore[index] + + +def test_free_text_litellm_fallback_probes_and_uses_sentinel(monkeypatch): + calls: list[dict[str, object]] = [] + litellm = types.ModuleType("litellm") + litellm.supports_response_schema = lambda **_: False # type: ignore[attr-defined] + + def completion(**kwargs: object) -> SimpleNamespace: + calls.append(kwargs) + if len(calls) == 1: + raise RuntimeError("response_format is not supported") + return _response("use docker") + + litellm.completion = completion # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "litellm", litellm) + + fallback = create_litellm_fallback("ollama/demo-model") + + assert fallback("please use docker") == "use docker" + assert "response_format" in calls[0] + assert "response_format" not in calls[1] diff --git a/python/tests/test_litellm_proxy_with_directive_drafter.py b/python/tests/test_litellm_proxy_with_directive_drafter.py index 4769b58..aa548b2 100644 --- a/python/tests/test_litellm_proxy_with_directive_drafter.py +++ b/python/tests/test_litellm_proxy_with_directive_drafter.py @@ -9,6 +9,7 @@ import pytest from context_compiler.grammar import decompose_directive from context_compiler_directive_drafter import ( + REASON_INVALID_CANDIDATE, REASON_MULTIPLE_DIRECTIVES, REASON_NON_DIRECTIVE, RejectedDirective, @@ -410,14 +411,14 @@ def test_fallback_returns_raw_candidate_text(monkeypatch) -> None: monkeypatch.setenv("OPENAI_API_KEY", "dummy") monkeypatch.setenv("MODEL", "openai/demo-model") monkeypatch.setattr( - module, - "_get_litellm_completion", - lambda: lambda **_: {"choices": [{"message": {"content": "use docker"}}]}, + module, "create_litellm_fallback", lambda **_: lambda _message: "use docker" ) + module._create_directive_drafter.cache_clear() - result = module._llm_fallback_candidate("please use docker") + result = module._draft_last_user_message("please use docker") - assert result == "use docker" + assert isinstance(result.result, module.CanonicalDirective) + assert result.result.text == "use docker" def test_fallback_returns_raw_no_directive_sentinel(monkeypatch) -> None: @@ -425,14 +426,14 @@ def test_fallback_returns_raw_no_directive_sentinel(monkeypatch) -> None: monkeypatch.setenv("OPENAI_API_KEY", "dummy") monkeypatch.setenv("MODEL", "openai/demo-model") monkeypatch.setattr( - module, - "_get_litellm_completion", - lambda: lambda **_: {"choices": [{"message": {"content": ""}}]}, + module, "create_litellm_fallback", lambda **_: lambda _message: None ) + module._create_directive_drafter.cache_clear() - result = module._llm_fallback_candidate("hello there") + result = module._draft_last_user_message("hello there") - assert result == "" + assert isinstance(result.result, RejectedDirective) + assert result.result.reason == REASON_NON_DIRECTIVE def test_fallback_returns_raw_unknown_candidate_text(monkeypatch) -> None: @@ -441,13 +442,15 @@ def test_fallback_returns_raw_unknown_candidate_text(monkeypatch) -> None: monkeypatch.setenv("MODEL", "openai/demo-model") monkeypatch.setattr( module, - "_get_litellm_completion", - lambda: lambda **_: {"choices": [{"message": {"content": "clear everything"}}]}, + "create_litellm_fallback", + lambda **_: lambda _message: "clear everything", ) + module._create_directive_drafter.cache_clear() - result = module._llm_fallback_candidate("clear everything") + result = module._draft_last_user_message("clear everything") - assert result == "clear everything" + assert isinstance(result.result, RejectedDirective) + assert result.result.reason == REASON_INVALID_CANDIDATE def test_fallback_uses_shared_converter_prompt(monkeypatch) -> None: @@ -456,17 +459,18 @@ def test_fallback_uses_shared_converter_prompt(monkeypatch) -> None: monkeypatch.setenv("MODEL", "openai/demo-model") seen: dict[str, Any] = {} - def completion(**kwargs): + def fallback_factory(**kwargs): seen.update(kwargs) - return {"choices": [{"message": {"content": "use docker"}}]} + return lambda _message: "use docker" - monkeypatch.setattr(module, "_get_litellm_completion", lambda: completion) - monkeypatch.setattr(module, "get_converter_prompt", lambda: "shared prompt") + monkeypatch.setattr(module, "create_litellm_fallback", fallback_factory) + module._create_directive_drafter.cache_clear() - result = module._llm_fallback_candidate("please use docker") + result = module._draft_last_user_message("please use docker") - assert result == "use docker" - assert seen["messages"][0] == {"role": "system", "content": "shared prompt"} + assert isinstance(result.result, module.CanonicalDirective) + assert result.result.text == "use docker" + assert seen["model"] == "openai/demo-model" def test_no_removed_replay_api_remains(monkeypatch) -> None: diff --git a/python/tests/test_litellm_with_directive_drafter.py b/python/tests/test_litellm_with_directive_drafter.py index 91620cc..13028f4 100644 --- a/python/tests/test_litellm_with_directive_drafter.py +++ b/python/tests/test_litellm_with_directive_drafter.py @@ -37,8 +37,8 @@ def step_with_capture(user_input: str): monkeypatch.setattr(engine, "step", step_with_capture) monkeypatch.setattr( module, - "_DIRECTIVE_DRAFTER", - DirectiveDrafter(fallback=lambda _message: "use docker"), + "_get_directive_drafter", + lambda: DirectiveDrafter(fallback=lambda _message: "use docker"), ) result = module.handle_turn( @@ -63,8 +63,8 @@ def step_with_capture(user_input: str): monkeypatch.setattr(engine, "step", step_with_capture) monkeypatch.setattr( module, - "_DIRECTIVE_DRAFTER", - DirectiveDrafter(fallback=lambda _message: "use docker"), + "_get_directive_drafter", + lambda: DirectiveDrafter(fallback=lambda _message: "use docker"), ) llm_calls: list[list[dict[str, str]]] = [] @@ -98,8 +98,8 @@ def step_with_capture(user_input: str): monkeypatch.setattr(engine, "step", step_with_capture) monkeypatch.setattr( module, - "_DIRECTIVE_DRAFTER", - DirectiveDrafter(fallback=lambda _message: None), + "_get_directive_drafter", + lambda: DirectiveDrafter(fallback=lambda _message: None), ) def downstream(messages: list[dict[str, str]]) -> str: @@ -128,8 +128,10 @@ def step_with_capture(user_input: str): monkeypatch.setattr(engine, "step", step_with_capture) monkeypatch.setattr( module, - "_DIRECTIVE_DRAFTER", - DirectiveDrafter(fallback=lambda _message: "use docker and prohibit peanuts"), + "_get_directive_drafter", + lambda: DirectiveDrafter( + fallback=lambda _message: "use docker and prohibit peanuts" + ), ) def downstream(messages: list[dict[str, str]]) -> str: @@ -188,8 +190,8 @@ def should_not_call(messages: list[dict[str, str]]) -> str: monkeypatch.setattr(module, "_call_litellm", should_not_call) monkeypatch.setattr( module, - "_DIRECTIVE_DRAFTER", - DirectiveDrafter(fallback=lambda _message: "use docker"), + "_get_directive_drafter", + lambda: DirectiveDrafter(fallback=lambda _message: "use docker"), ) update_engine = Engine() @@ -199,8 +201,8 @@ def should_not_call(messages: list[dict[str, str]]) -> str: monkeypatch.setattr( module, - "_DIRECTIVE_DRAFTER", - DirectiveDrafter(fallback=lambda _message: None), + "_get_directive_drafter", + lambda: DirectiveDrafter(fallback=lambda _message: None), ) assert update == "State updated." @@ -219,8 +221,8 @@ def downstream(messages: list[dict[str, str]]) -> str: monkeypatch.setattr(module, "_call_litellm", downstream) monkeypatch.setattr( module, - "_DIRECTIVE_DRAFTER", - DirectiveDrafter(fallback=lambda _message: None), + "_get_directive_drafter", + lambda: DirectiveDrafter(fallback=lambda _message: None), ) clarify_engine = Engine() @@ -319,34 +321,40 @@ def test_call_litellm_logs_startup_config_once( def test_preprocessor_model_defaults_to_model(monkeypatch) -> None: seen: dict[str, object] = {} - def completion(**kwargs: Any) -> dict[str, object]: + def fallback_factory(**kwargs: Any): seen.update(kwargs) - return {"choices": [{"message": {"content": "use docker"}}]} + return lambda _message: "use docker" monkeypatch.setenv("OPENAI_API_KEY", "dummy") monkeypatch.setenv("MODEL", "openai/main-model") monkeypatch.delenv("PREPROCESSOR_MODEL", raising=False) - monkeypatch.setattr(module, "_get_litellm_completion", lambda: completion) - monkeypatch.setattr(module, "get_converter_prompt", lambda: "prompt") + monkeypatch.setattr(module, "create_litellm_fallback", fallback_factory) + module._create_directive_drafter.cache_clear() - assert module._llm_fallback_candidate("please use docker") == "use docker" + assert ( + module._get_directive_drafter().draft_directive("please use docker").result.text + == "use docker" + ) assert seen["model"] == "openai/main-model" def test_preprocessor_model_override_wins(monkeypatch) -> None: seen: dict[str, object] = {} - def completion(**kwargs: Any) -> dict[str, object]: + def fallback_factory(**kwargs: Any): seen.update(kwargs) - return {"choices": [{"message": {"content": "use docker"}}]} + return lambda _message: "use docker" monkeypatch.setenv("OPENAI_API_KEY", "dummy") monkeypatch.setenv("MODEL", "openai/main-model") monkeypatch.setenv("PREPROCESSOR_MODEL", "openai/preprocessor-model") - monkeypatch.setattr(module, "_get_litellm_completion", lambda: completion) - monkeypatch.setattr(module, "get_converter_prompt", lambda: "prompt") + monkeypatch.setattr(module, "create_litellm_fallback", fallback_factory) + module._create_directive_drafter.cache_clear() - assert module._llm_fallback_candidate("please use docker") == "use docker" + assert ( + module._get_directive_drafter().draft_directive("please use docker").result.text + == "use docker" + ) assert seen["model"] == "openai/preprocessor-model" @@ -358,17 +366,15 @@ def test_fallback_accepts_structurally_valid_output_without_source_awareness( monkeypatch.delenv("PREPROCESSOR_MODEL", raising=False) monkeypatch.setattr( module, - "_get_litellm_completion", - lambda: ( - lambda **_: { - "choices": [{"message": {"content": "set premise concise replies"}}] - } - ), + "create_litellm_fallback", + lambda **_: lambda _message: "set premise concise replies", ) - monkeypatch.setattr(module, "get_converter_prompt", lambda: "prompt") + module._create_directive_drafter.cache_clear() assert ( - module._llm_fallback_candidate("set premise to concise replies") + module._get_directive_drafter() + .draft_directive("set premise to concise replies") + .result.text == "set premise concise replies" ) @@ -383,19 +389,17 @@ def fallback(_message: str) -> str | None: fallback_calls += 1 return None - monkeypatch.setattr(module, "_llm_fallback_candidate", fallback) monkeypatch.setattr( module, - "_DIRECTIVE_DRAFTER", - DirectiveDrafter( - fallback=module._llm_fallback_candidate, - fallback_source="litellm_fallback", - ), + "_get_directive_drafter", + lambda: DirectiveDrafter(fallback=fallback, fallback_source="litellm_fallback"), ) monkeypatch.setattr(module, "_call_litellm", lambda _messages: "downstream reply") assert module.handle_turn("use docker instead of", Engine()) == "downstream reply" - drafted_result = module._DIRECTIVE_DRAFTER.draft_directive("use docker instead of") + drafted_result = module._get_directive_drafter().draft_directive( + "use docker instead of" + ) assert isinstance(drafted_result.result, RejectedDirective) assert drafted_result.result.reason == REASON_INCOMPLETE assert fallback_calls == 0 @@ -412,8 +416,10 @@ def downstream(_messages: list[dict[str, str]]) -> str: monkeypatch.setattr(module, "_call_litellm", downstream) monkeypatch.setattr( module, - "_DIRECTIVE_DRAFTER", - DirectiveDrafter(fallback=lambda _message: "use docker and prohibit peanuts"), + "_get_directive_drafter", + lambda: DirectiveDrafter( + fallback=lambda _message: "use docker and prohibit peanuts" + ), ) result = module.handle_turn( From 9fa75d2bbbc32423f097c80c4c38859ab1e5c1f9 Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Wed, 2 Sep 2026 20:08:37 -0400 Subject: [PATCH 4/5] test: keep integration tests focused on wiring --- ...st_litellm_proxy_with_directive_drafter.py | 54 +------------------ .../test_litellm_with_directive_drafter.py | 37 +------------ 2 files changed, 4 insertions(+), 87 deletions(-) diff --git a/python/tests/test_litellm_proxy_with_directive_drafter.py b/python/tests/test_litellm_proxy_with_directive_drafter.py index aa548b2..d85acae 100644 --- a/python/tests/test_litellm_proxy_with_directive_drafter.py +++ b/python/tests/test_litellm_proxy_with_directive_drafter.py @@ -9,7 +9,6 @@ import pytest from context_compiler.grammar import decompose_directive from context_compiler_directive_drafter import ( - REASON_INVALID_CANDIDATE, REASON_MULTIPLE_DIRECTIVES, REASON_NON_DIRECTIVE, RejectedDirective, @@ -406,54 +405,7 @@ def test_compound_directives_fall_through_to_normal_forwarding_when_not_applied( assert checkpoint["policies"] == {} -def test_fallback_returns_raw_candidate_text(monkeypatch) -> None: - module = _load_module(monkeypatch, "litellm_proxy_with_drafter_fallback_directive") - monkeypatch.setenv("OPENAI_API_KEY", "dummy") - monkeypatch.setenv("MODEL", "openai/demo-model") - monkeypatch.setattr( - module, "create_litellm_fallback", lambda **_: lambda _message: "use docker" - ) - module._create_directive_drafter.cache_clear() - - result = module._draft_last_user_message("please use docker") - - assert isinstance(result.result, module.CanonicalDirective) - assert result.result.text == "use docker" - - -def test_fallback_returns_raw_no_directive_sentinel(monkeypatch) -> None: - module = _load_module(monkeypatch, "litellm_proxy_with_drafter_fallback_none") - monkeypatch.setenv("OPENAI_API_KEY", "dummy") - monkeypatch.setenv("MODEL", "openai/demo-model") - monkeypatch.setattr( - module, "create_litellm_fallback", lambda **_: lambda _message: None - ) - module._create_directive_drafter.cache_clear() - - result = module._draft_last_user_message("hello there") - - assert isinstance(result.result, RejectedDirective) - assert result.result.reason == REASON_NON_DIRECTIVE - - -def test_fallback_returns_raw_unknown_candidate_text(monkeypatch) -> None: - module = _load_module(monkeypatch, "litellm_proxy_with_drafter_fallback_unknown") - monkeypatch.setenv("OPENAI_API_KEY", "dummy") - monkeypatch.setenv("MODEL", "openai/demo-model") - monkeypatch.setattr( - module, - "create_litellm_fallback", - lambda **_: lambda _message: "clear everything", - ) - module._create_directive_drafter.cache_clear() - - result = module._draft_last_user_message("clear everything") - - assert isinstance(result.result, RejectedDirective) - assert result.result.reason == REASON_INVALID_CANDIDATE - - -def test_fallback_uses_shared_converter_prompt(monkeypatch) -> None: +def test_fallback_adapter_receives_preprocessor_model(monkeypatch) -> None: module = _load_module(monkeypatch, "litellm_proxy_with_drafter_shared_prompt") monkeypatch.setenv("OPENAI_API_KEY", "dummy") monkeypatch.setenv("MODEL", "openai/demo-model") @@ -466,10 +418,8 @@ def fallback_factory(**kwargs): monkeypatch.setattr(module, "create_litellm_fallback", fallback_factory) module._create_directive_drafter.cache_clear() - result = module._draft_last_user_message("please use docker") + module._get_directive_drafter() - assert isinstance(result.result, module.CanonicalDirective) - assert result.result.text == "use docker" assert seen["model"] == "openai/demo-model" diff --git a/python/tests/test_litellm_with_directive_drafter.py b/python/tests/test_litellm_with_directive_drafter.py index 13028f4..7ed0fdd 100644 --- a/python/tests/test_litellm_with_directive_drafter.py +++ b/python/tests/test_litellm_with_directive_drafter.py @@ -7,7 +7,6 @@ from context_compiler_directive_drafter import ( DirectiveDrafter, DraftResult, - REASON_INCOMPLETE, REASON_NON_DIRECTIVE, RejectedDirective, UnknownDirective, @@ -331,10 +330,7 @@ def fallback_factory(**kwargs: Any): monkeypatch.setattr(module, "create_litellm_fallback", fallback_factory) module._create_directive_drafter.cache_clear() - assert ( - module._get_directive_drafter().draft_directive("please use docker").result.text - == "use docker" - ) + module._get_directive_drafter() assert seen["model"] == "openai/main-model" @@ -351,34 +347,10 @@ def fallback_factory(**kwargs: Any): monkeypatch.setattr(module, "create_litellm_fallback", fallback_factory) module._create_directive_drafter.cache_clear() - assert ( - module._get_directive_drafter().draft_directive("please use docker").result.text - == "use docker" - ) + module._get_directive_drafter() assert seen["model"] == "openai/preprocessor-model" -def test_fallback_accepts_structurally_valid_output_without_source_awareness( - monkeypatch, -) -> None: - monkeypatch.setenv("OPENAI_API_KEY", "dummy") - monkeypatch.setenv("MODEL", "openai/main-model") - monkeypatch.delenv("PREPROCESSOR_MODEL", raising=False) - monkeypatch.setattr( - module, - "create_litellm_fallback", - lambda **_: lambda _message: "set premise concise replies", - ) - module._create_directive_drafter.cache_clear() - - assert ( - module._get_directive_drafter() - .draft_directive("set premise to concise replies") - .result.text - == "set premise concise replies" - ) - - def test_directive_shaped_malformed_inputs_can_fall_through_to_normal_turn_flow( monkeypatch, ) -> None: @@ -397,11 +369,6 @@ def fallback(_message: str) -> str | None: monkeypatch.setattr(module, "_call_litellm", lambda _messages: "downstream reply") assert module.handle_turn("use docker instead of", Engine()) == "downstream reply" - drafted_result = module._get_directive_drafter().draft_directive( - "use docker instead of" - ) - assert isinstance(drafted_result.result, RejectedDirective) - assert drafted_result.result.reason == REASON_INCOMPLETE assert fallback_calls == 0 From d93d6b1a36e71a80828e04728ed96e7ff083aa2b Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Wed, 2 Sep 2026 21:46:31 -0400 Subject: [PATCH 5/5] fix: remove obsolete drafter prompt profile --- PROVIDER_CONTRACT.md | 5 ++--- python/reference_integrations/litellm_proxy/README.md | 5 ++--- python/reference_integrations/openwebui_pipe/README.md | 2 -- .../open_webui_pipe_with_directive_drafter.py | 9 +-------- 4 files changed, 5 insertions(+), 16 deletions(-) diff --git a/PROVIDER_CONTRACT.md b/PROVIDER_CONTRACT.md index ad9c66e..70e2304 100644 --- a/PROVIDER_CONTRACT.md +++ b/PROVIDER_CONTRACT.md @@ -16,7 +16,6 @@ export PROVIDER=openai export MODEL=openai/gpt-4o-mini export PREPROCESSOR_MODEL=openai/gpt-4o-mini export OPENAI_BASE_URL=... -export PREPROCESSOR_PROMPT_PROFILE=default ``` Provider mode contract (`PROVIDER`) is strict: @@ -52,5 +51,5 @@ Startup emits one concise config line showing resolved `mode`, `base_url`, `MODEL` and `PREPROCESSOR_MODEL` use LiteLLM format: `/`. `PREPROCESSOR_MODEL` is optional and defaults to `MODEL`. -For heuristic-first usage, keep `PREPROCESSOR_PROMPT_PROFILE=default`. -Use `llama` only for LLM-only preprocessing with Llama-family models. +The directive-drafter integration always uses heuristic-first processing with +the configured fallback model when needed. diff --git a/python/reference_integrations/litellm_proxy/README.md b/python/reference_integrations/litellm_proxy/README.md index 3e4d8e0..1461198 100644 --- a/python/reference_integrations/litellm_proxy/README.md +++ b/python/reference_integrations/litellm_proxy/README.md @@ -202,13 +202,12 @@ Optional env vars for directive-drafter fallback: ```shell export PREPROCESSOR_MODEL=openai/gpt-4o-mini -export PREPROCESSOR_PROMPT_PROFILE=default ``` `PREPROCESSOR_MODEL` is optional and defaults to `MODEL`. -For heuristic-first usage, keep `PREPROCESSOR_PROMPT_PROFILE=default`. -Use `llama` only for LLM-only fallback drafting with Llama-family models. +The directive-drafter integration always uses heuristic-first processing with +the configured fallback model when needed. ## Notes diff --git a/python/reference_integrations/openwebui_pipe/README.md b/python/reference_integrations/openwebui_pipe/README.md index 41323cd..f155371 100644 --- a/python/reference_integrations/openwebui_pipe/README.md +++ b/python/reference_integrations/openwebui_pipe/README.md @@ -69,10 +69,8 @@ If using `open_webui_pipe_with_directive_drafter.py`: - Install directive-drafter support if needed: `pip install "context-compiler>=0.9.0dev13" "context-compiler-directive-drafter>=0.2.0dev5"` -- Set `PREPROCESSOR_PROMPT_PROFILE=default` for heuristic-first behavior - Optionally set `PREPROCESSOR_MODEL_ID` to use a separate fallback drafting model - If `PREPROCESSOR_MODEL_ID` is unset, fallback uses `BASE_MODEL_ID` -- Use `llama` only for LLM-only fallback drafting with Llama-family models Model fallback output is structurally validated before handoff. This does not prove that the model interpreted the user correctly. The automated fallback path is experimental pending a separate source-aware acceptance policy and reviewed drafting workflow. diff --git a/python/reference_integrations/openwebui_pipe/open_webui_pipe_with_directive_drafter.py b/python/reference_integrations/openwebui_pipe/open_webui_pipe_with_directive_drafter.py index 21c76e1..6ad328d 100644 --- a/python/reference_integrations/openwebui_pipe/open_webui_pipe_with_directive_drafter.py +++ b/python/reference_integrations/openwebui_pipe/open_webui_pipe_with_directive_drafter.py @@ -24,7 +24,7 @@ import logging import re from collections.abc import AsyncIterator -from typing import Any, Literal, TypedDict, cast +from typing import Any, TypedDict, cast from fastapi import Request # type: ignore[import-not-found] from open_webui.models.users import Users # type: ignore[import-not-found] @@ -383,10 +383,6 @@ class Valves(BaseModel): "Optional model id for fallback drafting (defaults to BASE_MODEL_ID)." ), ) - PREPROCESSOR_PROMPT_PROFILE: Literal["default", "llama"] = Field( - default="default", - description="Prompt profile for LLM fallback drafting.", - ) ALLOW_MISSING_BASE_MODEL_FOR_DEBUG: bool = Field( default=False, description="Allow missing BASE_MODEL_ID for debug/testing only.", @@ -697,10 +693,8 @@ async def _preprocess_user_input( *, request: Request, user_payload: dict[str, Any], - prompt_profile: str, model_id: str | None, ) -> tuple[DraftResult, str | None]: - del prompt_profile self._last_preprocessor_error = None drafted_result = await self._draft_user_input( message, @@ -894,7 +888,6 @@ async def pipe( latest_user_text, request=__request__, user_payload=__user__, - prompt_profile=self.valves.PREPROCESSOR_PROMPT_PROFILE, model_id=effective_preprocessor_model, ) if preprocess_error is not None: