diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 894fecc9a..46bc98b40 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -122,12 +122,28 @@ jobs: - name: Start serving ollama run: nohup ollama serve & - name: Pull models + # granite-vision-4.1 is not in the Ollama library, so it is pulled from + # IBM's official GGUF repo on Hugging Face (that build ships the mmproj + # projector Ollama needs for image input). Drop the hf.co prefix if the + # model is ever published to the Ollama library directly. run: | - for i in 1 2 3 4 5; do - ollama pull granite4.1:3b && break - echo "Attempt $i failed, retrying in 20s..." - sleep 20 + for model in granite4.1:3b hf.co/ibm-granite/granite-vision-4.1-4b-GGUF:Q4_K_M; do + pulled=false + for i in 1 2 3 4 5; do + ollama pull "$model" && { pulled=true; break; } + echo "Attempt $i to pull $model failed, retrying in 20s..." + sleep 20 + done + # Without this the loop exits 0 after exhausting every attempt: the + # `&&` above puts the failure in a tested context, so `bash -e` does + # not fire, and the loop's status comes from `sleep`. The step would + # go green and the miss would surface 20 minutes later as a test + # failure instead of an infra one. + $pulled || { echo "::error::failed to pull $model after 5 attempts"; exit 1; } done + # The names below are what the tests match against; print them so a + # tag-name mismatch is diagnosable from the log alone. + ollama list - name: Run Tests id: tests env: diff --git a/docs/docs/how-to/use-images-and-vision.md b/docs/docs/how-to/use-images-and-vision.md index 00697bad2..8a4cb57e7 100644 --- a/docs/docs/how-to/use-images-and-vision.md +++ b/docs/docs/how-to/use-images-and-vision.md @@ -11,8 +11,9 @@ Mellea supports multimodal input: pass images alongside your text prompt to any running. > **Backend note:** The default Ollama model (`granite4.1:3b`) does not support image -> input. You must switch to a vision-capable model such as `granite3.2-vision` or -> `llava`. Not all backends support vision — see backend notes below. +> input. You must switch to a vision-capable model such as `granite3.2-vision`, or +> `granite-vision-4.1` via the Hugging Face tag shown below. Not all backends support +> vision — see backend notes below. --- @@ -38,6 +39,40 @@ print(str(result)) Other vision-capable Ollama models: `llava`, `llava-phi3`, `moondream`, `qwen2.5vl:7b`. +### Granite Vision 4.1 + +`granite-vision-4.1` is the current-generation Granite vision model. It is not in the +Ollama library, so pull IBM's official GGUF build from Hugging Face instead — that +build ships the `mmproj` projector Ollama needs for image input: + +```bash +ollama pull hf.co/ibm-granite/granite-vision-4.1-4b-GGUF:Q4_K_M +``` + +Reference it with the `IBM_GRANITE_VISION_4_1_4B` constant rather than typing the tag: + +```python +# Requires: mellea, pillow +# Returns: str +from PIL import Image +from mellea import start_session +from mellea.backends import ModelOption, model_ids + +m = start_session( + model_id=model_ids.IBM_GRANITE_VISION_4_1_4B, + model_options={ModelOption.CONTEXT_WINDOW: 4096}, +) + +img = Image.open("photo.jpg") +result = m.instruct("Is the subject in this image smiling?", images=[img]) +print(str(result)) +# Output will vary — LLM responses depend on model and temperature. +``` + +Setting `CONTEXT_WINDOW` explicitly is worth the line. Ollama otherwise allocates the +model's full 131072-token window — close to 9 GB — for a request that needs a few +thousand tokens. Capping it at 4096 brings that down to roughly 2.5 GB. + --- ## Using ImageBlock for explicit control diff --git a/docs/docs/integrations/ollama.md b/docs/docs/integrations/ollama.md index 5f35443f9..006b69e2e 100644 --- a/docs/docs/integrations/ollama.md +++ b/docs/docs/integrations/ollama.md @@ -88,6 +88,7 @@ ollama pull mistral:7b | `IBM_GRANITE_4_1_8B` | `granite4.1:8b` | Higher quality, ~5 GB. | | `IBM_GRANITE_3_3_8B` | `granite3.3:8b` | Higher quality, ~5 GB. | | `IBM_GRANITE_3_3_VISION_2B` | `ibm/granite3.3-vision:2b` | Vision model for image inputs. | +| `IBM_GRANITE_VISION_4_1_4B` | `hf.co/ibm-granite/granite-vision-4.1-4b-GGUF:Q4_K_M` | Current-generation vision model, pulled from Hugging Face. | | `META_LLAMA_3_2_3B` | `llama3.2:3b` | Compact Llama model. | | `MISTRALAI_MISTRAL_0_3_7B` | `mistral:7b` | Mistral 7B. | | `QWEN3_8B` | `qwen3:8b` | Qwen3 8B. | @@ -180,7 +181,14 @@ or `chat()` apply to that call only and take precedence. ## Vision models Ollama hosts vision-capable models. Use `IBM_GRANITE_3_3_VISION_2B` or any Ollama -vision model via the OpenAI-compatible endpoint: +vision model via the OpenAI-compatible endpoint. + +For the current-generation Granite vision model, use `IBM_GRANITE_VISION_4_1_4B`. It is +not in the Ollama library, so pull IBM's official GGUF build from Hugging Face first — +`ollama pull hf.co/ibm-granite/granite-vision-4.1-4b-GGUF:Q4_K_M` — and cap +`ModelOption.CONTEXT_WINDOW` (its full 131072-token window allocates close to 9 GB). +See [Use Images and Vision Models](../how-to/use-images-and-vision.md) for a worked +example. ```python # Requires: mellea, pillow diff --git a/docs/examples/image_text_models/README.md b/docs/examples/image_text_models/README.md index faa08d1cd..0daec6f9a 100644 --- a/docs/examples/image_text_models/README.md +++ b/docs/examples/image_text_models/README.md @@ -55,7 +55,10 @@ response = m.chat( ## Supported Models -- **Ollama**: granite3.2-vision, llava, bakllava, llava-phi3, moondream, qwen2.5vl:7b +- **Ollama**: granite3.2-vision, llava, bakllava, llava-phi3, moondream, + qwen2.5vl:7b, and granite-vision-4.1 as + `hf.co/ibm-granite/granite-vision-4.1-4b-GGUF:Q4_K_M` (pulled from Hugging + Face rather than the Ollama library, so the full tag is required) - **OpenAI**: gpt-4-vision-preview, gpt-4o - **LiteLLM**: Various vision models through unified interface @@ -66,9 +69,13 @@ Pull a vision-capable model before running these examples: ```bash ollama pull granite3.2-vision # ~2.4 GB — primary recommended model ollama pull qwen2.5vl:7b # ~4.7 GB — used in vision_openai_examples.py + +# granite-vision-4.1 is the current-generation Granite vision model. It is not in +# the Ollama library, so pull IBM's official GGUF build from Hugging Face: +ollama pull hf.co/ibm-granite/granite-vision-4.1-4b-GGUF:Q4_K_M # ~3.3 GB ``` ## Related Documentation - See `test/backends/test_vision_*.py` for more examples -- See `mellea/stdlib/components/chat.py` for Message API \ No newline at end of file +- See `mellea/stdlib/components/chat.py` for Message API diff --git a/mellea/backends/model_ids.py b/mellea/backends/model_ids.py index c5aabe5dc..06ab71b09 100644 --- a/mellea/backends/model_ids.py +++ b/mellea/backends/model_ids.py @@ -151,6 +151,16 @@ class ModelIdentifier: context_length=131072, ) +# Granite 4.1 Vision Model (4B). Not in the Ollama library; the `hf.co/...` tag +# pulls IBM's official GGUF build (which ships the `mmproj` projector needed for +# image input) straight from Hugging Face. +IBM_GRANITE_VISION_4_1_4B = ModelIdentifier( + hf_model_name="ibm-granite/granite-vision-4.1-4b", + ollama_name="hf.co/ibm-granite/granite-vision-4.1-4b-GGUF:Q4_K_M", + watsonx_name=None, + context_length=131072, +) + IBM_GRANITE_GUARDIAN_3_0_2B = ModelIdentifier( hf_model_name="ibm-granite/granite-guardian-3.0-2b", ollama_name="granite3-guardian:2b", diff --git a/test/backends/test_vision_ollama.py b/test/backends/test_vision_ollama.py index 3838cc937..c8042afe3 100644 --- a/test/backends/test_vision_ollama.py +++ b/test/backends/test_vision_ollama.py @@ -9,10 +9,13 @@ 2. **Structural payload** (unit, mocked) — verify mellea correctly embeds images into the Ollama conversation payload. The Ollama transport is mocked so no server or vision model is needed. Runs in CI unconditionally. -3. **Dormant live e2e** (e2e, qualitative) — full round-trip against a real - vision-capable Ollama model. Skipped today because granite-vision-4.1 is not - yet in the Ollama library. Reactivates automatically once the model is pulled. - See #1187 for the activation checklist. +3. **Live e2e** (e2e) — full round-trip against a real vision-capable Ollama + model. The assertions are structural (a thunk/message with non-empty content), + so these are `e2e` and not `qualitative`: no assertion here can be broken by + swapping the model version, and marking them `qualitative` would exclude them + from CI, where `CICD=1` skips that tier. Requires `granite-vision-4.1` to be + pulled; skipped locally with a pull command if it is not, and failed in CI, + where the workflow is responsible for pulling it. """ import base64 @@ -27,6 +30,7 @@ from mellea import MelleaSession from mellea.backends import ModelOption +from mellea.backends.model_ids import IBM_GRANITE_VISION_4_1_4B from mellea.core import ( AudioBlock, AudioUrlBlock, @@ -36,14 +40,22 @@ ) from mellea.stdlib.components import Instruction, Message -# Ollama name for the target vision model; bump to the model_ids constant once -# IBM_GRANITE_VISION_4_1_4B is added to mellea/backends/model_ids.py. -_VISION_MODEL = "granite-vision-4.1" +# granite-vision-4.1 is not in the Ollama library, so the model is pulled from +# IBM's official GGUF repo on Hugging Face. Match on the full tag rather than a +# bare name so the check cannot be satisfied by an unrelated local alias. +_VISION_MODEL = IBM_GRANITE_VISION_4_1_4B +# Asserted rather than coerced with str(): ollama_name is `str | None`, and +# str(None) would silently make the tag the literal "None". +assert IBM_GRANITE_VISION_4_1_4B.ollama_name is not None +_VISION_MODEL_TAG = IBM_GRANITE_VISION_4_1_4B.ollama_name _SKIP_REASON = ( - f"Vision model {_VISION_MODEL!r} not available in Ollama — " - "see https://github.com/generative-computing/mellea/issues/1187" + f"Vision model not pulled locally — run `ollama pull {_VISION_MODEL_TAG}`" ) +# The model's full 131072-token context window loads ~9 GB for a job that needs a +# few thousand tokens; capping it keeps the live tests near 2.5 GB. +_VISION_CONTEXT_WINDOW = 4096 + # ── Shared image fixture ────────────────────────────────────────────────────── @@ -294,18 +306,15 @@ def test_image_block_in_chat(mocked_session: MelleaSession, pil_image: Image.Ima assert image_list[0] == str(image_block) -# ── Tier 3: Dormant live e2e ────────────────────────────────────────────────── -# -# Full round-trip against a real vision-capable Ollama model. Currently skipped -# because granite-vision-4.1 is not yet in the Ollama library. +# ── Tier 3: Live e2e ────────────────────────────────────────────────────────── # -# To activate: ensure `ollama pull granite-vision-4.1` succeeds, then remove the -# pytest.skip() call from vision_session below and add the model to the CI pull -# step in .github/workflows/quality.yml. See #1187 for the full checklist. +# Full round-trip against a real vision-capable Ollama model. CI pulls the model +# in the "Pull models" step of .github/workflows/quality.yml; locally these skip +# unless you have pulled it yourself (the skip message gives the command). -def _ollama_vision_model_available() -> bool: - """Return True if _VISION_MODEL is present in the local Ollama model list.""" +def _pulled_ollama_models() -> set[str]: + """Return the model names Ollama reports locally; empty if it is unreachable.""" import requests host = os.environ.get("OLLAMA_HOST", "127.0.0.1") @@ -317,21 +326,51 @@ def _ollama_vision_model_available() -> bool: try: resp = requests.get(f"{base_url}/api/tags", timeout=5) resp.raise_for_status() - pulled = {m.get("name", "") for m in resp.json().get("models", [])} - return any(_VISION_MODEL in name for name in pulled) + return {m.get("name", "") for m in resp.json().get("models", [])} except Exception: - return False + return set() + + +def _vision_model_pulled(pulled: set[str]) -> bool: + """Return True if the exact vision model tag is among the pulled model names.""" + return _VISION_MODEL_TAG in pulled + + +def test_vision_model_tag_matches_the_pulled_tag(): + """Only the exact pinned tag counts; anything else may be a different model.""" + assert _vision_model_pulled({_VISION_MODEL_TAG}) + assert not _vision_model_pulled(set()) + assert not _vision_model_pulled({"granite4.1:3b", "granite3.2-vision:latest"}) + # A bare alias of that name could point anywhere, and a longer tag sharing + # the prefix is a different model -- neither counts. + assert not _vision_model_pulled({"granite-vision-4.1:latest"}) + assert not _vision_model_pulled({f"{_VISION_MODEL_TAG}-extra"}) @pytest.fixture -def vision_session(): - if not _ollama_vision_model_available(): +def vision_session(gh_run: int): + pulled = _pulled_ollama_models() + if not _vision_model_pulled(pulled): + if gh_run: + # CI pulls this model in the "Pull models" step, so a miss here is a real + # failure -- either the pull did not happen or Ollama reports the model + # under a name this check does not recognise. Skipping would hide both + # behind a green tick. + pytest.fail( + f"{_SKIP_REASON}\nOllama reported: " + f"{sorted(pulled) if pulled else ''}" + ) pytest.skip(_SKIP_REASON) from mellea import start_session m = start_session( - "ollama", model_id=_VISION_MODEL, model_options={ModelOption.MAX_NEW_TOKENS: 5} + "ollama", + model_id=_VISION_MODEL, + model_options={ + ModelOption.MAX_NEW_TOKENS: 5, + ModelOption.CONTEXT_WINDOW: _VISION_CONTEXT_WINDOW, + }, ) yield m del m @@ -339,11 +378,10 @@ def vision_session(): @pytest.mark.e2e @pytest.mark.ollama -@pytest.mark.qualitative def test_vision_instruct_live_e2e( vision_session: MelleaSession, pil_image: Image.Image ): - """Live vision instruct round-trip; skips until granite-vision-4.1 lands on Ollama.""" + """Live vision instruct round-trip against granite-vision-4.1.""" image_block: ImageBlock | ImageUrlBlock = ImageBlock.from_pil_image(pil_image) instr = vision_session.instruct( "Is this image mainly blue? Answer yes or no.", @@ -357,9 +395,8 @@ def test_vision_instruct_live_e2e( @pytest.mark.e2e @pytest.mark.ollama -@pytest.mark.qualitative def test_vision_chat_live_e2e(vision_session: MelleaSession, pil_image: Image.Image): - """Live vision chat round-trip; skips until granite-vision-4.1 lands on Ollama.""" + """Live vision chat round-trip against granite-vision-4.1.""" ct = vision_session.chat( "Is this image mainly blue? Answer yes or no.", images=[pil_image] )