diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 341b3d8..5bc911d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Contributing to granite-switch-systemone +# Contributing to granite-switch-decision ## Getting Started @@ -6,8 +6,8 @@ 2. Clone your fork and install the development dependencies: ```bash - git clone https://github.com//granite-switch-systemone.git - cd granite-switch-systemone + git clone https://github.com//granite-switch-decision.git + cd granite-switch-decision pip install -e ".[dev]" ``` diff --git a/Dockerfile b/Dockerfile index b5092fb..ec3514a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,11 +2,11 @@ # /v1/decisions and /v1/systemone plugin routes, on one engine and one KV cache. # The checkpoint is not in the image; mount it at $MODEL_DIR (default /repository). # -# docker build -t /granite-switch-systemone: . +# docker build -t /granite-switch-decision: . # # The audio libraries are in every build. WITH_ASR=1 also bakes in the speech model (see -# ASR_MODEL and ASR_REVISION); serve.sh keeps speech off unless SYSTEMONE_SPEECH_INPUT=1. -# docker build --build-arg WITH_ASR=1 -t /granite-switch-systemone:-asr . +# ASR_MODEL and ASR_REVISION); serve.sh keeps speech off unless DECISION_SPEECH_INPUT=1. +# docker build --build-arg WITH_ASR=1 -t /granite-switch-decision:-asr . # # vLLM 0.30 is the version tested live. The default tag is the CUDA 13 build (torch # 2.13.0+cu130), the stack the tests ran. The -cu129 tag ships torch 2.14.0, which the @@ -42,17 +42,17 @@ RUN if [ "$WITH_ASR" = "1" ]; then \ fi ENV HF_HUB_CACHE=/opt/hf-cache HF_HUB_OFFLINE=1 -# The plugin: a vllm.endpoint_plugins entry point (see src/granite_switch_systemone/plugin.py). -COPY pyproject.toml README.md /opt/granite-switch-systemone/ -COPY src /opt/granite-switch-systemone/src -RUN uv pip install --system --no-deps /opt/granite-switch-systemone +# The plugin: a vllm.endpoint_plugins entry point (see src/granite_switch_decision/plugin.py). +COPY pyproject.toml README.md /opt/granite-switch-decision/ +COPY src /opt/granite-switch-decision/src +RUN uv pip install --system --no-deps /opt/granite-switch-decision COPY serve.sh /usr/local/bin/serve.sh RUN chmod +x /usr/local/bin/serve.sh # Endpoint plugins load only when VLLM_PLUGINS names them, and setting it stops vLLM loading # any others, so register_granite_switch has to be listed too. -ENV VLLM_PLUGINS=register_granite_switch,granite_switch_systemone \ - SYSTEMONE_ADAPTER_NAME=decision \ - SYSTEMONE_INVOCATION_MARKER="### DECISION" +ENV VLLM_PLUGINS=register_granite_switch,granite_switch_decision \ + DECISION_ADAPTER_NAME=decision \ + DECISION_INVOCATION_MARKER="### DECISION" EXPOSE 8000 ENTRYPOINT ["/usr/local/bin/serve.sh"] diff --git a/README.md b/README.md index 3df409e..9f8bf68 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# granite-switch-systemone +# granite-switch-decision Decision APIs for [Granite Switch](https://github.com/generative-computing/granite-switch), as a vLLM plugin. Ask a question about some text and get back probabilities, from the same server and @@ -22,7 +22,7 @@ You need a Linux machine with an NVIDIA GPU. The 8b checkpoint fits on one 80 GB ### Container Release images are published to GitHub Container Registry as -`ghcr.io/generative-computing/granite-switch-systemone` for linux/amd64 and linux/arm64, so one tag +`ghcr.io/generative-computing/granite-switch-decision` for linux/amd64 and linux/arm64, so one tag works on both. `latest` is the newest release, and each release is also tagged with its version. The pull is about 9 GB and the image takes about 30 GB on disk (amd64 figures). You also need Docker and the NVIDIA container toolkit. @@ -33,7 +33,7 @@ hf download ibm-granite/granite-switch-4.1-8b --local-dir /srv/models/granite-sw # Run the image docker run -d --name gss --gpus all --ipc=host -p 8000:8000 \ -v /srv/models/granite-switch-4.1-8b:/repository:ro \ - ghcr.io/generative-computing/granite-switch-systemone:0.1.3 + ghcr.io/generative-computing/granite-switch-decision:0.2.0 ``` Loading takes a few minutes; follow it with `docker logs -f gss`. `--ipc=host` is vLLM's usual @@ -41,7 +41,7 @@ setting for shared memory, and it matters most when you run with tensor parallel The ghcr package is currently private, so `docker pull` needs `docker login ghcr.io` with an account that has access. Until it is public, build the image yourself: clone this repository at a release tag -and run `docker build -t granite-switch-systemone .`. +and run `docker build -t granite-switch-decision .`. The image is `vllm/vllm-openai:v0.30.0` plus `granite-switch[audio]==0.1.1`, `transformers==5.17.0` and this plugin. It has no weights in it. It reads the checkpoint from `/repository` (override with @@ -56,9 +56,9 @@ and `v0.1.0` was also served on 0.26 to 0.29, but nothing since has been checked ```bash pip install "granite-switch[vllm]==0.1.1" "transformers>=5.16" -pip install "git+https://github.com/generative-computing/granite-switch-systemone@v0.1.3" -export VLLM_PLUGINS=register_granite_switch,granite_switch_systemone -export SYSTEMONE_INVOCATION_MARKER="### DECISION" +pip install "git+https://github.com/generative-computing/granite-switch-decision@v0.2.0" +export VLLM_PLUGINS=register_granite_switch,granite_switch_decision +export DECISION_INVOCATION_MARKER="### DECISION" vllm serve ibm-granite/granite-switch-4.1-8b --served-model-name granite-switch \ --dtype bfloat16 --attention-config.flash_attn_version=2 --enable-prompt-tokens-details ``` @@ -88,7 +88,7 @@ python3 scripts/e2e_check.py --base-url http://localhost:8000 `e2e_check.py` needs only the standard library and works against any host, so run it from a checkout of this repository. It exits 0 when the wire formats on both routes, route parity, chat and prefix-cache sharing all pass. If the server stops at startup, its log names the cause: an unknown -`SYSTEMONE_ADAPTER_NAME`, a checkpoint with no `decision` adapter, or an invocation marker that does not +`DECISION_ADAPTER_NAME`, a checkpoint with no `decision` adapter, or an invocation marker that does not activate the adapter in the chat template. ## Make a request @@ -115,7 +115,7 @@ The server listens on `0.0.0.0` and, as shipped, asks for no credentials on any gateway that authenticates callers, or turn on vLLM's `--api-key`, which `serve.sh` passes through: ```bash -docker run -d --gpus all --ipc=host -p 8000:8000 -v … granite-switch-systemone --api-key "$KEY" +docker run -d --gpus all --ipc=host -p 8000:8000 -v … granite-switch-decision --api-key "$KEY" ``` In vLLM 0.30.0 that key check covers every `/v1` route, including `/v1/decisions` and @@ -123,7 +123,7 @@ In vLLM 0.30.0 that key check covers every `/v1` route, including `/v1/decisions have not run it against a live server, so after enabling it, confirm that an unauthenticated `curl` to `/v1/decisions` returns 401. -If the server also handles chat, set `SYSTEMONE_MAX_CONCURRENT_REQUESTS` (8 to 32 is a sensible +If the server also handles chat, set `DECISION_MAX_CONCURRENT_REQUESTS` (8 to 32 is a sensible range). It is off by default, and without it a burst of decisions slowed chat p50 from 235 ms to 1,088 ms in our load test (see [docs/STATUS.md](docs/STATUS.md)). @@ -134,14 +134,14 @@ speech input in their configuration. Loading a speech model slows startup and de so `serve.sh` switches it off (`--hf-overrides '{"asr_enabled": false}'`). The checkpoint then starts with nothing else to download, and `/v1/chat/completions` accepts no audio. -To turn it on, set `SYSTEMONE_SPEECH_INPUT=1`. The image runs with the Hugging Face Hub offline, so the +To turn it on, set `DECISION_SPEECH_INPUT=1`. The image runs with the Hugging Face Hub offline, so the speech model ([`ibm-granite/granite-speech-5.0-470m-turboctc`](https://huggingface.co/ibm-granite/granite-speech-5.0-470m-turboctc), about 1 GB) has to be available. Any one of these works: - Use an image with the model built in: the `-asr` tag of a release (releases made with the current workflow publish one), or build it yourself: - `docker build --build-arg WITH_ASR=1 -t granite-switch-systemone:asr .` + `docker build --build-arg WITH_ASR=1 -t granite-switch-decision:asr .` - Mount a Hub cache that holds the model and set `HF_HUB_CACHE` to that directory. - Set `HF_HUB_OFFLINE=0` so the model downloads at startup. @@ -168,12 +168,12 @@ name is passed on to `vllm serve`. | `MAX_MODEL_LEN` | `32768` | Longest prompt plus output. | | `GPU_MEMORY_UTILIZATION` | `0.90` | Share of GPU memory vLLM may use. | | `TENSOR_PARALLEL_SIZE` | every visible GPU | GPUs to split the model across. | -| `SYSTEMONE_ADAPTER_NAME` | the only adapter, else `decision` | Which adapter answers. Checked against the checkpoint at startup; an unknown name stops the server. | -| `SYSTEMONE_INVOCATION_MARKER` | none (`### DECISION` in the image) | The text the adapter was trained to activate on. It is required: with a wrong marker the chat template activates the adapter in the wrong place without raising an error. | -| `SYSTEMONE_SPEECH_INPUT` | `0` | `1` keeps speech input; see [Speech input](#speech-input-optional). | -| `SYSTEMONE_MAX_CONCURRENT_REQUESTS` | `0` (no cap) | Decision requests allowed in flight across both routes. Past the cap a request gets 503 at once, so a burst cannot queue ahead of chat. | -| `SYSTEMONE_RENDERER` | `granite_switch_systemone.renderer:render` | A `module:function` that builds the prompt and label ids, for an adapter trained differently. | -| `SYSTEMONE_EXPERIMENTAL_SYSTEM_PROMPT` | `0` | Experimental, may be removed. `1` adds the trained decision-engine system prompt, as used for the model card's published score. It stops chat requests sharing the decision prefix cache. | +| `DECISION_ADAPTER_NAME` | the only adapter, else `decision` | Which adapter answers. Checked against the checkpoint at startup; an unknown name stops the server. | +| `DECISION_INVOCATION_MARKER` | none (`### DECISION` in the image) | The text the adapter was trained to activate on. It is required: with a wrong marker the chat template activates the adapter in the wrong place without raising an error. | +| `DECISION_SPEECH_INPUT` | `0` | `1` keeps speech input; see [Speech input](#speech-input-optional). | +| `DECISION_MAX_CONCURRENT_REQUESTS` | `0` (no cap) | Decision requests allowed in flight across both routes. Past the cap a request gets 503 at once, so a burst cannot queue ahead of chat. | +| `DECISION_RENDERER` | `granite_switch_decision.renderer:render` | A `module:function` that builds the prompt and label ids, for an adapter trained differently. | +| `DECISION_EXPERIMENTAL_SYSTEM_PROMPT` | `0` | Experimental, may be removed. `1` adds the trained decision-engine system prompt, as used for the model card's published score. It stops chat requests sharing the decision prefix cache. | Both routes share these settings; nothing is configured per route. @@ -201,7 +201,7 @@ caching stays at vLLM's default, which is on. On every decision request the plugin itself sets `temperature=0.0`, `max_completion_tokens=1`, `logprob_token_ids` (the option labels' token ids, read exactly rather than from the top-K) and -`chat_template_kwargs={"adapter_name": }`. Callers cannot change them. vLLM +`chat_template_kwargs={"adapter_name": }`. Callers cannot change them. vLLM caps `logprob_token_ids` at 128 ids, so a `choice` with more options runs as several requests over the same prompt, served from the prefix cache. @@ -249,7 +249,7 @@ absent. You can drop it once you move to a release that has them. The upstream e details: this plugin follows jev's format and OpenAI's schema as published. The plugin refuses to start if either path is already registered, so on a vLLM that has the upstream -endpoints it fails at startup and does not silently pick one. Remove `granite_switch_systemone` from +endpoints it fails at startup and does not silently pick one. Remove `granite_switch_decision` from `VLLM_PLUGINS` there. vLLM 0.32 mounts its own `/v1/systemone` (newer versions add `/v1/decisions`). Neither knows Switch adapters, and granite-switch 0.1.1 does not support those versions yet. @@ -263,7 +263,7 @@ migration is a Granite Switch read strategy, and `renderer.render(tok, state, in The pip path is what most of the live testing used. The container image is lightly tested: it builds for amd64 and arm64, with and without the speech model, and the packages and plugin entry points were checked -inside it. The published `0.1.3` image was served on H100 GPUs with only the checkpoint mounted and +inside it. The published `0.1.3` image (then named `granite-switch-systemone`) was served on H100 GPUs with only the checkpoint mounted and speech input off, the 8b on one GPU and the 30b on two with tensor parallelism, and the wire, robustness and accuracy checks passed. That was run under enroot and not with `docker run`. The arm64 image was also served with `docker run` on an NVIDIA GB10 (the 8b, default image). The arm64 `-asr` image was also served with `docker run` and passed the speech check; see [docs/STATUS.md](docs/STATUS.md) for exactly what was and was not run. If you run the image, treat the diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index d1397c9..820eae2 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -9,7 +9,7 @@ licence shown. (`src/system_one_adapter/_utils/confidence_metrics.py` and `tests/utils/test_confidence_metrics.py`). - Used in: `_choice_confidence`, `_score_confidence` and `_normalize_probs` in - `src/granite_switch_systemone/plugin.py` (adapted), and the confidence test vectors in + `src/granite_switch_decision/plugin.py` (adapted), and the confidence test vectors in `tests/test_plugin.py`. ``` diff --git a/ci/check_headers.py b/ci/check_headers.py index 76b1338..d11095c 100644 --- a/ci/check_headers.py +++ b/ci/check_headers.py @@ -20,7 +20,7 @@ DEFAULT_HEADER = "# SPDX-License-Identifier: Apache-2.0" # These two carry MIT-adapted code (see THIRD_PARTY_NOTICES.md) and are headed # "Apache-2.0 AND MIT"; the script checks for a header, never assumes which. -AND_MIT_FILES = {"src/granite_switch_systemone/plugin.py", "tests/test_plugin.py"} +AND_MIT_FILES = {"src/granite_switch_decision/plugin.py", "tests/test_plugin.py"} def has_header(path: pathlib.Path) -> bool: diff --git a/docs/STATUS.md b/docs/STATUS.md index 91e36a1..ee1e27f 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -2,38 +2,42 @@ How to run each test is in [TESTING.md](TESTING.md). -Last updated 2026-10-23. "Live" means a real server on H100s: granite-switch 0.1.1, +Last updated 2026-10-25. "Live" means a real server on H100s: granite-switch 0.1.1, transformers 5.17, vLLM 0.26.0 / 0.27.1 / 0.28.0 / 0.29.0 / 0.30.0 as noted per row, and `ibm-granite/granite-switch-4.1-8b` unless a row names another checkpoint. +Renamed in 0.2.0 (`granite-switch-systemone` to `granite-switch-decision`, `SYSTEMONE_*` to `DECISION_*`). +The rows below were run at 0.1.3 or earlier under the old names. After the rename we repeated only the unit +and spec suites and the 8b battery (`e2e_check`, `probe_check`, `accuracy_check`, pip install, one H100). + ## Tested | Area | How | Result | |---|---|---| | Unit and route tests | `pytest`, engine stubbed, in CI | pass | -| `/v1/decisions` against OpenAI's OpenAPI schema | `tests/test_openai_spec.py` (42 cases), plus the same request set live | pass, with the exceptions listed in the README | +| `/v1/decisions` against OpenAI's OpenAPI schema | `tests/test_openai_spec.py` (41 cases), plus the same request set live | pass, with the exceptions listed in the README | | `/v1/systemone` against jev's wire format | `tests/test_systemone_spec.py` (24 cases) against `typesafe-sdk`'s generated models, stubbed | pass; the plugin also rejects unknown fields, which jev's format allows | | Live wire battery | `scripts/e2e_check.py` (11 checks) | 11/11 | | Prompt rendering | server prompt token counts against an offline render with the checkpoint's tokenizer and chat template | identical | | Prefix-cache sharing | `tests/test_prefix_sharing.py`, and `cached_tokens` live, cross-checked with `/metrics` | as documented in the README | | Route parity | 800 labelled items through `/v1/decisions` and through the same prompt over chat completions | no item answers differently | | Escaping, size guard, disconnect cancellation, error shapes | live probes, now `scripts/probe_check.py` | as documented in the README | -| Question cap and `SYSTEMONE_MAX_CONCURRENT_REQUESTS` | live: 200 questions accepted, 201 rejected; a 32-request burst against a cap of 8 gave 8 accepted and 24 rejected with 503 | pass | -| `SYSTEMONE_EXPERIMENTAL_SYSTEM_PROMPT=1` | live: the 800-item comparison and `e2e_check.py --system-prompt` | answers identical to the same prompt over chat completions; decisions about one state still share the cache, chat no longer does | +| Question cap and `DECISION_MAX_CONCURRENT_REQUESTS` | live: 200 questions accepted, 201 rejected; a 32-request burst against a cap of 8 gave 8 accepted and 24 rejected with 503 | pass | +| `DECISION_EXPERIMENTAL_SYSTEM_PROMPT=1` | live: the 800-item comparison and `e2e_check.py --system-prompt` | answers identical to the same prompt over chat completions; decisions about one state still share the cache, chat no longer does | | FlashAttention 3 | FA2 and FA3 servers side by side: the 800-item comparison, 40 cached one-token reads (the case in granite-switch#139), load at 8 and 32 clients | no difference beyond batching noise in answers or speed; FA2 kept in `serve.sh` while #139 is open | | Load | 1 to 128 concurrent clients, three-question requests | about 140 requests/s plateau, no errors | | Plugin-only install | the README's steps in a fresh environment, then its `vllm serve` command and the e2e battery | pass | -| Install from this repository | `pip install git+https://github.com/generative-computing/granite-switch-systemone` into a fresh venv, over HTTPS and SSH with authenticated access while the repository was private, with the entry-point check and the unit and spec suites against the installed copy from a neutral directory (184 tests, pass); then anonymously (no git config, credentials or SSH agent) once public, at v0.1.2 | the anonymous install resolved and registered the `vllm.endpoint_plugins` entry point; the suites were not rerun on it | +| Install from this repository | `pip install git+https://github.com/generative-computing/granite-switch-systemone` (the repository's name then) into a fresh venv, over HTTPS and SSH with authenticated access while the repository was private, with the entry-point check and the unit and spec suites against the installed copy from a neutral directory (184 tests, pass); then anonymously (no git config, credentials or SSH agent) once public, at v0.1.2 | the anonymous install resolved and registered the `vllm.endpoint_plugins` entry point; the suites were not rerun on it | | vLLM 0.26.0, 0.27.1, 0.28.0, 0.29.0 live (at `v0.1.0` only; not repeated for later changes) | one server per version on one H100 (torch 2.11.0+cu130 on 0.26, torch 2.13.0+cu130 on the rest): the e2e battery, `probe_check`, `test_decisions_e2e`, `test_prefix_sharing`, and the unit suite under each version's own venv | 11/11 e2e and every suite passes on all four versions | | The 30b checkpoint | `granite-switch-4.1-30b` on vLLM 0.30.0, `--tensor-parallel-size 2` on two H100s: the full battery | pass | | Tensor parallelism | the 8b with `--tensor-parallel-size 2`: the full battery | pass | | Long contexts | the 8b with `--max-model-len 131072`, `scripts/longctx_check.py` (9 checks) | 9/9: a 98k-token request answered in one engine read with the danger-zone header and rejected without it; the 131,072 backstop and the default 32k/64k caps enforced; `/v1/decisions` has no header escape | | Docker image build | `docker build --platform linux/amd64`, default and `WITH_ASR=1` variants | both build; the stack inside is package-identical to the live-tested one (PyPI vLLM 0.30.0, torch 2.13.0+cu130, transformers 5.17.0, granite-switch 0.1.1); entry points and imports verified in-container; the `WITH_ASR=1` image holds the pinned speech-model revision as its only snapshot, with `refs/main` pointing at it | | Serving from the image | the default image (no speech model inside) under enroot (no Docker daemon on the GPU nodes), through `serve.sh`, only the checkpoint mounted, Hub offline: the 8b on one H100 and the 30b on two H100s (`--tensor-parallel-size 2`, set from the GPU count). On each: `e2e_check`, `probe_check`, `accuracy_check` | pass on both: 11/11 and 6/6, accuracy as on the pip path; the log shows `hf_overrides: {'asr_enabled': False}` and never names the speech model. Repeated with the published `ghcr.io/generative-computing/granite-switch-systemone:0.1.3` image (image manifest digest `sha256:77a56fb19768727de6e714d160d32842ddd1715964d51b9a5da5acba604e7971`, every layer's digest verified on download): same results on the 8b and the 30b, and in the speech-input row below | -| Speech input on (`SYSTEMONE_SPEECH_INPUT=1`) | the 8b on the default image with the speech model supplied from a mounted Hub cache (`HF_HUB_CACHE`); and, as a negative control, the same switch with no speech model anywhere | the 8b starts and passes the same three checks; the control stops in `serve.sh` with the preflight message naming the model and the fixes, before vLLM starts. The `WITH_ASR=1` image (model baked in) was served on the 8b in an earlier run | +| Speech input on (`DECISION_SPEECH_INPUT=1`) | the 8b on the default image with the speech model supplied from a mounted Hub cache (`HF_HUB_CACHE`); and, as a negative control, the same switch with no speech model anywhere | the 8b starts and passes the same three checks; the control stops in `serve.sh` with the preflight message naming the model and the fixes, before vLLM starts. The `WITH_ASR=1` image (model baked in) was served on the 8b in an earlier run | | Container on arm64 | `docker build` of the default image on an aarch64 NVIDIA GB10 host (vLLM 0.30.0, granite-switch 0.1.1 resolved, imports checked in the image); the public `granite-switch-4.1-8b` served from it with `docker run --gpus all`: `e2e_check`, `probe_check`, `accuracy_check` | pass; BoolQ accuracy 0.890 and log-loss 0.283 on arm64, against 0.890 and 0.282 on amd64 with the same checkpoint | | Multi-arch image build | `image.yml` run by hand (build only, then with push): each of the default and `-asr` images builds natively for amd64 and arm64, and a second job joins them into one tag | both tags are manifest lists holding a `linux/amd64` and a `linux/arm64` image (plus attestations), read from the workflow's own `imagetools inspect` step | -| Speech input check | `scripts/speech_check.py` against a server with `SYSTEMONE_SPEECH_INPUT=1` and the speech model from a Hub cache: two spoken English sentences made at run time | both clips repeated back in full, on the pip path and on the pushed arm64 `-asr` image (pulled from the registry, served with `docker run` on the GB10 with the public 8b); without clips and without a synthesizer the script exits 2 | +| Speech input check | `scripts/speech_check.py` against a server with `DECISION_SPEECH_INPUT=1` and the speech model from a Hub cache: two spoken English sentences made at run time | both clips repeated back in full, on the pip path and on the pushed arm64 `-asr` image (pulled from the registry, served with `docker run` on the GB10 with the public 8b); without clips and without a synthesizer the script exits 2 | | Decision Index runs | samples of the Decision Index suite (edition 0.3) through `/v1/systemone` at `--max-model-len 131072`, on the 8b and the 30b, as an end-to-end validation of the endpoint | completes with no errors; scores are not reported here while the methodology is verified, and the model card's published figures are the reference (see the caveats below) | ## Not tested @@ -74,10 +78,10 @@ show that the serving path is identical to the one that produced the card's figu route is hit is unverified; the plugin warns at startup when `speculative_config` is set. - **Decision load slows chat.** Chat p50 for a 32-token reply went from 235 ms idle to 1,088 ms with 128 concurrent decision clients and no cap. With - `SYSTEMONE_MAX_CONCURRENT_REQUESTS=8` and 32 clients it was 362 ms, the excess decision + `DECISION_MAX_CONCURRENT_REQUESTS=8` and 32 clients it was 362 ms, the excess decision requests getting 503s. The cap is off by default, and there is no priority between the two kinds of traffic. - **No system prompt by default.** The model card's published score used one; this plugin omits it so decisions can share the prefix cache with chat. How it affects accuracy is - still being evaluated. `SYSTEMONE_EXPERIMENTAL_SYSTEM_PROMPT=1` turns it on (tested + still being evaluated. `DECISION_EXPERIMENTAL_SYSTEM_PROMPT=1` turns it on (tested live); the default stays off until that is settled. diff --git a/docs/TESTING.md b/docs/TESTING.md index 913bd47..c11bad0 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -36,8 +36,8 @@ it needs `gitleaks` on `PATH` for the last of those. - `tests/test_openai_spec.py`: `/v1/decisions` requests and responses validated with a JSON Schema validator against OpenAI's published schema. The schema is downloaded on first run from `openai/openai-openapi` (MIT) at a pinned commit, checked against a - pinned SHA-256 and cached in `~/.cache/granite-switch-systemone`; it is never committed. - Offline, the test skips and says how to supply a local copy (`SYSTEMONE_OPENAI_SPEC`). + pinned SHA-256 and cached in `~/.cache/granite-switch-decision`; it is never committed. + Offline, the test skips and says how to supply a local copy (`DECISION_OPENAI_SPEC`). Deliberate limits are listed as named exceptions. - `tests/test_systemone_spec.py`: `/v1/systemone` validated against jev's wire models from TypeSafe's MIT-licensed `typesafe-sdk` (a dev dependency), which TypeSafe generates @@ -52,7 +52,7 @@ it needs `gitleaks` on `PATH` for the last of those. ```bash pip install "transformers>=5.16" -SYSTEMONE_TOKENIZER=ibm-granite/granite-switch-4.1-8b pytest tests/test_prefix_sharing.py +DECISION_TOKENIZER=ibm-granite/granite-switch-4.1-8b pytest tests/test_prefix_sharing.py ``` `tests/test_prefix_sharing.py` renders the real prompt with the checkpoint's tokenizer @@ -68,15 +68,15 @@ Python standard library unless noted. | Command | What it checks | |---|---| -| `python3 scripts/e2e_check.py --base-url URL` | Both routes' answer shapes and parity, input flattening, boolean values, error shapes, usage, chat on the same server, prefix-cache hits. Add `--system-prompt` when the server runs with `SYSTEMONE_EXPERIMENTAL_SYSTEM_PROMPT=1`. | +| `python3 scripts/e2e_check.py --base-url URL` | Both routes' answer shapes and parity, input flattening, boolean values, error shapes, usage, chat on the same server, prefix-cache hits. Add `--system-prompt` when the server runs with `DECISION_EXPERIMENTAL_SYSTEM_PROMPT=1`. | | `python3 scripts/probe_check.py --base-url URL` | Question cap, size guard (no engine work for oversized requests), body limits, special-token and marker escaping, options without descriptions, disconnect cancellation. | -| `python3 scripts/probe_check.py --base-url URL --max-concurrent N` | Adds the concurrency limit: a burst against `SYSTEMONE_MAX_CONCURRENT_REQUESTS=N` gets 200s and 503s. | +| `python3 scripts/probe_check.py --base-url URL --max-concurrent N` | Adds the concurrency limit: a burst against `DECISION_MAX_CONCURRENT_REQUESTS=N` gets 200s and 503s. | | `python3 scripts/probe_check.py --base-url URL --load 30` | Adds a load test at 1, 8 and 32 clients with chat latency alongside. Reported, not asserted. | | `python3 scripts/longctx_check.py --base-url URL --tokenizer /path/to/granite-switch-4.1-8b` | Long-context caps, on a server started with `--max-model-len 131072` and the 8b: the danger-zone header path (a ~98k-token request answered with the header, rejected without it), the 131,072 backstop, the default 32k/64k caps, and that `/v1/decisions` has no header escape. Unlike the scripts above it needs the checkpoint's tokenizer (transformers). | | `python3 scripts/accuracy_check.py --base-url URL` | Accuracy, log-loss and Brier score on 200 BoolQ validation rows (public data, downloaded when it runs, needs internet). Fails below 75% accuracy, which catches a wrong invocation marker. To check an upgrade or a changed setting, run it on each deployment with `--save`, then `--compare` the second against the first: it reports items only one run got right and an exact McNemar p-value. | -| `python3 scripts/speech_check.py --base-url URL` | Speech input, on a server started with `SYSTEMONE_SPEECH_INPUT=1` (see the README). Sends spoken clips to chat completions as `input_audio` and checks that the reply repeats at least 80% of the words. Makes its own clips with `espeak-ng` or macOS `say`, so no audio is stored here; `--clip FILE.wav "the words"` uses your own. Exits 2 if it has no clips. A server missing the audio libraries passes `/health` and fails here with HTTP 500. | +| `python3 scripts/speech_check.py --base-url URL` | Speech input, on a server started with `DECISION_SPEECH_INPUT=1` (see the README). Sends spoken clips to chat completions as `input_audio` and checks that the reply repeats at least 80% of the words. Makes its own clips with `espeak-ng` or macOS `say`, so no audio is stored here; `--clip FILE.wav "the words"` uses your own. Exits 2 if it has no clips. A server missing the audio libraries passes `/health` and fails here with HTTP 500. | | `python3 scripts/load_test.py --base-url URL` | Throughput and latency of `/v1/decisions` at 1, 8, 32, 64 and 128 clients, with the server's prefix-cache hit rate. `--mode unique` (default) gives every request a fresh state; `--mode identical` repeats one body and flatters throughput. Run both and report both. See below. | -| `SYSTEMONE_BASE_URL=URL pytest tests/test_decisions_e2e.py` | `/v1/decisions` and `/v1/systemone` return the same answers for equivalent requests. | +| `DECISION_BASE_URL=URL pytest tests/test_decisions_e2e.py` | `/v1/decisions` and `/v1/systemone` return the same answers for equivalent requests. | ### A short deployment check @@ -111,7 +111,7 @@ How to read it: - Throughput levels off once the GPU is saturated; past that point more clients only add latency. Where it levels off depends on the GPU, the model and the length of the state, so measure on your hardware instead of reusing a figure from another deployment. -- With `SYSTEMONE_MAX_CONCURRENT_REQUESTS` set, requests above the cap get HTTP 503 and are not queued. +- With `DECISION_MAX_CONCURRENT_REQUESTS` set, requests above the cap get HTTP 503 and are not queued. The client resends at once by default, which makes a capped server look worse than a caller that backs off would see. Pass `--backoff SECONDS` to wait after a non-200. The cap protects chat latency on a shared server; it does not raise decision throughput. @@ -124,11 +124,11 @@ How to read it: - The server refuses to start if a probe decision prompt does not carry exactly one adapter control token inside the question turn. The usual cause is a wrong - `SYSTEMONE_INVOCATION_MARKER` or `SYSTEMONE_ADAPTER_NAME`; the error names both. The probe + `DECISION_INVOCATION_MARKER` or `DECISION_ADAPTER_NAME`; the error names both. The probe logs nothing when it passes. - `serve.sh` switches the checkpoint's speech input off by default, which decisions do not use, so - the 8b and 30b start without a speech model. With `SYSTEMONE_SPEECH_INPUT=1` it first runs - `python3 -m granite_switch_systemone.preflight`: if the Hub is offline and the speech model is not + the 8b and 30b start without a speech model. With `DECISION_SPEECH_INPUT=1` it first runs + `python3 -m granite_switch_decision.preflight`: if the Hub is offline and the speech model is not in the cache, it stops with a message naming the model and the ways to fix it, instead of failing later inside the engine. The image sets `HF_HUB_OFFLINE=1`. - Each question logs the probability mass the option labels hold. Below 0.9 it logs a warning diff --git a/pyproject.toml b/pyproject.toml index fa82b4a..fc9bd47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,8 +3,8 @@ requires = ["setuptools>=77"] build-backend = "setuptools.build_meta" [project] -name = "granite-switch-systemone" -version = "0.1.3" +name = "granite-switch-decision" +version = "0.2.0" description = "jev system-one (/v1/systemone) and OpenAI Decisions (/v1/decisions) APIs as vLLM endpoint plugins for granite-switch, mounted alongside /v1/chat/completions on the same server." readme = "README.md" # SPDX license expressions and license-files need setuptools 77. @@ -27,7 +27,7 @@ dependencies = [] dev = ["fastapi", "pydantic", "pytest", "ruff==0.17.1", "ty==0.0.90", "httpx", "jsonschema", "typesafe-sdk==0.7.5"] [project.entry-points."vllm.endpoint_plugins"] -granite_switch_systemone = "granite_switch_systemone.plugin:register" +granite_switch_decision = "granite_switch_decision.plugin:register" [tool.setuptools.packages.find] where = ["src"] @@ -44,7 +44,7 @@ target-version = "py311" select = ["E", "F", "I", "UP"] [tool.ruff.lint.isort] -known-first-party = ["granite_switch_systemone"] +known-first-party = ["granite_switch_decision"] [tool.ty.rules] # vLLM is the runtime dependency that matters most and the one we can never install diff --git a/scripts/accuracy_check.py b/scripts/accuracy_check.py index c437ed3..75cf084 100644 --- a/scripts/accuracy_check.py +++ b/scripts/accuracy_check.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 -"""Accuracy check for a live granite-switch-systemone server, on public labelled data. +"""Accuracy check for a live granite-switch-decision server, on public labelled data. e2e_check.py shows the server speaks the wire formats; this shows the answers are still good. It sends yes/no questions from the BoolQ validation split (google/boolq, CC BY-SA diff --git a/scripts/e2e_check.py b/scripts/e2e_check.py index 7c8b427..aec0fc7 100755 --- a/scripts/e2e_check.py +++ b/scripts/e2e_check.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 -"""End-to-end check for a live granite-switch-systemone server. +"""End-to-end check for a live granite-switch-decision server. Runs the full wire-format battery against any deployment of this plugin (local, container or remote), using only the Python standard library, so it @@ -511,7 +511,7 @@ def main() -> int: parser.add_argument( "--system-prompt", action="store_true", - help="the server runs with SYSTEMONE_EXPERIMENTAL_SYSTEM_PROMPT=1 (changes the cache expectations)", + help="the server runs with DECISION_EXPERIMENTAL_SYSTEM_PROMPT=1 (changes cache expectations)", ) args = parser.parse_args() diff --git a/scripts/load_test.py b/scripts/load_test.py index d441e66..8a26e03 100644 --- a/scripts/load_test.py +++ b/scripts/load_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 -"""Throughput and latency of /v1/decisions on a live granite-switch-systemone server. +"""Throughput and latency of /v1/decisions on a live granite-switch-decision server. Standard library only. Each client loops on one keep-alive connection and sends the same kind of request: three questions (a predicate, a 20-way choice and a 5-level score) over a state of about @@ -19,7 +19,7 @@ Run it from a different machine than the server, on a server with no other traffic, and give each level enough time to settle (the default is 45 seconds). One run per level gives no spread. -A caller that gets HTTP 503 (see SYSTEMONE_MAX_CONCURRENT_REQUESTS) here resends immediately, which +A caller that gets HTTP 503 (see DECISION_MAX_CONCURRENT_REQUESTS) here resends immediately, which is harsher than a client that backs off, so a capped server's throughput looks worse than a well-behaved caller would see. Use --backoff SECONDS to wait after a non-200. """ diff --git a/scripts/longctx_check.py b/scripts/longctx_check.py index 46e017f..685d161 100644 --- a/scripts/longctx_check.py +++ b/scripts/longctx_check.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 -"""Long-context checks for a live granite-switch-systemone server started with the +"""Long-context checks for a live granite-switch-decision server started with the 8b checkpoint and --max-model-len 131072 (the checkpoint's maximum). Needs the checkpoint's tokenizer (transformers, unlike the other scripts, which are stdlib-only) to size the probe states. The 262,144-token danger-zone cap clamps to --max-model-len, diff --git a/scripts/probe_check.py b/scripts/probe_check.py index 8ff7c52..d2ecb13 100644 --- a/scripts/probe_check.py +++ b/scripts/probe_check.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 -"""Robustness checks for a live granite-switch-systemone server: limits, escaping, +"""Robustness checks for a live granite-switch-decision server: limits, escaping, cancellation, error handling, and optional load. Standard library only. python3 scripts/probe_check.py --base-url http://localhost:8000 @@ -17,7 +17,7 @@ descriptions an option without a description scores the same in any position cancellation a client that disconnects mid-request leaves the engine idle concurrency with --max-concurrent N (the server's - SYSTEMONE_MAX_CONCURRENT_REQUESTS), a burst gets 200s and 503s + DECISION_MAX_CONCURRENT_REQUESTS), a burst gets 200s and 503s load with --load SECONDS, throughput and latency at 1, 8 and 32 clients, and chat latency alongside (reported, not asserted) @@ -279,7 +279,7 @@ def main() -> int: parser.add_argument("--base-url", required=True) parser.add_argument("--model", default="granite-switch") parser.add_argument("--timeout", type=float, default=300.0) - parser.add_argument("--max-concurrent", type=int, default=0, help="the server's SYSTEMONE_MAX_CONCURRENT_REQUESTS") + parser.add_argument("--max-concurrent", type=int, default=0, help="the server's DECISION_MAX_CONCURRENT_REQUESTS") parser.add_argument("--load", type=float, default=0, help="seconds per load level (0 skips the load test)") args = parser.parse_args() p = Probe(args.base_url, args.model, args.timeout) diff --git a/scripts/speech_check.py b/scripts/speech_check.py index 9ed26c0..ae8fbcf 100644 --- a/scripts/speech_check.py +++ b/scripts/speech_check.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 -"""Speech-input smoke test for a live granite-switch-systemone server started with SYSTEMONE_SPEECH_INPUT=1. +"""Speech-input smoke test for a live granite-switch-decision server started with +DECISION_SPEECH_INPUT=1. Standard library only. Sends short spoken clips to /v1/chat/completions as `input_audio` and checks that the model heard them: the reply has to repeat most of the words that were spoken. Decisions take diff --git a/serve.sh b/serve.sh index d7f649d..77ed9af 100755 --- a/serve.sh +++ b/serve.sh @@ -5,21 +5,21 @@ # MODEL_DIR default /repository (HF Inference Endpoints' mount point) # TENSOR_PARALLEL_SIZE default: every visible GPU # MAX_MODEL_LEN (32768), GPU_MEMORY_UTILIZATION (0.90), PORT (8000), SERVED_MODEL_NAME -# SYSTEMONE_ADAPTER_NAME default "decision"; SYSTEMONE_INVOCATION_MARKER, see README -# SYSTEMONE_SPEECH_INPUT default 0: speech input is switched off, so no speech model is needed. +# DECISION_ADAPTER_NAME default "decision"; DECISION_INVOCATION_MARKER, see README +# DECISION_SPEECH_INPUT default 0: speech input is switched off, so no speech model is needed. # 1 keeps the checkpoint's speech input; see README, "Speech input". set -e MODEL_DIR=${MODEL_DIR:-/repository} # The 8b and 30b checkpoints enable speech input (asr_enabled) and granite-switch then loads a # speech model at startup, which decisions never use. Off by default; with it on, fail early, # with the cause, if the speech model is not in the offline cache. -case "${SYSTEMONE_SPEECH_INPUT:-0}" in +case "${DECISION_SPEECH_INPUT:-0}" in 0) SPEECH_ARGS=(--hf-overrides '{"asr_enabled": false}') ;; - 1) SPEECH_ARGS=(); python3 -m granite_switch_systemone.preflight "$MODEL_DIR" ;; - *) echo "SYSTEMONE_SPEECH_INPUT must be 0 or 1, not '$SYSTEMONE_SPEECH_INPUT'" >&2; exit 2 ;; + 1) SPEECH_ARGS=(); python3 -m granite_switch_decision.preflight "$MODEL_DIR" ;; + *) echo "DECISION_SPEECH_INPUT must be 0 or 1, not '$DECISION_SPEECH_INPUT'" >&2; exit 2 ;; esac TP=${TENSOR_PARALLEL_SIZE:-$(python3 -c "import torch; print(max(torch.cuda.device_count(), 1))")} -export SYSTEMONE_ADAPTER_NAME=${SYSTEMONE_ADAPTER_NAME:-decision} +export DECISION_ADAPTER_NAME=${DECISION_ADAPTER_NAME:-decision} # --dtype bfloat16: composed checkpoints carry torch_dtype null (granite-switch#126). # flash_attn_version=2: avoids the FA3 schedule bug (granite-switch#139, still open). # --max-logprobs 256: lets chat clients that read labels by top-K ask for more than 20. diff --git a/src/granite_switch_decision/__init__.py b/src/granite_switch_decision/__init__.py new file mode 100644 index 0000000..2979ceb --- /dev/null +++ b/src/granite_switch_decision/__init__.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 +"""granite-switch-decision: OpenAI Decisions (`POST /v1/decisions`) and jev system-one +(`POST /v1/systemone`) endpoints for granite-switch's vLLM backend, mounted on the same +server that already serves `/v1/chat/completions`: one engine, one KV cache, one container. + +See `plugin.py`'s module docstring for the full design. `renderer.py` is the one +adapter-specific piece this package carries today (the decision adapter's trained +prompt convention); `plugin.py` itself has no knowledge of any specific adapter. +""" + +__version__ = "0.2.0" diff --git a/src/granite_switch_systemone/plugin.py b/src/granite_switch_decision/plugin.py similarity index 94% rename from src/granite_switch_systemone/plugin.py rename to src/granite_switch_decision/plugin.py index 1a54d31..b85c325 100644 --- a/src/granite_switch_systemone/plugin.py +++ b/src/granite_switch_decision/plugin.py @@ -5,7 +5,7 @@ answer through `openai_serving_chat.create_chat_completion`, the same engine call `/v1/chat/completions` uses. Each question is one exact read of its option labels' logprobs (`logprob_token_ids`, chunked above vLLM's 128-id cap). The adapter-specific -prompt is built by the renderer named in `SYSTEMONE_RENDERER`. +prompt is built by the renderer named in `DECISION_RENDERER`. """ from __future__ import annotations @@ -34,9 +34,9 @@ _log = logging.getLogger(__name__) -_RENDERER_ENV = "SYSTEMONE_RENDERER" -_DEFAULT_RENDERER = "granite_switch_systemone.renderer:render" -_ADAPTER_NAME_ENV = "SYSTEMONE_ADAPTER_NAME" +_RENDERER_ENV = "DECISION_RENDERER" +_DEFAULT_RENDERER = "granite_switch_decision.renderer:render" +_ADAPTER_NAME_ENV = "DECISION_ADAPTER_NAME" _DEFAULT_ADAPTER_NAME = "decision" # jev: request body <= 32 MiB. @@ -52,13 +52,13 @@ MAX_LABEL_OPTIONS = 128 # Questions per request on both routes (OpenAI's limit on /v1/decisions). MAX_QUESTIONS = 200 -_MAX_CONCURRENT_ENV = "SYSTEMONE_MAX_CONCURRENT_REQUESTS" +_MAX_CONCURRENT_ENV = "DECISION_MAX_CONCURRENT_REQUESTS" # Engine reads in flight per request, so one wide request cannot flood the engine. MAX_READS_PER_REQUEST = 16 def _load_renderer(): - """Resolve `SYSTEMONE_RENDERER` to a callable at plugin init. It is loaded by name, so + """Resolve `DECISION_RENDERER` to a callable at plugin init. It is loaded by name, so this module never imports the configured renderer directly.""" path = os.environ.get(_RENDERER_ENV, _DEFAULT_RENDERER) if ":" not in path: @@ -73,7 +73,7 @@ def _load_renderer(): def _resolve_adapter_name(chat) -> str: """The decision adapter's name, checked against the checkpoint's own adapter list: - `SYSTEMONE_ADAPTER_NAME` if set, else the only adapter, else one named `decision`. + `DECISION_ADAPTER_NAME` if set, else the only adapter, else one named `decision`. A wrong name must fail startup: the chat template silently activates no adapter for a name it does not know.""" adapter_names = getattr(chat.model_config.hf_config, "adapter_names", None) or [] @@ -609,7 +609,8 @@ def ids(add_generation_prompt: bool) -> list[int]: if full.count(control_id) != 1 or not in_conversation: raise RuntimeError( f"a probe decision prompt does not carry exactly one {control} inside the question turn: " - "check SYSTEMONE_INVOCATION_MARKER and SYSTEMONE_ADAPTER_NAME against the checkpoint's chat template" + "check DECISION_INVOCATION_MARKER and DECISION_ADAPTER_NAME " + "against the checkpoint's chat template" ) @@ -793,11 +794,11 @@ async def _answer_all(state, state_value: Any, questions: list[tuple[str, Questi key, q, state_value, - state.systemone_chat, - state.systemone_tokenizer, - state.systemone_served_model, - state.systemone_adapter_name, - state.systemone_renderer, + state.decision_chat, + state.decision_tokenizer, + state.decision_served_model, + state.decision_adapter_name, + state.decision_renderer, reads, ) ) @@ -896,7 +897,7 @@ def _internal_answer_to_decisions( async def _handle_systemone(raw_request: Request) -> Response: state = raw_request.app.state - chat = getattr(state, "systemone_chat", None) + chat = getattr(state, "decision_chat", None) if chat is None: return _error(HTTPStatus.NOT_IMPLEMENTED, "the model does not support generation") @@ -922,8 +923,8 @@ async def _handle_systemone(raw_request: Request) -> Response: danger_zone = raw_request.headers.get(DANGER_ZONE_HEADER, "").strip().lower() == "true" try: _enforce_context_limits( - state.systemone_tokenizer, - state.systemone_max_model_len, + state.decision_tokenizer, + state.decision_max_model_len, danger_zone, req.state, req.questions, @@ -931,7 +932,7 @@ async def _handle_systemone(raw_request: Request) -> Response: except ValueError as e: return _error(HTTPStatus.BAD_REQUEST, str(e)) - async with state.systemone_limiter.slot() as admitted: + async with state.decision_limiter.slot() as admitted: if not admitted: return _error(HTTPStatus.SERVICE_UNAVAILABLE, _BUSY) try: @@ -948,7 +949,7 @@ async def _handle_systemone(raw_request: Request) -> Response: return JSONResponse( content={ - "model": state.systemone_served_model, + "model": state.decision_served_model, "answers": answers, "usage": {"input_tokens": usage.input_tokens, "output_tokens": usage.output_tokens}, } @@ -958,7 +959,7 @@ async def _handle_systemone(raw_request: Request) -> Response: async def _handle_decisions(raw_request: Request) -> Response: """POST /v1/decisions: OpenAI's wire format over the same engine path as /v1/systemone.""" state = raw_request.app.state - chat = getattr(state, "systemone_chat", None) + chat = getattr(state, "decision_chat", None) if chat is None: return _openai_error(HTTPStatus.NOT_IMPLEMENTED, "the model does not support generation") @@ -993,8 +994,8 @@ async def _handle_decisions(raw_request: Request) -> Response: try: _enforce_context_limits( - state.systemone_tokenizer, - state.systemone_max_model_len, + state.decision_tokenizer, + state.decision_max_model_len, False, # no danger-zone concept on the OpenAI route state_text, {key: internal for key, internal, _ in translated}, @@ -1003,7 +1004,7 @@ async def _handle_decisions(raw_request: Request) -> Response: return _openai_error(HTTPStatus.BAD_REQUEST, str(e)) work = _answer_all(state, state_text, [(key, internal) for key, internal, _ in translated]) - async with state.systemone_limiter.slot() as admitted: + async with state.decision_limiter.slot() as admitted: if not admitted: work.close() return _openai_error(HTTPStatus.SERVICE_UNAVAILABLE, _BUSY) @@ -1023,8 +1024,8 @@ async def _handle_decisions(raw_request: Request) -> Response: return JSONResponse( content={ "model": req.model - if req.model in getattr(state, "systemone_served_names", ()) - else state.systemone_served_model, + if req.model in getattr(state, "decision_served_names", ()) + else state.decision_served_model, "answers": answers, "usage": { "input_tokens": usage.input_tokens, @@ -1062,11 +1063,11 @@ def _route_exists(app: FastAPI, path: str) -> bool: return any(route.matches(scope)[0] == Match.FULL for route in app.routes) -class SystemOnePlugin: +class DecisionPlugin: """`vllm.endpoint_plugins` entry point (see module docstring). Loaded only when - `granite_switch_systemone` is in `VLLM_PLUGINS`; see this repo's README and Dockerfile.""" + `granite_switch_decision` is in `VLLM_PLUGINS`; see this repo's README and Dockerfile.""" - name = "granite_switch_systemone" + name = "granite_switch_decision" required_tasks: tuple[str, ...] | None = ("generate",) def attach_router(self, app: FastAPI) -> None: @@ -1075,7 +1076,7 @@ def attach_router(self, app: FastAPI) -> None: taken = sorted(path for path in _ROUTE_PATHS if _route_exists(app, path)) if taken: raise RuntimeError( - f"granite_switch_systemone would shadow routes already registered by vLLM: {', '.join(taken)}. " + f"granite_switch_decision would shadow routes already registered by vLLM: {', '.join(taken)}. " "Remove it from VLLM_PLUGINS, or use vLLM's own endpoints." ) app.include_router(router) @@ -1083,20 +1084,20 @@ def attach_router(self, app: FastAPI) -> None: async def init_state(self, engine_client: EngineClient | None, state: State, args: Namespace) -> None: chat = getattr(state, "openai_serving_chat", None) if chat is None: - _log.warning("granite_switch_systemone: no chat serving object; decision routes return 501") + _log.warning("granite_switch_decision: no chat serving object; decision routes return 501") return - state.systemone_chat = chat - state.systemone_tokenizer = chat.renderer.tokenizer - state.systemone_max_model_len = chat.model_config.max_model_len - state.systemone_served_names = list(args.served_model_name or [args.model]) - state.systemone_served_model = state.systemone_served_names[0] + state.decision_chat = chat + state.decision_tokenizer = chat.renderer.tokenizer + state.decision_max_model_len = chat.model_config.max_model_len + state.decision_served_names = list(args.served_model_name or [args.model]) + state.decision_served_model = state.decision_served_names[0] _warn_if_speculative(args) # A bad adapter name or renderer must stop the server, not serve wrong answers. - state.systemone_adapter_name = _resolve_adapter_name(chat) - state.systemone_renderer = _load_renderer() - state.systemone_limiter = _Limiter(_max_concurrent()) - _check_renderer_settings(state.systemone_renderer) - _check_activation(state.systemone_tokenizer, state.systemone_renderer, state.systemone_adapter_name) + state.decision_adapter_name = _resolve_adapter_name(chat) + state.decision_renderer = _load_renderer() + state.decision_limiter = _Limiter(_max_concurrent()) + _check_renderer_settings(state.decision_renderer) + _check_activation(state.decision_tokenizer, state.decision_renderer, state.decision_adapter_name) def _check_renderer_settings(renderer) -> None: @@ -1107,5 +1108,5 @@ def _check_renderer_settings(renderer) -> None: check() -def register() -> SystemOnePlugin: - return SystemOnePlugin() +def register() -> DecisionPlugin: + return DecisionPlugin() diff --git a/src/granite_switch_systemone/preflight.py b/src/granite_switch_decision/preflight.py similarity index 86% rename from src/granite_switch_systemone/preflight.py rename to src/granite_switch_decision/preflight.py index 0ee844e..d7fc680 100644 --- a/src/granite_switch_systemone/preflight.py +++ b/src/granite_switch_decision/preflight.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 """Checks to run before `vllm serve` starts; `serve.sh` calls this module when -SYSTEMONE_SPEECH_INPUT=1 (with speech input off the speech model is never loaded). +DECISION_SPEECH_INPUT=1 (with speech input off the speech model is never loaded). Granite Switch's audio processor loads a speech model while the engine profiles memory. The 8b and 30b checkpoints enable it (`asr_enabled`) without naming a model, so granite-switch falls @@ -8,7 +8,7 @@ the cache stops the engine deep in startup with a Hub lookup error that does not say what to do. This check fails earlier and names the cause and the fixes. - python3 -m granite_switch_systemone.preflight /repository + python3 -m granite_switch_decision.preflight /repository """ from __future__ import annotations @@ -24,7 +24,7 @@ # passes /health, then answers every audio request with HTTP 500 "install vllm[audio]". AUDIO_MODULES = ("av", "scipy", "soundfile", "soxr") # Test hook: a comma-separated list of the modules to treat as installed (an empty value checks nothing). -_AUDIO_MODULES_ENV = "SYSTEMONE_PREFLIGHT_AUDIO_MODULES" +_AUDIO_MODULES_ENV = "DECISION_PREFLIGHT_AUDIO_MODULES" # granite-switch 0.1.1's DEFAULT_ASR_MODEL_ID (granite_switch/vllm/audio/asr.py), used when the # checkpoint's `asr_model_id` is null. tests/test_preflight.py compares it with the installed copy. @@ -78,17 +78,17 @@ def missing_audio_modules(model_dir: Path) -> list[str]: def main(argv: list[str]) -> int: if len(argv) != 1: - print("usage: python3 -m granite_switch_systemone.preflight MODEL_DIR", file=sys.stderr) + print("usage: python3 -m granite_switch_decision.preflight MODEL_DIR", file=sys.stderr) return 2 modules = missing_audio_modules(Path(argv[0])) if modules: print( - "granite-switch-systemone: speech input is on, but these audio libraries are not installed: " + "granite-switch-decision: speech input is on, but these audio libraries are not installed: " f"{', '.join(modules)}.\n" "Without them the server starts and passes /health, then answers every audio request with " 'HTTP 500 "Please install vllm[audio]". Install vllm[audio] (or granite-switch[audio]) into ' - "this environment, or use a release image, which includes them. Leave SYSTEMONE_SPEECH_INPUT " - "unset or 0 if you do not need audio.", + "this environment, or use a release image, which includes them. " + "Leave DECISION_SPEECH_INPUT unset or 0 if you do not need audio.", file=sys.stderr, ) return 1 @@ -96,11 +96,11 @@ def main(argv: list[str]) -> int: if model_id is None: return 0 print( - f"granite-switch-systemone: this checkpoint enables speech input (asr_enabled in config.json) and needs\n" + f"granite-switch-decision: this checkpoint enables speech input (asr_enabled in config.json) and needs\n" f'the speech model "{model_id}", but the Hub is offline and it is not in the cache at\n' f"{hub_cache_dir(os.environ)}. Do one of:\n" " - use an image built with --build-arg WITH_ASR=1, which bakes the model in\n" - " - leave speech input off (SYSTEMONE_SPEECH_INPUT unset or 0), which decisions do not need\n" + " - leave speech input off (DECISION_SPEECH_INPUT unset or 0), which decisions do not need\n" " - mount a cache that holds it and point HF_HUB_CACHE at that directory\n" " - set HF_HUB_OFFLINE=0 so the model can be downloaded at startup", file=sys.stderr, diff --git a/src/granite_switch_systemone/renderer.py b/src/granite_switch_decision/renderer.py similarity index 96% rename from src/granite_switch_systemone/renderer.py rename to src/granite_switch_decision/renderer.py index 0c968fd..352f7ae 100644 --- a/src/granite_switch_systemone/renderer.py +++ b/src/granite_switch_decision/renderer.py @@ -6,7 +6,7 @@ \nReply with exactly one option label. answered by the next token, read as a softmax over the labels' logits. This matches the -layout on the model card. `SYSTEMONE_INVOCATION_MARKER` has no default: if the marker is +layout on the model card. `DECISION_INVOCATION_MARKER` has no default: if the marker is wrong, the chat template activates the adapter in a different place without raising. """ @@ -18,8 +18,8 @@ PLAIN_LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" -_INVOCATION_MARKER_ENV = "SYSTEMONE_INVOCATION_MARKER" -_SYSTEM_PROMPT_ENV = "SYSTEMONE_EXPERIMENTAL_SYSTEM_PROMPT" +_INVOCATION_MARKER_ENV = "DECISION_INVOCATION_MARKER" +_SYSTEM_PROMPT_ENV = "DECISION_EXPERIMENTAL_SYSTEM_PROMPT" # The trained system prompt, used for the model card's published score. Off by default so # a decision's state turn can share the prefix cache with chat; experimental. SYSTEM_PROMPT = ( diff --git a/src/granite_switch_systemone/__init__.py b/src/granite_switch_systemone/__init__.py deleted file mode 100644 index 76b26e4..0000000 --- a/src/granite_switch_systemone/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""granite-switch-systemone: a jev system-one API (`POST /v1/systemone`) bridge for -granite-switch's vLLM backend, mounted on the SAME server that already serves -`/v1/chat/completions`: one engine, one KV cache, one container. - -See `plugin.py`'s module docstring for the full design. `renderer.py` is the one -adapter-specific piece this package carries today (the decision adapter's trained -prompt convention); `plugin.py` itself has no knowledge of any specific adapter. -""" - -__version__ = "0.1.3" diff --git a/tests/specs.py b/tests/specs.py index 8fe3c64..3d6cce0 100644 --- a/tests/specs.py +++ b/tests/specs.py @@ -3,7 +3,7 @@ - OpenAI Decisions: `openapi.json` from openai/openai-openapi (MIT, Copyright (c) OpenAI), fetched once at a pinned commit, checked against a pinned SHA-256 and cached outside - the repository. `SYSTEMONE_OPENAI_SPEC=` uses a local copy instead. + the repository. `DECISION_OPENAI_SPEC=` uses a local copy instead. - jev system-one: built from the pydantic models in TypeSafe's MIT-licensed `typesafe-sdk` package (a dev dependency). TypeSafe's own OpenAPI document carries no licence, so it is not fetched or bundled. @@ -22,19 +22,19 @@ OPENAI_COMMIT = "33248a638234a53846ed3ab4b2a7c77069bc1030" OPENAI_URL = f"https://raw.githubusercontent.com/openai/openai-openapi/{OPENAI_COMMIT}/openapi.json" OPENAI_SHA256 = "c09eea21c178f883549c690752aa082a9fef494a719b7a08ca85341bd033b221" -_CACHE = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "granite-switch-systemone" +_CACHE = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "granite-switch-decision" OPENAI_REASON = ( "OpenAI's OpenAPI spec is unavailable. It is downloaded on first run from " f"{OPENAI_URL} (MIT) into {_CACHE}; allow network access once, or set " - "SYSTEMONE_OPENAI_SPEC to a local copy of that file." + "DECISION_OPENAI_SPEC to a local copy of that file." ) JEV_REASON = "jev schemas need TypeSafe's MIT-licensed SDK: pip install typesafe-sdk==0.7.5 (in the dev extra)." def openai_spec() -> dict | None: """OpenAI's full OpenAPI document at the pinned commit, or None if unavailable.""" - local = os.environ.get("SYSTEMONE_OPENAI_SPEC") + local = os.environ.get("DECISION_OPENAI_SPEC") path = Path(local) if local else _CACHE / f"openai-openapi-{OPENAI_COMMIT[:12]}.json" if not path.exists() and not local: try: diff --git a/tests/test_decisions.py b/tests/test_decisions.py index 93784a6..a3b2a8a 100644 --- a/tests/test_decisions.py +++ b/tests/test_decisions.py @@ -14,7 +14,7 @@ import pytest from pydantic import ValidationError -from granite_switch_systemone import plugin as sp +from granite_switch_decision import plugin as sp def _predicate(**kw): @@ -317,13 +317,13 @@ async def fake_answer(name, q, state, chat, tok, served_model, adapter_name, ren app = FastAPI() app.include_router(sp.router) - app.state.systemone_chat = object() - app.state.systemone_tokenizer = lambda text, add_special_tokens=False: {"input_ids": [0] * len(str(text).split())} - app.state.systemone_max_model_len = 8192 - app.state.systemone_served_model = "granite-switch" - app.state.systemone_adapter_name = "decision" - app.state.systemone_renderer = lambda *args: ([], []) - app.state.systemone_limiter = sp._Limiter(0) + app.state.decision_chat = object() + app.state.decision_tokenizer = lambda text, add_special_tokens=False: {"input_ids": [0] * len(str(text).split())} + app.state.decision_max_model_len = 8192 + app.state.decision_served_model = "granite-switch" + app.state.decision_adapter_name = "decision" + app.state.decision_renderer = lambda *args: ([], []) + app.state.decision_limiter = sp._Limiter(0) return TestClient(app) @@ -458,8 +458,8 @@ async def failing(*args): def test_full_limiter_returns_503_on_both_routes(client): - client.app.state.systemone_limiter = sp._Limiter(1) - client.app.state.systemone_limiter.in_flight = 1 + client.app.state.decision_limiter = sp._Limiter(1) + client.app.state.decision_limiter.in_flight = 1 body = {"model": "m", "input": "x", "questions": [{"type": "predicate", "instructions": "q?"}]} resp = client.post("/v1/decisions", json=body) assert resp.status_code == 503 and resp.json()["error"]["type"] == "server_error" @@ -514,7 +514,7 @@ async def upstream(): # stands in for vLLM's own endpoint return {} with pytest.raises(RuntimeError, match="/v1/decisions"): - sp.SystemOnePlugin().attach_router(app) + sp.DecisionPlugin().attach_router(app) def test_attach_router_sees_routes_added_through_include_router(): @@ -530,19 +530,19 @@ async def theirs(): app = FastAPI() app.include_router(upstream) with pytest.raises(RuntimeError, match="/v1/systemone"): - sp.SystemOnePlugin().attach_router(app) + sp.DecisionPlugin().attach_router(app) def test_attach_router_on_a_clean_app_mounts_both_routes(): from fastapi import FastAPI app = FastAPI() - sp.SystemOnePlugin().attach_router(app) + sp.DecisionPlugin().attach_router(app) assert sp._route_exists(app, "/v1/decisions") and sp._route_exists(app, "/v1/systemone") def test_response_echoes_the_requested_model_alias(client): - client.app.state.systemone_served_names = ["granite-switch", "alias-two"] + client.app.state.decision_served_names = ["granite-switch", "alias-two"] resp = client.post( "/v1/decisions", json={ diff --git a/tests/test_decisions_e2e.py b/tests/test_decisions_e2e.py index ed9e985..1c43fb3 100644 --- a/tests/test_decisions_e2e.py +++ b/tests/test_decisions_e2e.py @@ -5,7 +5,7 @@ tests/test_decisions.py's parity tests); this checks the wiring end to end, including the routes' registration on the real app. - SYSTEMONE_BASE_URL=http://host:8000 python3 -m pytest tests/test_decisions_e2e.py -v + DECISION_BASE_URL=http://host:8000 python3 -m pytest tests/test_decisions_e2e.py -v """ import json @@ -15,9 +15,9 @@ import pytest -BASE = os.environ.get("SYSTEMONE_BASE_URL") +BASE = os.environ.get("DECISION_BASE_URL") -pytestmark = pytest.mark.skipif(not BASE, reason="SYSTEMONE_BASE_URL not set") +pytestmark = pytest.mark.skipif(not BASE, reason="DECISION_BASE_URL not set") STATE = "My payouts have failed three times. The bank says everything is fine." # Engine reads are not bit-identical between batched and solo requests (measured up to 0.04). diff --git a/tests/test_openai_spec.py b/tests/test_openai_spec.py index 354f417..4a1f4de 100644 --- a/tests/test_openai_spec.py +++ b/tests/test_openai_spec.py @@ -14,7 +14,7 @@ import pytest -from granite_switch_systemone import plugin as sp +from granite_switch_decision import plugin as sp pytest.importorskip("jsonschema") from specs import OPENAI_REASON, openai_validators # noqa: E402 @@ -159,13 +159,13 @@ async def fake_answer(name, q, state, chat, tok, served_model, adapter_name, ren app = FastAPI() app.include_router(sp.router) - app.state.systemone_chat = object() - app.state.systemone_tokenizer = lambda text, add_special_tokens=False: {"input_ids": [0] * len(str(text).split())} - app.state.systemone_max_model_len = 8192 - app.state.systemone_served_model = "granite-switch" - app.state.systemone_adapter_name = "decision" - app.state.systemone_renderer = lambda *args: ([], []) - app.state.systemone_limiter = sp._Limiter(0) + app.state.decision_chat = object() + app.state.decision_tokenizer = lambda text, add_special_tokens=False: {"input_ids": [0] * len(str(text).split())} + app.state.decision_max_model_len = 8192 + app.state.decision_served_model = "granite-switch" + app.state.decision_adapter_name = "decision" + app.state.decision_renderer = lambda *args: ([], []) + app.state.decision_limiter = sp._Limiter(0) return TestClient(app) diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 0c1c80e..7377adf 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -13,7 +13,7 @@ import pytest from pydantic import ValidationError as PydanticValidationError -from granite_switch_systemone import plugin as sp +from granite_switch_decision import plugin as sp # -------------------------------------------------------------------------------------------- # Question / SystemOneRequest schema validation (jev's 422s) @@ -325,26 +325,26 @@ def test_resolve_adapter_name_auto_resolves_single_adapter(): def test_resolve_adapter_name_prefers_decision_among_several(monkeypatch): """The released Switch checkpoints carry guardian-core and decision.""" - monkeypatch.delenv("SYSTEMONE_ADAPTER_NAME", raising=False) + monkeypatch.delenv("DECISION_ADAPTER_NAME", raising=False) chat = _FakeChat(["guardian-core", "decision"]) assert sp._resolve_adapter_name(chat) == "decision" def test_resolve_adapter_name_requires_env_var_with_several_and_no_decision(monkeypatch): - monkeypatch.delenv("SYSTEMONE_ADAPTER_NAME", raising=False) + monkeypatch.delenv("DECISION_ADAPTER_NAME", raising=False) chat = _FakeChat(["guardian-core", "classifier"]) - with pytest.raises(RuntimeError, match="SYSTEMONE_ADAPTER_NAME"): + with pytest.raises(RuntimeError, match="DECISION_ADAPTER_NAME"): sp._resolve_adapter_name(chat) def test_resolve_adapter_name_requires_env_var_with_zero_adapters(): chat = _FakeChat([]) - with pytest.raises(RuntimeError, match="SYSTEMONE_ADAPTER_NAME"): + with pytest.raises(RuntimeError, match="DECISION_ADAPTER_NAME"): sp._resolve_adapter_name(chat) def test_resolve_adapter_name_honors_env_override(monkeypatch): - monkeypatch.setenv("SYSTEMONE_ADAPTER_NAME", "decision") + monkeypatch.setenv("DECISION_ADAPTER_NAME", "decision") chat = _FakeChat(["guardian-core", "decision"]) assert sp._resolve_adapter_name(chat) == "decision" @@ -352,7 +352,7 @@ def test_resolve_adapter_name_honors_env_override(monkeypatch): def test_resolve_adapter_name_env_override_still_validated(monkeypatch): """An override naming an adapter the checkpoint doesn't have must still fail loudly, not silently fall back to anything.""" - monkeypatch.setenv("SYSTEMONE_ADAPTER_NAME", "typo-adapter-name") + monkeypatch.setenv("DECISION_ADAPTER_NAME", "typo-adapter-name") chat = _FakeChat(["decision"]) with pytest.raises(RuntimeError, match="typo-adapter-name"): sp._resolve_adapter_name(chat) @@ -369,7 +369,7 @@ def test_load_renderer_default_resolves_to_this_packages_own_renderer(): _os.environ.pop(sp._RENDERER_ENV, None) fn = sp._load_renderer() - from granite_switch_systemone.renderer import render + from granite_switch_decision.renderer import render assert fn is render @@ -381,7 +381,7 @@ def test_load_renderer_rejects_a_path_without_a_colon(monkeypatch): def test_load_renderer_rejects_a_missing_attribute(monkeypatch): - monkeypatch.setenv(sp._RENDERER_ENV, "granite_switch_systemone.renderer:nonexistent_fn") + monkeypatch.setenv(sp._RENDERER_ENV, "granite_switch_decision.renderer:nonexistent_fn") with pytest.raises(RuntimeError, match="nonexistent_fn"): sp._load_renderer() @@ -601,7 +601,7 @@ def test_systemone_rejects_more_questions_than_the_cap(): def test_concurrency_limit_rejects_with_503_when_full(): - """SYSTEMONE_MAX_CONCURRENT_REQUESTS bounds decision requests in flight; past it, 503.""" + """DECISION_MAX_CONCURRENT_REQUESTS bounds decision requests in flight; past it, 503.""" async def run(): limiter = sp._Limiter(1) @@ -638,29 +638,29 @@ async def run(): def test_init_state_fails_startup_on_a_bad_system_prompt_flag(monkeypatch): from types import SimpleNamespace as NS - monkeypatch.setenv("SYSTEMONE_INVOCATION_MARKER", "### DECISION") - monkeypatch.setenv("SYSTEMONE_EXPERIMENTAL_SYSTEM_PROMPT", "yes") + monkeypatch.setenv("DECISION_INVOCATION_MARKER", "### DECISION") + monkeypatch.setenv("DECISION_EXPERIMENTAL_SYSTEM_PROMPT", "yes") chat = _FakeChat(["decision"]) chat.renderer = NS(tokenizer=None) chat.model_config.max_model_len = 8192 state = NS(openai_serving_chat=chat) args = NS(served_model_name=["m"], model="m") - with pytest.raises(RuntimeError, match="SYSTEMONE_EXPERIMENTAL_SYSTEM_PROMPT"): - asyncio.run(sp.SystemOnePlugin().init_state(None, state, args)) + with pytest.raises(RuntimeError, match="DECISION_EXPERIMENTAL_SYSTEM_PROMPT"): + asyncio.run(sp.DecisionPlugin().init_state(None, state, args)) def test_init_state_fails_startup_without_an_invocation_marker(monkeypatch): from types import SimpleNamespace as NS - monkeypatch.delenv("SYSTEMONE_INVOCATION_MARKER", raising=False) - monkeypatch.delenv("SYSTEMONE_EXPERIMENTAL_SYSTEM_PROMPT", raising=False) + monkeypatch.delenv("DECISION_INVOCATION_MARKER", raising=False) + monkeypatch.delenv("DECISION_EXPERIMENTAL_SYSTEM_PROMPT", raising=False) chat = _FakeChat(["decision"]) chat.renderer = NS(tokenizer=None) chat.model_config.max_model_len = 8192 state = NS(openai_serving_chat=chat) args = NS(served_model_name=["m"], model="m") - with pytest.raises(RuntimeError, match="SYSTEMONE_INVOCATION_MARKER"): - asyncio.run(sp.SystemOnePlugin().init_state(None, state, args)) + with pytest.raises(RuntimeError, match="DECISION_INVOCATION_MARKER"): + asyncio.run(sp.DecisionPlugin().init_state(None, state, args)) def test_parse_strict_json_rejects_deep_nesting_before_parsing(): @@ -743,11 +743,11 @@ async def fake_fetch(chat, served_model, adapter_name, msgs, label_ids_chunk): from types import SimpleNamespace as NS state = NS( - systemone_chat=None, - systemone_tokenizer=None, - systemone_served_model="m", - systemone_adapter_name="decision", - systemone_renderer=lambda tok, s, i, descs: ([], list(range(len(descs)))), + decision_chat=None, + decision_tokenizer=None, + decision_served_model="m", + decision_adapter_name="decision", + decision_renderer=lambda tok, s, i, descs: ([], list(range(len(descs)))), ) questions = [(str(i), sp.Question(type="noul", instructions="q?")) for i in range(60)] asyncio.run(sp._answer_all(state, "s", questions)) diff --git a/tests/test_prefix_sharing.py b/tests/test_prefix_sharing.py index e7fac3f..094b170 100644 --- a/tests/test_prefix_sharing.py +++ b/tests/test_prefix_sharing.py @@ -13,22 +13,22 @@ If one of the "shares no full block" tests starts failing, the decision layout changed: that is a cache win, but the layout is no longer the trained one, so re-check accuracy. - SYSTEMONE_TOKENIZER=ibm-granite/granite-switch-4.1-8b python3 -m pytest tests/test_prefix_sharing.py -v + DECISION_TOKENIZER=ibm-granite/granite-switch-4.1-8b python3 -m pytest tests/test_prefix_sharing.py -v Needs `transformers` new enough for the checkpoint's tokenizer class (5.16 works); -skipped when it is missing or SYSTEMONE_TOKENIZER is unset. +skipped when it is missing or DECISION_TOKENIZER is unset. """ import os import pytest -from granite_switch_systemone import plugin, renderer +from granite_switch_decision import plugin, renderer transformers = pytest.importorskip("transformers") -TOKENIZER = os.environ.get("SYSTEMONE_TOKENIZER") -pytestmark = pytest.mark.skipif(not TOKENIZER, reason="SYSTEMONE_TOKENIZER not set") +TOKENIZER = os.environ.get("DECISION_TOKENIZER") +pytestmark = pytest.mark.skipif(not TOKENIZER, reason="DECISION_TOKENIZER not set") BLOCK_SIZE = 16 # vLLM CacheConfig.DEFAULT_BLOCK_SIZE; serve.sh does not override it MARKER = "### DECISION" @@ -51,7 +51,7 @@ def tok(): @pytest.fixture(autouse=True) def _marker(monkeypatch): - monkeypatch.setenv("SYSTEMONE_INVOCATION_MARKER", MARKER) + monkeypatch.setenv("DECISION_INVOCATION_MARKER", MARKER) def _ids(tok, messages, add_generation_prompt=True, **template_kwargs): @@ -133,7 +133,7 @@ def test_chat_with_its_own_system_prompt_shares_no_full_block(tok): def test_activation_probe_accepts_the_real_template_and_refuses_a_wrong_marker(tok, monkeypatch): plugin._check_activation(tok, renderer.render, "decision") - monkeypatch.setenv("SYSTEMONE_INVOCATION_MARKER", "## DECISION") + monkeypatch.setenv("DECISION_INVOCATION_MARKER", "## DECISION") with pytest.raises(RuntimeError, match="exactly one"): plugin._check_activation(tok, renderer.render, "decision") diff --git a/tests/test_preflight.py b/tests/test_preflight.py index ae6992a..520a69c 100644 --- a/tests/test_preflight.py +++ b/tests/test_preflight.py @@ -11,7 +11,7 @@ import pytest -from granite_switch_systemone import preflight +from granite_switch_decision import preflight DEFAULT = preflight.DEFAULT_ASR_MODEL_ID @@ -99,7 +99,7 @@ def test_hub_cache_dir_follows_huggingface_hub_precedence(env, expected): def test_main_exits_1_and_says_what_to_do(tmp_path, monkeypatch, capsys): model_dir = _checkpoint(tmp_path, asr_enabled=True, asr_model_id=None) - monkeypatch.setenv("SYSTEMONE_PREFLIGHT_AUDIO_MODULES", "") # this test is about the speech model + monkeypatch.setenv("DECISION_PREFLIGHT_AUDIO_MODULES", "") # this test is about the speech model monkeypatch.setenv("HF_HUB_OFFLINE", "1") monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path / "cache")) assert preflight.main([str(model_dir)]) == 1 @@ -107,7 +107,7 @@ def test_main_exits_1_and_says_what_to_do(tmp_path, monkeypatch, capsys): assert DEFAULT in err assert str(tmp_path / "cache") in err assert "WITH_ASR=1" in err - assert "SYSTEMONE_SPEECH_INPUT" in err + assert "DECISION_SPEECH_INPUT" in err assert "HF_HUB_OFFLINE=0" in err diff --git a/tests/test_renderer.py b/tests/test_renderer.py index ece231a..53c11d1 100644 --- a/tests/test_renderer.py +++ b/tests/test_renderer.py @@ -6,7 +6,7 @@ import pytest -from granite_switch_systemone import renderer +from granite_switch_decision import renderer class _FakeTok: @@ -74,14 +74,14 @@ def test_text_of_json_dumps_dict_and_list(): def test_render_requires_invocation_marker_env_var(monkeypatch): - monkeypatch.delenv("SYSTEMONE_INVOCATION_MARKER", raising=False) + monkeypatch.delenv("DECISION_INVOCATION_MARKER", raising=False) tok = _FakeTok() - with pytest.raises(RuntimeError, match="SYSTEMONE_INVOCATION_MARKER"): + with pytest.raises(RuntimeError, match="DECISION_INVOCATION_MARKER"): renderer.render(tok, "a state", "a question?", ["descA", "descB"]) def test_render_produces_two_user_turns_with_marker_in_the_second(monkeypatch): - monkeypatch.setenv("SYSTEMONE_INVOCATION_MARKER", "### DECISION") + monkeypatch.setenv("DECISION_INVOCATION_MARKER", "### DECISION") tok = _FakeTok() msgs, label_ids = renderer.render(tok, "a state", "a question?", ["descA", "descB"]) assert [m["role"] for m in msgs] == ["user", "user"] @@ -111,7 +111,7 @@ def test_escape_special_tokens_leaves_ordinary_text_alone(): def test_render_escapes_state_instructions_and_descriptions(monkeypatch): - monkeypatch.setenv("SYSTEMONE_INVOCATION_MARKER", "### DECISION") + monkeypatch.setenv("DECISION_INVOCATION_MARKER", "### DECISION") msgs, _ = renderer.render(_SpecialTok(), "s<|end_of_text|>", "q<|decision|>?", ["d", "e"]) joined = "".join(m["content"] for m in msgs) for token in _SpecialTok().get_added_vocab(): @@ -121,30 +121,30 @@ def test_render_escapes_state_instructions_and_descriptions(monkeypatch): def test_render_escapes_the_invocation_marker_in_question_text(monkeypatch): """The chat template activates the adapter at the LAST marker in the turn, so a marker inside instructions or a description would move the activation point.""" - monkeypatch.setenv("SYSTEMONE_INVOCATION_MARKER", "### DECISION") + monkeypatch.setenv("DECISION_INVOCATION_MARKER", "### DECISION") msgs, _ = renderer.render(_FakeTok(), "s", "### DECISION is it urgent?", ["### DECISION a", "b"]) assert msgs[1]["content"].count("### DECISION") == 1 assert msgs[1]["content"].startswith("### DECISION\n") def test_render_sends_no_system_prompt_by_default(monkeypatch): - monkeypatch.setenv("SYSTEMONE_INVOCATION_MARKER", "### DECISION") - monkeypatch.delenv("SYSTEMONE_EXPERIMENTAL_SYSTEM_PROMPT", raising=False) + monkeypatch.setenv("DECISION_INVOCATION_MARKER", "### DECISION") + monkeypatch.delenv("DECISION_EXPERIMENTAL_SYSTEM_PROMPT", raising=False) msgs, _ = renderer.render(_FakeTok(), "s", "q?", ["a", "b"]) assert [m["role"] for m in msgs] == ["user", "user"] def test_render_adds_the_trained_system_prompt_when_enabled(monkeypatch): """Experimental: the layout the model card's published score used.""" - monkeypatch.setenv("SYSTEMONE_INVOCATION_MARKER", "### DECISION") - monkeypatch.setenv("SYSTEMONE_EXPERIMENTAL_SYSTEM_PROMPT", "1") + monkeypatch.setenv("DECISION_INVOCATION_MARKER", "### DECISION") + monkeypatch.setenv("DECISION_EXPERIMENTAL_SYSTEM_PROMPT", "1") msgs, _ = renderer.render(_FakeTok(), "s", "q?", ["a", "b"]) assert [m["role"] for m in msgs] == ["system", "user", "user"] assert msgs[0]["content"] == renderer.SYSTEM_PROMPT def test_render_rejects_an_unclear_system_prompt_flag(monkeypatch): - monkeypatch.setenv("SYSTEMONE_INVOCATION_MARKER", "### DECISION") - monkeypatch.setenv("SYSTEMONE_EXPERIMENTAL_SYSTEM_PROMPT", "yes") - with pytest.raises(RuntimeError, match="SYSTEMONE_EXPERIMENTAL_SYSTEM_PROMPT"): + monkeypatch.setenv("DECISION_INVOCATION_MARKER", "### DECISION") + monkeypatch.setenv("DECISION_EXPERIMENTAL_SYSTEM_PROMPT", "yes") + with pytest.raises(RuntimeError, match="DECISION_EXPERIMENTAL_SYSTEM_PROMPT"): renderer.render(_FakeTok(), "s", "q?", ["a", "b"]) diff --git a/tests/test_serve_sh.py b/tests/test_serve_sh.py index 17ebeca..ed9d667 100644 --- a/tests/test_serve_sh.py +++ b/tests/test_serve_sh.py @@ -33,7 +33,8 @@ def _run(tmp_path: Path, extra_env: dict[str, str], *args: str, asr_enabled: boo "STUB_OUT": str(out), "HF_HUB_OFFLINE": "1", "HF_HUB_CACHE": str(tmp_path / "empty-cache"), - "SYSTEMONE_PREFLIGHT_AUDIO_MODULES": "", # these tests are about the speech model, not the audio libraries + # these tests are about the speech model, not the audio libraries + "DECISION_PREFLIGHT_AUDIO_MODULES": "", **extra_env, } done = subprocess.run(["bash", str(SERVE_SH), *args], env=env, capture_output=True, text=True) @@ -47,7 +48,7 @@ def test_speech_input_is_off_by_default_and_needs_no_speech_model(tmp_path): def test_speech_input_on_without_the_model_stops_before_vllm(tmp_path): - done, argv = _run(tmp_path, {"SYSTEMONE_SPEECH_INPUT": "1"}) + done, argv = _run(tmp_path, {"DECISION_SPEECH_INPUT": "1"}) assert done.returncode == 1 assert argv == [] # vllm never started assert "WITH_ASR=1" in done.stderr @@ -57,14 +58,18 @@ def test_speech_input_on_with_the_model_cached_starts_without_the_override(tmp_p cache = tmp_path / "empty-cache" / "models--ibm-granite--granite-speech-5.0-470m-turboctc" / "snapshots" / "abc" cache.mkdir(parents=True) (cache / "config.json").write_text("{}") - done, argv = _run(tmp_path, {"SYSTEMONE_SPEECH_INPUT": "1"}) + done, argv = _run(tmp_path, {"DECISION_SPEECH_INPUT": "1"}) assert done.returncode == 0, done.stderr assert "--hf-overrides" not in argv def test_speech_input_on_without_the_audio_libraries_stops_before_vllm(tmp_path): done, argv = _run( - tmp_path, {"SYSTEMONE_SPEECH_INPUT": "1", "SYSTEMONE_PREFLIGHT_AUDIO_MODULES": "no_such_audio_module_xyz"} + tmp_path, + { + "DECISION_SPEECH_INPUT": "1", + "DECISION_PREFLIGHT_AUDIO_MODULES": "no_such_audio_module_xyz", + }, ) assert done.returncode == 1 assert argv == [] # vllm never started @@ -80,7 +85,7 @@ def test_extra_arguments_still_reach_vllm(tmp_path): @pytest.mark.parametrize("value", ["2", "yes", ""]) def test_bad_switch_value_is_refused(tmp_path, value): - done, argv = _run(tmp_path, {"SYSTEMONE_SPEECH_INPUT": value}) + done, argv = _run(tmp_path, {"DECISION_SPEECH_INPUT": value}) if value == "": assert done.returncode == 0 # empty falls back to the default, off else: diff --git a/tests/test_systemone_spec.py b/tests/test_systemone_spec.py index 99c4779..368a978 100644 --- a/tests/test_systemone_spec.py +++ b/tests/test_systemone_spec.py @@ -11,7 +11,7 @@ import pytest -from granite_switch_systemone import plugin as sp +from granite_switch_decision import plugin as sp pytest.importorskip("jsonschema") from specs import JEV_REASON, jev_validators # noqa: E402 @@ -107,13 +107,13 @@ async def fake_answer(name, q, state, chat, tok, served_model, adapter_name, ren app = FastAPI() app.include_router(sp.router) - app.state.systemone_chat = object() - app.state.systemone_tokenizer = lambda text, add_special_tokens=False: {"input_ids": [0] * len(str(text).split())} - app.state.systemone_max_model_len = 8192 - app.state.systemone_served_model = "granite-switch" - app.state.systemone_adapter_name = "decision" - app.state.systemone_renderer = lambda *args: ([], []) - app.state.systemone_limiter = sp._Limiter(0) + app.state.decision_chat = object() + app.state.decision_tokenizer = lambda text, add_special_tokens=False: {"input_ids": [0] * len(str(text).split())} + app.state.decision_max_model_len = 8192 + app.state.decision_served_model = "granite-switch" + app.state.decision_adapter_name = "decision" + app.state.decision_renderer = lambda *args: ([], []) + app.state.decision_limiter = sp._Limiter(0) return TestClient(app)