diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ccbe853..bd16dbf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,9 +28,8 @@ jobs: - name: pytest run: pytest -v --tb=short - # Build the artefact that ships and test THAT, not the checkout: install the - # wheel (src/ layout, so the checkout never shadows it) and run the same - # suite against the installed package. + # Build the artefact that ships and test that, not the checkout. The src/ layout + # means the checkout cannot shadow the installed wheel. build-and-install: runs-on: ubuntu-latest steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8309b48..4bd568f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ fail_fast: true -# SPDX/DCO hooks run at their own stages; everything else at pre-commit only. +# Hooks run at pre-commit unless they set stages (the DCO check runs at commit-msg). default_stages: [pre-commit] repos: - repo: https://github.com/pre-commit/pre-commit-hooks @@ -35,7 +35,6 @@ repos: language: system stages: [commit-msg] - # Needs gitleaks on PATH (brew install gitleaks / apt install gitleaks). - repo: https://github.com/gitleaks/gitleaks rev: v8.30.1 hooks: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c0d4fb9..54b9b00 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,9 +11,8 @@ pip install -e ".[dev]" ``` - The unit and spec tests need no GPU. Two of them fetch pinned third-party material - on first run (OpenAI's OpenAPI schema; TypeSafe's wire models come from the - `typesafe-sdk` dev dependency) and skip with a message when offline. See + The unit and spec tests need no GPU. `tests/test_openai_spec.py` downloads OpenAI's + OpenAPI schema on first run and skips with a message when offline. See [docs/TESTING.md](docs/TESTING.md) for what each test covers and how to run the live-server checks. @@ -24,6 +23,7 @@ ruff format --check src tests scripts ci examples ruff check src tests scripts ci examples ty check src + python3 ci/check_headers.py pytest ``` @@ -57,8 +57,8 @@ git commit --amend -s --no-edit To sign off several commits, use `git rebase --signoff ` and force-push. -The repo's pre-commit config also runs a local DCO check at `commit-msg` time, so -unsigned commits fail before they are pushed. +The repo's pre-commit config has a local DCO check that runs at `commit-msg` time. Install it with +`pre-commit install --hook-type commit-msg`, and unsigned commits then fail before they are pushed. ## Contribution Guidelines @@ -77,4 +77,4 @@ participating. ## Questions -Open an issue or start a discussion. +Open an issue. diff --git a/Dockerfile b/Dockerfile index ec3514a..1c2fd33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ ARG GRANITE_SWITCH_VERSION=0.1.1 ARG WITH_ASR=0 # granite-switch is pinned to a PyPI release so the image does not drift with upstream; -# transformers 5.17 is the version it supports. The [audio] extra (av, scipy, soundfile, +# transformers is pinned to 5.17.0. The [audio] extra (av, scipy, soundfile, # soxr) is in every build: without it audio requests fail with HTTP 500 while /health and # decisions look fine. The constraint file keeps the base image's vLLM, torch and # torchcodec as they are. @@ -30,8 +30,8 @@ RUN python3 -m pip list --format=freeze | grep -i -E "^(vllm|torch|torchaudio|to # The speech model the checkpoint's config names, baked in as a Hub cache at the tested # commit (without the repo's unused .py files). The server runs with the Hub offline, so a -# missing file fails startup instead of fetching something else. A download by commit -# writes no refs/main, which the offline lookup for "main" reads, so it is written here. +# missing file fails startup instead of fetching something else. Downloading by commit +# does not create refs/main, which the offline lookup needs, so the next lines write it. ARG ASR_MODEL=ibm-granite/granite-speech-5.0-470m-turboctc ARG ASR_REVISION=1dd23f46f6d4e6eae2531982ba2e33220086dd97 RUN if [ "$WITH_ASR" = "1" ]; then \ diff --git a/README.md b/README.md index 019a235..571a9dc 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,20 @@ # granite-switch-decision -Decision APIs for [Granite Switch](https://github.com/generative-computing/granite-switch), as a -vLLM plugin. Ask a question about some text and get back probabilities, from the same server and -GPU cache that already serves `/v1/chat/completions`. +Decision APIs for [Granite Switch](https://github.com/generative-computing/granite-switch) +([`granite-switch-4.1-8b`](https://huggingface.co/ibm-granite/granite-switch-4.1-8b) and +[`-30b`](https://huggingface.co/ibm-granite/granite-switch-4.1-30b) on Hugging Face), as a vLLM +plugin. Ask a question about some text and get back probabilities, from the same server and GPU cache +that already serves `/v1/chat/completions`. + +Serving a Switch model takes two vLLM plugins. The standard `granite-switch` plugin +(`register_granite_switch`) registers the model, and this one (`granite_switch_decision`) adds the +decision endpoints on top. The container image has both, and the pip install below sets up both. This +project makes the decision endpoints usable in the meantime (see [Upstream status](#upstream-status)). - `POST /v1/decisions` follows [OpenAI's Decisions API](https://developers.openai.com/api/docs/guides/decisions) for text input, and the official OpenAI SDKs work once you change `base_url`. A few parts of the API aren't supported, and those requests get a clear error ([details](#what-v1decisions-doesnt-support)). -- `POST /v1/systemone` follows [jev's system-one API](https://docs.typesafe.ai/api). +- `POST /v1/systemone` follows TypeSafe's [jev system-one API](https://docs.typesafe.ai/api). Each question is answered with one engine read of the adapter's single-token option labels, so callers do not need a prompt format, an invocation marker or a label scheme. @@ -15,8 +22,7 @@ callers do not need a prompt format, an invocation marker or a label scheme. With the public Granite Switch checkpoints, results are comparable with those on their model cards. You can run it as a container or add it to a vLLM install you already have. Start with -[Quick start](#quick-start). What has and has not been tested is in -[docs/STATUS.md](docs/STATUS.md), and how to run the tests is in [docs/TESTING.md](docs/TESTING.md). +[Quick start](#quick-start). To check a deployment or run the tests, see [Testing](#testing). ## Quick start @@ -24,10 +30,18 @@ You need a Linux machine with an NVIDIA GPU. The 8b checkpoint fits on one 80 GB ### Container -Release images are published to GitHub Container Registry as +Images are published to GitHub Container Registry as `ghcr.io/generative-computing/granite-switch-decision` for linux/amd64 and linux/arm64, so one tag -works on both. `latest` is the newest release, and each release is also tagged with its version. The -pull is about 9 GB and the image takes about 30 GB on disk (amd64 figures). You also need Docker and the NVIDIA container toolkit. +works on both. There are two variants of each release: + +| Tag | What it is | +|---|---| +| `latest`, `0.2.0` | Decisions and chat. Use this unless you need speech. | +| `latest-asr`, `0.2.0-asr` | The same, plus the speech model ([`ibm-granite/granite-speech-5.0-470m-turboctc`](https://huggingface.co/ibm-granite/granite-speech-5.0-470m-turboctc)) built in, so `/v1/chat/completions` accepts spoken input (see [Speech input](#speech-input-optional)). About 1 GB larger. | + +`latest` and `latest-asr` always point at the newest release of their variant; the version tags stay +fixed, so use one of those when you want a repeatable deployment. The pull is about 9 GB and the image takes +about 30 GB on disk (amd64 figures). You also need Docker and the NVIDIA container toolkit. ```bash # Download the checkpoint (public, not gated, about 18 GB) @@ -42,20 +56,17 @@ docker run -d --name gss --gpus all --ipc=host -p 8000:8000 \ Loading takes a few minutes; follow it with `docker logs -f gss`. `--ipc=host` is vLLM's usual setting for shared memory, and it matters most when you run with tensor parallelism. -The ghcr package is currently private, so `docker pull` needs `docker login ghcr.io` with an account -that has access. Until it is public, build the image yourself: clone this repository at a release tag -and run `docker build -t granite-switch-decision .`. - -The image is `vllm/vllm-openai:v0.30.0` plus `granite-switch[audio]==0.1.1`, `transformers==5.17.0` and -this plugin. It has no weights in it. It reads the checkpoint from `/repository` (override with -`MODEL_DIR`), so the same image serves the 8b and the 30b. We tested the 30b on two 80 GB GPUs; the +The image is built `FROM vllm/vllm-openai:v0.30.0` (Docker Hub), and adds `granite-switch[audio]==0.1.1` +(the standard Switch plugin), `transformers==5.17.0` and this plugin. Both plugins are enabled for you. It +has no Granite Switch weights in it: it reads the checkpoint from `/repository` (override with +`MODEL_DIR`), so the same image serves the 8b and the 30b. The one exception is the `-asr` image, which +also carries the speech model. We tested the 30b on two 80 GB GPUs; the container uses every GPU it can see unless you set `TENSOR_PARALLEL_SIZE`. ### Existing vLLM environment -You need Python 3.11 or later and PyTorch 2.11 or later (CUDA 13 builds are tested). **Use vLLM -0.30.0**: it is the version the live tests ran on. granite-switch 0.1.1 accepts `vllm>=0.26,<0.31`, -and `v0.1.0` was also served on 0.26 to 0.29, but nothing since has been checked there. +You need Python 3.11 to 3.13 and PyTorch 2.11 or later (CUDA 13 builds are tested). **Use vLLM +0.30.0**: it is the version the live tests ran on. granite-switch 0.1.1 accepts `vllm>=0.26,<0.31`. ```bash pip install "granite-switch[vllm]==0.1.1" "transformers>=5.16" @@ -63,7 +74,8 @@ pip install "git+https://github.com/generative-computing/granite-switch-decision export VLLM_PLUGINS=register_granite_switch,granite_switch_decision export DECISION_INVOCATION_MARKER="### DECISION" vllm serve ibm-granite/granite-switch-4.1-8b --served-model-name granite-switch \ - --dtype bfloat16 --attention-config.flash_attn_version=2 --enable-prompt-tokens-details + --dtype bfloat16 --attention-config.flash_attn_version=2 --enable-prompt-tokens-details \ + --hf-overrides '{"asr_enabled": false}' ``` List both plugins in `VLLM_PLUGINS`. vLLM loads endpoint plugins only when they are named, and once @@ -77,8 +89,8 @@ Create an endpoint for the model repository and choose a custom container. Huggi says the model weights are stored under `/repository`, which is where this image looks. The image serves on port `8000` and answers `/health`, so those are the values for the endpoint's container port and health route. The guide lists public images and private ones on Docker Hub, AWS ECR, Azure ACR -and Google GCR; GitHub Container Registry is not in that list, so the package has to be public. -**We have not tried this on Inference Endpoints yet.** +and Google GCR. GitHub Container Registry is not in that list, but the image is public, so it needs no +registry credentials. ### Check that it works @@ -119,11 +131,12 @@ are. - **`cache_salt`** keeps a request's prefix cache separate from everyone else's. The cache is shared, so a fast response (or a `cached_tokens` count) can show that someone has already sent the same prompt. If you serve several - tenants, have your gateway set a long random `cache_salt` for each. It's a privacy measure, not authentication. + tenants, have your gateway set a long random `cache_salt` for each. It separates caches; it does not authenticate callers. `/v1/decisions` accepts it too. - **`chat_template_kwargs`** may only be `{"enable_thinking": false}`. The server always chooses the adapter. -- **`joint`, `images`, `audio`, `videos`, `priority`, `request_id`** are accepted at their defaults. This model reads - text, one question at a time, so other values get a 422. +- **`joint`, `images`, `audio`, `videos`, `priority`** are accepted at their defaults. This model reads text, one + question at a time, so other values get a 422. +- **`request_id`** is accepted and ignored. - **`model`** is required here, though vLLM makes it optional. Each question in a `/v1/systemone` response also carries `diagnostics`: `label_mass`, how much probability the model @@ -133,11 +146,17 @@ comparable with vLLM's. ## Secure it before you expose it +Check the security of any image you run, including this one and the `vllm/vllm-openai` image it is built +on. For the most control, build your own image from this repository's `Dockerfile` (change the base image +or pinned versions as you need), or install the plugins straight into a vLLM environment you already +maintain, as in [Existing vLLM environment](#existing-vllm-environment). + The server listens on `0.0.0.0` and, as shipped, asks for no credentials on any route. Put it behind a gateway that authenticates callers, or turn on vLLM's `--api-key`, which `serve.sh` passes through: ```bash -docker run -d --gpus all --ipc=host -p 8000:8000 -v … granite-switch-decision --api-key "$KEY" +docker run -d --gpus all --ipc=host -p 8000:8000 -v /srv/models/granite-switch-4.1-8b:/repository:ro \ + ghcr.io/generative-computing/granite-switch-decision:0.2.0 --api-key "$KEY" ``` In vLLM 0.30.0 that key check covers every `/v1` route, including `/v1/decisions` and @@ -146,8 +165,7 @@ have not run it against a live server, so after enabling it, confirm that an una `/v1/decisions` returns 401. If the server also handles chat, set `DECISION_MAX_CONCURRENT_REQUESTS` (8 to 32 is a sensible -range). It is off by default, and without it a burst of decisions slowed chat p50 from 235 ms to -1,088 ms in our load test (see [docs/STATUS.md](docs/STATUS.md)). +range). It is off by default, and without it a burst of decisions can queue ahead of chat. ## Speech input (optional) @@ -161,14 +179,13 @@ speech model ([`ibm-granite/granite-speech-5.0-470m-turboctc`](https://huggingface.co/ibm-granite/granite-speech-5.0-470m-turboctc), about 1 GB) has to be available. Any one of these works: -- Use an image with the model built in: the `-asr` tag of a release (releases made with the current - workflow publish one), or build it yourself: +- Use an image with the model built in: the `-asr` tag (for example `latest-asr`), or build it yourself: `docker build --build-arg WITH_ASR=1 -t granite-switch-decision:asr .` - Mount a Hub cache that holds the model and set `HF_HUB_CACHE` to that directory. - Set `HF_HUB_OFFLINE=0` so the model downloads at startup. -On startup `serve.sh` checks for the speech model and for the audio libraries (`av`, `scipy`, -`soundfile`, `soxr`), and stops with a message that names whatever is missing. The images already +With `DECISION_SPEECH_INPUT=1`, `serve.sh` checks at startup for the speech model and for the audio libraries +(`av`, `scipy`, `soundfile`, `soxr`), and stops with a message that names whatever is missing. The images already include the libraries. If you run the plugin in your own environment, install `vllm[audio]`. Without it the server starts and passes `/health`, then answers every audio request with HTTP 500 "Please install vllm[audio]". @@ -190,8 +207,8 @@ name is passed on to `vllm serve`. | `MAX_MODEL_LEN` | `32768` | Longest prompt plus output. | | `GPU_MEMORY_UTILIZATION` | `0.90` | Share of GPU memory vLLM may use. | | `TENSOR_PARALLEL_SIZE` | every visible GPU | GPUs to split the model across. | -| `DECISION_ADAPTER_NAME` | the only adapter, else `decision` | Which adapter answers. Checked against the checkpoint at startup; an unknown name stops the server. | -| `DECISION_INVOCATION_MARKER` | none (`### DECISION` in the image) | The text the adapter was trained to activate on. It is required: with a wrong marker the chat template activates the adapter in the wrong place without raising an error. | +| `DECISION_ADAPTER_NAME` | `decision` (the plugin alone uses the checkpoint's only adapter, else `decision`) | Which adapter answers. Checked against the checkpoint at startup; an unknown name stops the server. | +| `DECISION_INVOCATION_MARKER` | none (`### DECISION` in the image) | The text the adapter was trained to activate on. It is required. Startup checks that a probe prompt activates the adapter exactly once and stops if not, but it cannot tell a working marker from the one the adapter was trained on. | | `DECISION_SPEECH_INPUT` | `0` | `1` keeps speech input; see [Speech input](#speech-input-optional). | | `DECISION_MAX_CONCURRENT_REQUESTS` | `0` (no cap) | Decision requests allowed in flight across both routes. Past the cap a request gets 503 at once, so a burst cannot queue ahead of chat. | | `DECISION_RENDERER` | `granite_switch_decision.renderer:render` | A `module:function` that builds the prompt and label ids, for an adapter trained differently. | @@ -213,7 +230,7 @@ model repository moves. The checkpoint is never a build argument: you mount it w | Flag | Why | |---|---| | `--dtype bfloat16` | Composed checkpoints carry `torch_dtype: null` ([granite-switch#126](https://github.com/generative-computing/granite-switch/issues/126)), and vLLM would otherwise load float32. | -| `--attention-config.flash_attn_version=2` | Avoids an FA3 schedule bug with CUDA graphs and prefix caching ([granite-switch#139](https://github.com/generative-computing/granite-switch/issues/139), open). With granite-switch 0.1.1 we measured FA3 the same as FA2 for answers and speed, so this is a precaution. | +| `--attention-config.flash_attn_version=2` | Avoids an FA3 schedule bug with CUDA graphs and prefix caching ([granite-switch#139](https://github.com/generative-computing/granite-switch/issues/139), open). | | `--enable-prompt-tokens-details` | vLLM reports each request's prefix-cache hits, and `/v1/decisions` returns them as `usage.input_tokens_details`. Without the flag both counts are 0. | | `--max-logprobs 256` | For chat clients that read labels by top-K logprobs (vLLM's default cap is 20). The plugin does not need it. | | `--enable-auto-tool-choice --tool-call-parser granite4` | Tool calling on the shared chat route. | @@ -261,7 +278,7 @@ same prompt, served from the prefix cache. - **Repeat requests.** The same request can return slightly different probabilities, depending on how it is batched and whether it hits the prefix cache, so a near-tie can flip. A different GPU or tensor-parallel size can differ a little more. vLLM's `VLLM_BATCH_INVARIANT=1` aims to make repeats exact; we haven't tried it here. -- **Not provided.** Authentication (use vLLM's `--api-key`), rate limiting and `refusal` answers. +- **Not provided.** Authentication (use vLLM's `--api-key`), per-client rate limiting and `refusal` answers. ### What /v1/decisions doesn't support @@ -277,45 +294,39 @@ The route never returns `refusal` answers, and `/v1/systemone` likewise refuses ## Upstream status -vLLM now has its own `/v1/systemone` ([#59299](https://github.com/vllm-project/vllm/pull/59299), -merged 2026-10-08) and `/v1/decisions` ([#60465](https://github.com/vllm-project/vllm/pull/60465), -merged 2026-10-13). Both are on `main` after 0.30.0, so the next vLLM release will carry them. This -package is for vLLM 0.30, and for 0.26 to 0.29 (untested since `v0.1.0`), where those routes are -absent. You can drop it once you move to a release that has them. The upstream endpoints differ in wire -details: this plugin follows jev's format and OpenAI's schema as published. - -The plugin refuses to start if either path is already registered, so on a vLLM that has the upstream -endpoints it fails at startup and does not silently pick one. Remove `granite_switch_decision` from -`VLLM_PLUGINS` there. vLLM 0.32 mounts its own `/v1/systemone` (newer versions add `/v1/decisions`). -Neither knows Switch adapters, and granite-switch 0.1.1 does not support those versions yet. +This project is temporary. vLLM has added its own `/v1/systemone` +([#59299](https://github.com/vllm-project/vllm/pull/59299), in 0.32.0) and `/v1/decisions` +([#60465](https://github.com/vllm-project/vllm/pull/60465), merged to `main`). We intend to work with the +vLLM project on bringing base Granite Switch support +([#60232](https://github.com/vllm-project/vllm/pull/60232)) and decision support into vLLM itself or a +common plugin, and this package goes away once that is done. -Still open upstream are native Granite Switch support -([#60232](https://github.com/vllm-project/vllm/pull/60232)) and modular read strategies -([RFC #59365](https://github.com/vllm-project/vllm/issues/59365)). If the RFC is accepted, the natural -migration is a Granite Switch read strategy, and `renderer.render(tok, state, instructions, descs) -> -(messages, label_ids)` is already close to what a strategy needs. +Use it with vLLM 0.30; later releases are untested. A vLLM that already has those routes makes the plugin stop at startup, rather than +quietly choosing one, so remove `granite_switch_decision` from `VLLM_PLUGINS` there. The upstream +endpoints differ in some wire details: this plugin follows jev's format and OpenAI's schema as published. -## How well tested is this? +## Testing -The pip path is what most of the live testing used. The container image is lightly tested: it builds -for amd64 and arm64, with and without the speech model, and the packages and plugin entry points were checked -inside it. The published `0.1.3` image (then named `granite-switch-systemone`) was served on H100 GPUs with only the checkpoint mounted and -speech input off, the 8b on one GPU and the 30b on two with tensor parallelism, and the wire, -robustness and accuracy checks passed. That was run under enroot and not with `docker run`. The arm64 image was also served with `docker run` on an NVIDIA GB10 (the 8b, default image). The arm64 `-asr` image was also served with `docker run` and passed the speech check; see -[docs/STATUS.md](docs/STATUS.md) for exactly what was and was not run. If you run the image, treat the -first start as a test and tell us what you find. - -## Development +The unit and spec tests run in CI, with no GPU. The scripts in `scripts/` check a running server from +outside; they take `--base-url` and use only the standard library unless noted. We have run them on vLLM +0.30.0 with H100 GPUs, serving the 8b and the 30b, mostly on the pip install. We have not run the published +images with `docker run` on amd64. ```bash -pip install -e ".[dev]" && pytest # unit and spec tests, no GPU -python3 scripts/e2e_check.py --base-url http://localhost:8000 # live wire checks -python3 scripts/probe_check.py --base-url http://localhost:8000 # live robustness checks -python3 scripts/accuracy_check.py --base-url http://localhost:8000 # answers are still good (needs internet) +pip install -e ".[dev]" && pytest # unit and spec tests, no GPU ``` -[docs/TESTING.md](docs/TESTING.md) lists every test, what runs in CI, and what needs a tokenizer or a -live server. +| Script | What it checks | +|---|---| +| `e2e_check.py` | Wire formats on both routes, parity, input flattening, errors, usage, vLLM's request fields, prefix-cache sharing | +| `probe_check.py` | Question and size limits, escaping, disconnects; `--max-concurrent` and `--load` add the limiter and a load run | +| `accuracy_check.py` | Accuracy on BoolQ rows (needs internet); `--save` and `--compare` for before/after runs | +| `speech_check.py` | Speech input on a server started with `DECISION_SPEECH_INPUT=1` | +| `load_test.py` | Throughput and latency of `/v1/decisions` at 1 to 128 clients | +| `longctx_check.py` | Long-context caps (needs the tokenizer and a 131k-context server) | + +[docs/TESTING.md](docs/TESTING.md) has every test, the commands and options, what runs in CI, and what +needs a tokenizer or a live server. ## Licence diff --git a/SECURITY.md b/SECURITY.md index bee9d39..e1dd17d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -19,7 +19,8 @@ server alongside `/v1/chat/completions`. When deploying it: `--api-key`, which covers these routes too, whenever the server is reachable by anyone you do not fully trust - **Model access**: restrict who can read the checkpoint and log in to the serving machine -- **Caller text is untrusted input**: the plugin escapes it against prompt injection, - but a decision adapter's probabilities are a prior, not an authorisation decision +- **Caller text is untrusted input**: the plugin escapes special tokens and the invocation + marker, but the text can still steer the model, and a decision adapter's probabilities are a + prior, not an authorization decision - **Dependencies**: keep vLLM and the surrounding stack updated to receive security patches diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 820eae2..71cbd4d 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -1,6 +1,7 @@ # Third-party notices -This project is Apache-2.0. It includes the following third-party material, used under the +This project is Apache-2.0, except `src/granite_switch_decision/plugin.py` and `tests/test_plugin.py`, +which are Apache-2.0 AND MIT because they adapt the MIT code below. It includes the following third-party material, used under the licence shown. ## typesafe-ai/system-one-adapter-python diff --git a/docs/STATUS.md b/docs/STATUS.md deleted file mode 100644 index f8bf24f..0000000 --- a/docs/STATUS.md +++ /dev/null @@ -1,90 +0,0 @@ -# Status: what is tested, and what is not - -How to run each test is in [TESTING.md](TESTING.md). - -Last updated 2026-10-25. "Live" means a real server on H100s: granite-switch 0.1.1, -transformers 5.17, vLLM 0.26.0 / 0.27.1 / 0.28.0 / 0.29.0 / 0.30.0 as noted per row, -and `ibm-granite/granite-switch-4.1-8b` unless a row names another checkpoint. - -Renamed in 0.2.0 (`granite-switch-systemone` to `granite-switch-decision`, `SYSTEMONE_*` to `DECISION_*`). -The rows below were run at 0.1.3 or earlier under the old names. After the rename we repeated only the unit -and spec suites and the 8b battery (`e2e_check`, `probe_check`, `accuracy_check`, pip install, one H100). - -## Tested - -| Area | How | Result | -|---|---|---| -| Unit and route tests | `pytest`, engine stubbed, in CI | pass | -| `/v1/decisions` against OpenAI's OpenAPI schema | `tests/test_openai_spec.py` (against the general-availability schema), plus the same request set live | pass, with the exceptions listed in the README | -| `/v1/systemone` against jev's wire format | `tests/test_systemone_spec.py` (24 cases) against `typesafe-sdk`'s generated models, stubbed | pass; the plugin also rejects unknown fields, which jev's format allows | -| Live wire battery | `scripts/e2e_check.py` (12 checks, including vLLM's request fields and `cache_salt`) | 12/12 | -| Prompt rendering | server prompt token counts against an offline render with the checkpoint's tokenizer and chat template | identical | -| Prefix-cache sharing | `tests/test_prefix_sharing.py`, and `cached_tokens` live, cross-checked with `/metrics` | as documented in the README | -| Route parity | 800 labelled items through `/v1/decisions` and through the same prompt over chat completions | no item answers differently | -| Escaping, size guard, disconnect cancellation, error shapes | live probes, now `scripts/probe_check.py` | as documented in the README | -| Question cap and `DECISION_MAX_CONCURRENT_REQUESTS` | live: 400 questions accepted, 401 rejected; a 32-request burst against a cap of 8 gave 8 accepted and 24 rejected with 503 | pass | -| `DECISION_EXPERIMENTAL_SYSTEM_PROMPT=1` | live: the 800-item comparison and `e2e_check.py --system-prompt` | answers identical to the same prompt over chat completions; decisions about one state still share the cache, chat no longer does | -| FlashAttention 3 | FA2 and FA3 servers side by side: the 800-item comparison, 40 cached one-token reads (the case in granite-switch#139), load at 8 and 32 clients | no difference beyond batching noise in answers or speed; FA2 kept in `serve.sh` while #139 is open | -| Load | 1 to 128 concurrent clients, three-question requests | about 140 requests/s plateau, no errors | -| Plugin-only install | the README's steps in a fresh environment, then its `vllm serve` command and the e2e battery | pass | -| Install from this repository | `pip install git+https://github.com/generative-computing/granite-switch-systemone` (the repository's name then) into a fresh venv, over HTTPS and SSH with authenticated access while the repository was private, with the entry-point check and the unit and spec suites against the installed copy from a neutral directory (184 tests, pass); then anonymously (no git config, credentials or SSH agent) once public, at v0.1.2 | the anonymous install resolved and registered the `vllm.endpoint_plugins` entry point; the suites were not rerun on it | -| vLLM 0.26.0, 0.27.1, 0.28.0, 0.29.0 live (at `v0.1.0` only; not repeated for later changes) | one server per version on one H100 (torch 2.11.0+cu130 on 0.26, torch 2.13.0+cu130 on the rest): the e2e battery, `probe_check`, `test_decisions_e2e`, `test_prefix_sharing`, and the unit suite under each version's own venv | 11/11 e2e and every suite passes on all four versions | -| The 30b checkpoint | `granite-switch-4.1-30b` on vLLM 0.30.0, `--tensor-parallel-size 2` on two H100s: the full battery | pass | -| Tensor parallelism | the 8b with `--tensor-parallel-size 2`: the full battery | pass | -| Long contexts | the 8b with `--max-model-len 131072`, `scripts/longctx_check.py` (9 checks) | 9/9: a 98k-token request answered in one engine read with the danger-zone header and rejected without it; the 131,072 backstop and the default 32k/64k caps enforced; `/v1/decisions` has no header escape | -| Docker image build | `docker build --platform linux/amd64`, default and `WITH_ASR=1` variants | both build; the stack inside is package-identical to the live-tested one (PyPI vLLM 0.30.0, torch 2.13.0+cu130, transformers 5.17.0, granite-switch 0.1.1); entry points and imports verified in-container; the `WITH_ASR=1` image holds the pinned speech-model revision as its only snapshot, with `refs/main` pointing at it | -| Serving from the image | the default image (no speech model inside) under enroot (no Docker daemon on the GPU nodes), through `serve.sh`, only the checkpoint mounted, Hub offline: the 8b on one H100 and the 30b on two H100s (`--tensor-parallel-size 2`, set from the GPU count). On each: `e2e_check`, `probe_check`, `accuracy_check` | pass on both: 11/11 and 6/6, accuracy as on the pip path; the log shows `hf_overrides: {'asr_enabled': False}` and never names the speech model. Repeated with the published `ghcr.io/generative-computing/granite-switch-systemone:0.1.3` image (image manifest digest `sha256:77a56fb19768727de6e714d160d32842ddd1715964d51b9a5da5acba604e7971`, every layer's digest verified on download): same results on the 8b and the 30b, and in the speech-input row below | -| Speech input on (`DECISION_SPEECH_INPUT=1`) | the 8b on the default image with the speech model supplied from a mounted Hub cache (`HF_HUB_CACHE`); and, as a negative control, the same switch with no speech model anywhere | the 8b starts and passes the same three checks; the control stops in `serve.sh` with the preflight message naming the model and the fixes, before vLLM starts. The `WITH_ASR=1` image (model baked in) was served on the 8b in an earlier run | -| Container on arm64 | `docker build` of the default image on an aarch64 NVIDIA GB10 host (vLLM 0.30.0, granite-switch 0.1.1 resolved, imports checked in the image); the public `granite-switch-4.1-8b` served from it with `docker run --gpus all`: `e2e_check`, `probe_check`, `accuracy_check` | pass; BoolQ accuracy 0.890 and log-loss 0.283 on arm64, against 0.890 and 0.282 on amd64 with the same checkpoint | -| Multi-arch image build | `image.yml` run by hand (build only, then with push): each of the default and `-asr` images builds natively for amd64 and arm64, and a second job joins them into one tag | both tags are manifest lists holding a `linux/amd64` and a `linux/arm64` image (plus attestations), read from the workflow's own `imagetools inspect` step | -| Speech input check | `scripts/speech_check.py` against a server with `DECISION_SPEECH_INPUT=1` and the speech model from a Hub cache: two spoken English sentences made at run time | both clips repeated back in full, on the pip path and on the pushed arm64 `-asr` image (pulled from the registry, served with `docker run` on the GB10 with the public 8b); without clips and without a synthesizer the script exits 2 | -| Decision Index runs | samples of the Decision Index suite (edition 0.3) through `/v1/systemone` at `--max-model-len 131072`, on the 8b and the 30b, as an end-to-end validation of the endpoint | completes with no errors; scores are not reported here while the methodology is verified, and the model card's published figures are the reference (see the caveats below) | - -## Not tested - -- **Serving the image under Docker on amd64.** The image was served under enroot, on the 8b (one H100) - and the 30b (two H100s; see the table). The arm64 image was served with `docker run` (see the table). That runs the same filesystem and environment as the image, but not - `docker run` itself: its `ENTRYPOINT` (enroot ignores it, so `serve.sh` was started explicitly), - its `--gpus all` handling and `--ipc=host` were not exercised. -- **The 3b.** The only 3b checkpoint, `granite-switch-4.1-3b-preview`, ships twelve - adapters (`citations`, `query_rewrite`, `query_clarification`, `hallucination_detection`, - `answerability`, `factuality-detection`, `policy-guardrails`, `factuality-correction`, - `guardian-core`, `uncertainty`, `requirement-check`, `context-attribution`), none named - `decision`, so this plugin refuses to start on it. No servable 3b exists; the 8b and - 30b are the released checkpoints this plugin can serve. -- **The full 262,144-token danger-zone cap.** The cap clamps to `--max-model-len`, and - every released Switch checkpoint tops out at 131,072 tokens of context; the header path - is exercised to 131,072 (see the long-contexts row above). - -Decision Index caveats: - -- IBM's exact 1,068-row fast subset is unpublished, so the run uses the kit's own stratified - sample of the same size. The model card reports about 0.6 points of serving-path noise on - identical rows. -- The kit's rebuilt release-v1 rows differ in bytes from the lab's pinned file. Per-benchmark - counts match, and the added and GSM8K rows are byte-identical. Other public runs hit the - same condition. We worked around it by running the v2 cut on the rebuilt rows. -- Four sample rows are over 64k tokens, so the default caps reject them and the kit scores - them as wrong. - -These runs show that the endpoint works end to end on a realistic benchmark. They do not -show that the serving path is identical to the one that produced the card's figures. - -## Known gaps - -- **`/v1/decisions` covers OpenAI's schema for text only.** Images, developer, system and assistant - messages and tool calls get a 400, and `/v1/systemone` refuses request-level `instructions`. The adapter - reads one state turn and a question, and nothing else is known to work. The README lists each case. -- **Speculative decoding is untested and may crash the engine.** Decision reads use - `logprob_token_ids`; vLLM issue #42592 (fix #44727, open) crashes on that whenever a - speculative-decoding request shares the batch. We have not run MTP, so whether the decision - route is hit is unverified; the plugin warns at startup when `speculative_config` is set. -- **Decision load slows chat.** Chat p50 for a 32-token reply went from 235 ms idle to - 1,088 ms with 128 concurrent decision clients and no cap. With - `DECISION_MAX_CONCURRENT_REQUESTS=8` and 32 clients it was 362 ms, the excess decision - requests getting 503s. The cap is off by default, and there is no priority between the - two kinds of traffic. -- **No system prompt by default.** The model card's published score used one; this plugin - omits it so decisions can share the prefix cache with chat. How it affects accuracy is - still being evaluated. `DECISION_EXPERIMENTAL_SYSTEM_PROMPT=1` turns it on (tested - live); the default stays off until that is settled. diff --git a/docs/TESTING.md b/docs/TESTING.md index 5ab0d54..3912df9 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -1,7 +1,6 @@ # Testing -Results of each check are in [STATUS.md](STATUS.md). This page says what each test -covers and how to run it. +This page says what each test covers and how to run it. ## Automated (CI, every push and pull request) @@ -14,22 +13,24 @@ pytest ``` CI also builds the wheel and sdist (`python -m build`), checks that `LICENSE` and -`THIRD_PARTY_NOTICES.md` ship inside the wheel, installs the wheel into a clean -environment and runs the same suite against the installed package, and runs +`THIRD_PARTY_NOTICES.md` ship inside the wheel, installs the wheel and runs the +same suite against it (the checkout's `src` is not on the path), and runs [gitleaks](https://github.com/gitleaks/gitleaks) over the full history of all refs (the same scanner the pre-commit hook runs on staged files). -**What CI deliberately does not cover:** anything needing a GPU, a checkpoint or a -live server. Those run by hand (the tokenizer test, the live-server batteries and the load -probes below) because a CI runner cannot reproduce the model deployments they need. -The commands are in the two Manual sections that follow, and results are recorded in -[STATUS.md](STATUS.md). Locally, `pre-commit run --all-files` (after -`pre-commit install`) runs the same lint, header, DCO and secret checks the CI runs; -it needs `gitleaks` on `PATH` for the last of those. +**What CI deliberately does not cover:** anything needing a GPU, a checkpoint or a live +server. Those run by hand (the tokenizer test, the live-server batteries and the load probes +below), and the commands are in the two Manual sections that follow. + +Locally, `pre-commit install --hook-type pre-commit --hook-type commit-msg` sets up the hooks. +`pre-commit run --all-files` then runs the lint, header and secret checks, and each commit is checked +for a `Signed-off-by` line. The first run builds gitleaks, so it needs network access. - `tests/test_plugin.py`: request validation, body limits, the context guard, chunked label reads and their usage counts, adapter-name resolution, the concurrency limiter, - disconnect cancellation, startup checks. + disconnect cancellation, the startup checks and activation probe, and the label-mass log. +- `tests/test_preflight.py`: the speech-model and audio-library checks `serve.sh` runs before vLLM starts. +- `tests/test_serve_sh.py`: `serve.sh`'s speech-input switch, run against a stub `vllm`. - `tests/test_decisions.py`: the `/v1/decisions` wire format with the engine stubbed, including parity with `/v1/systemone` (both routes must build the same internal question) and error shapes. @@ -48,7 +49,6 @@ it needs `gitleaks` on `PATH` for the last of those. Without `typesafe-sdk` the test skips and says what to install. - `tests/test_renderer.py`: prompt layout, label pool, special-token and marker escaping, the experimental system prompt switch. -- `tests/test_plugin.py` also covers the startup activation probe and the label-mass log. ## Manual: needs the checkpoint's tokenizer, no GPU @@ -73,8 +73,8 @@ Python standard library unless noted. | `python3 scripts/e2e_check.py --base-url URL` | Both routes' answer shapes and parity, input flattening, error shapes, usage, vLLM's request fields and `cache_salt`, chat on the same server, prefix-cache hits. Add `--system-prompt` when the server runs with `DECISION_EXPERIMENTAL_SYSTEM_PROMPT=1`. | | `python3 scripts/probe_check.py --base-url URL` | Question cap, size guard (no engine work for oversized requests), body limits, special-token and marker escaping, options without descriptions, disconnect cancellation. | | `python3 scripts/probe_check.py --base-url URL --max-concurrent N` | Adds the concurrency limit: a burst against `DECISION_MAX_CONCURRENT_REQUESTS=N` gets 200s and 503s. | -| `python3 scripts/probe_check.py --base-url URL --load 30` | Adds a load test at 1, 8 and 32 clients with chat latency alongside. Reported, not asserted. | -| `python3 scripts/longctx_check.py --base-url URL --tokenizer /path/to/granite-switch-4.1-8b` | Long-context caps, on a server started with `--max-model-len 131072` and the 8b: the danger-zone header path (a ~98k-token request answered with the header, rejected without it), the 131,072 backstop, the default 32k/64k caps, and that `/v1/decisions` has no header escape. Unlike the scripts above it needs the checkpoint's tokenizer (transformers). | +| `python3 scripts/probe_check.py --base-url URL --load 30` | Adds a load test at 0 (chat only), 1, 8 and 32 clients with chat latency alongside. Reported, not asserted. | +| `python3 scripts/longctx_check.py --base-url URL --tokenizer /path/to/granite-switch-4.1-8b` | Long-context caps, on a server started with `--max-model-len 131072` and the 8b: the danger-zone header path (a request of at least 90k tokens answered with the header, rejected without it), the 131,072 backstop, the default 32k/64k caps, and that `/v1/decisions` has no header escape. Unlike the scripts above it needs the checkpoint's tokenizer (transformers). | | `python3 scripts/accuracy_check.py --base-url URL` | Accuracy, log-loss and Brier score on 200 BoolQ validation rows (public data, downloaded when it runs, needs internet). Fails below 75% accuracy, which catches a wrong invocation marker. To check an upgrade or a changed setting, run it on each deployment with `--save`, then `--compare` the second against the first: it reports items only one run got right and an exact McNemar p-value. | | `python3 scripts/speech_check.py --base-url URL` | Speech input, on a server started with `DECISION_SPEECH_INPUT=1` (see the README). Sends spoken clips to chat completions as `input_audio` and checks that the reply repeats at least 80% of the words. Makes its own clips with `espeak-ng` or macOS `say`, so no audio is stored here; `--clip FILE.wav "the words"` uses your own. Exits 2 if it has no clips. A server missing the audio libraries passes `/health` and fails here with HTTP 500. | | `python3 scripts/load_test.py --base-url URL` | Throughput and latency of `/v1/decisions` at 1, 8, 32, 64 and 128 clients, with the server's prefix-cache hit rate. `--mode unique` (default) gives every request a fresh state; `--mode identical` repeats one body and flatters throughput. Run both and report both. See below. | @@ -117,8 +117,6 @@ How to read it: The client resends at once by default, which makes a capped server look worse than a caller that backs off would see. Pass `--backoff SECONDS` to wait after a non-200. The cap protects chat latency on a shared server; it does not raise decision throughput. -- `--max-num-seqs` is not worth tuning for decisions: they are prefill-only, and vLLM's default was not the - limit in our runs. - Each level is one run of 45 seconds by default, so there is no measure of spread. Repeat a level before drawing conclusions from a difference of a few per cent. @@ -172,8 +170,7 @@ results row by row. A sample is a small fraction of the suite, so differences of so between two runs are within sampling noise unless you pair them on the same rows. Do not read a headline number from `decision_index score` on a sample. It divides by the full -suite's request counts, so a sampled run comes out far too low (a sample that answered every -row can score under 5). To get a number comparable with a full run, subclass the kit's `Suite` +suite's request counts, so a sampled run comes out far too low. To get a number comparable with a full run, subclass the kit's `Suite` so that `rows()` yields only the sampled rows, and pass that to the kit's `score_run`. Compare two runs of the same sample with each other; do not compare a sample's score with a published full-run score without allowing for the sampling noise. @@ -187,16 +184,3 @@ itself show that the serving path matches the one that produced them. Nothing in this repository is copied from another project's API specification. See [THIRD_PARTY_NOTICES.md](../THIRD_PARTY_NOTICES.md) for what is used and under which licence. TypeSafe's own `openapi.json` carries no licence and is not used. - -## Not in this repository - -`scripts/accuracy_check.py` is a small public version of the accuracy comparison (BoolQ -only). The larger comparison on labelled public datasets (BoolQ, SST-5, Banking77): the -`/v1/decisions` route against the same prompt sent over `/v1/chat/completions`, against -the layout with the system prompt, and FlashAttention 2 against 3. That harness and its -results are not published here; STATUS.md says what was compared. The Decision Index -reproduction (kit, suite build, runs and scores) likewise lives outside this repository. - -The Decision Index runs described in STATUS.md use the public reproduction kit -(`apolinario/decision-index`, MIT) driven against `/v1/systemone`; see the Decision Index -section above for how to run it yourself. diff --git a/examples/decisions.md b/examples/decisions.md index 631220c..83fd72a 100644 --- a/examples/decisions.md +++ b/examples/decisions.md @@ -2,8 +2,9 @@ The plugin serves OpenAI's Decisions wire format (`POST /v1/decisions`) alongside `/v1/systemone`, over the same engine and the same decision adapter. Anything written -against OpenAI's Decisions API (the official SDKs, pydantic-ai, plain HTTP) works -against this server by changing `base_url` only. +against OpenAI's Decisions API for text input (the official SDKs, plain HTTP) works by +changing `base_url`. Requests that use parts the adapter cannot read get a 400; see the +[README](../README.md#what-v1decisions-doesnt-support). ## curl @@ -69,16 +70,16 @@ illustrative): { "type": "score", "name": "severity", - "score": 1.87, + "score": 1.73, "probabilities": [ {"value": 0, "label": "can wait", "probability": 0.04}, {"value": 1, "label": "this week", "probability": 0.19}, {"value": 2, "label": "today", "probability": 0.77} ], - "confidence": 0.54 + "confidence": 0.6 } ], - "usage": {"input_tokens": 132, "output_tokens": 3, "total_tokens": 135} + "usage": {"input_tokens": 132, "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, "output_tokens": 3, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 135} } ``` @@ -151,4 +152,5 @@ curl http://localhost:8000/v1/systemone \ `true` with the string `"true"` is rejected (the model cannot distinguish them). - Image `input` parts are rejected with a 400, because the decision adapter is text-only. - Errors use OpenAI's `{"error": {"message", "type", "param", "code"}}` shape. -- `usage` reports input/output/total tokens, without OpenAI's `*_details` breakdowns. +- `usage` reports input, output and total tokens, plus `input_tokens_details.cached_tokens` (prefix-cache hits) and + `output_tokens_details.reasoning_tokens` (always 0). diff --git a/pyproject.toml b/pyproject.toml index fc9bd47..04d1204 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,10 +12,8 @@ license = "Apache-2.0 AND MIT" license-files = ["LICENSE", "THIRD_PARTY_NOTICES.md"] # granite-switch 0.1.1 requires Python 3.11-3.13. requires-python = ">=3.11" -# fastapi/pydantic/starlette are already runtime dependencies of vllm (which the -# container this ships in always has installed) -- not re-declared here on purpose, -# to use whatever versions the vLLM image already pins rather than risking a resolver -# conflict against them. +# fastapi, pydantic and starlette are already vLLM runtime dependencies, so they are not +# declared here. That avoids a resolver conflict with the versions the vLLM image pins. dependencies = [] [project.optional-dependencies] diff --git a/scripts/e2e_check.py b/scripts/e2e_check.py index e670e3b..5e12d79 100755 --- a/scripts/e2e_check.py +++ b/scripts/e2e_check.py @@ -2,11 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 """End-to-end check for a live granite-switch-decision server. -Runs the full wire-format battery against any deployment of this plugin -(local, container or remote), using only the Python standard library, so it -runs anywhere Python 3 runs. Deployment is out of scope by design: whatever -brings the server up (a local container, a cluster job, anything else) only -needs to hand this script the server's base URL. +Runs the wire-format checks against any deployment of this plugin, using only the +standard library. It needs only the server's base URL, however the server was started. python3 scripts/e2e_check.py --base-url http://localhost:8000 @@ -599,7 +596,7 @@ def probe(label: str, setup: tuple[str, dict], text: str, expect_hits: bool) -> def main() -> int: - parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--base-url", required=True, help="server base URL, e.g. http://host:8000") parser.add_argument("--model", default=None, help="model name (default: first /v1/models entry)") parser.add_argument("--tolerance", type=float, default=0.05, help="parity tolerance on probabilities") diff --git a/scripts/longctx_check.py b/scripts/longctx_check.py index 685d161..21b4717 100644 --- a/scripts/longctx_check.py +++ b/scripts/longctx_check.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 -"""Long-context checks for a live granite-switch-decision server started with the -8b checkpoint and --max-model-len 131072 (the checkpoint's maximum). Needs the +"""Long-context checks for a live granite-switch-decision server. + +Start the server with the 8b checkpoint and --max-model-len 131072 (the checkpoint's maximum). Needs the checkpoint's tokenizer (transformers, unlike the other scripts, which are stdlib-only) to size the probe states. The 262,144-token danger-zone cap clamps to --max-model-len, so this exercises it up to 131,072; no released Switch checkpoint has more context. @@ -11,12 +12,12 @@ Exit code 0 means every check passed. Checks: - default_caps a ~98k-token state is rejected without the header (the 32k + default_caps a state of at least 90k tokens is rejected without the header (the 32k per-question cap fires first; the 64k per-request cap is checked separately with three ~22k questions) danger_zone the same state with TypeSafe-Danger-Zone-Long-Context: true is answered in one engine read, and its usage counts the tokens - max_model_len a ~152k-token state with the header is rejected at the + max_model_len a state of at least 140k tokens with the header is rejected at the 131,072 backstop decisions_route /v1/decisions has no danger-zone escape: the big state is rejected there too @@ -64,9 +65,9 @@ def post(url, body, headers=None, timeout=1800): def main(): - ap = argparse.ArgumentParser() + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--base-url", required=True) - ap.add_argument("--tokenizer", required=True, help="path to the granite-switch-4.1-8b snapshot") + ap.add_argument("--tokenizer", required=True, help="the 8b checkpoint's tokenizer: a local path or a Hub id") ap.add_argument("--model", default="granite-switch") args = ap.parse_args() diff --git a/scripts/probe_check.py b/scripts/probe_check.py index 46fa240..e92cf6e 100644 --- a/scripts/probe_check.py +++ b/scripts/probe_check.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 -"""Robustness checks for a live granite-switch-decision server: limits, escaping, -cancellation, error handling, and optional load. Standard library only. +"""Robustness checks for a live granite-switch-decision server (standard library only). + +Limits, escaping, cancellation, error handling and, optionally, load. python3 scripts/probe_check.py --base-url http://localhost:8000 python3 scripts/probe_check.py --base-url http://localhost:8000 --load 30 @@ -18,8 +19,8 @@ cancellation a client that disconnects mid-request leaves the engine idle concurrency with --max-concurrent N (the server's DECISION_MAX_CONCURRENT_REQUESTS), a burst gets 200s and 503s - load with --load SECONDS, throughput and latency at 1, 8 and 32 - clients, and chat latency alongside (reported, not asserted) + load with --load SECONDS, throughput and latency at 0 (chat + only), 1, 8 and 32 clients, with chat latency alongside (reported, not asserted) Run it on a server with no other traffic: cancellation and load read vLLM's /metrics. """ @@ -275,7 +276,7 @@ def chat_loop() -> None: def main() -> int: - parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--base-url", required=True) parser.add_argument("--model", default="granite-switch") parser.add_argument("--timeout", type=float, default=300.0) diff --git a/serve.sh b/serve.sh index 77ed9af..8d93dbe 100755 --- a/serve.sh +++ b/serve.sh @@ -21,7 +21,7 @@ esac TP=${TENSOR_PARALLEL_SIZE:-$(python3 -c "import torch; print(max(torch.cuda.device_count(), 1))")} export DECISION_ADAPTER_NAME=${DECISION_ADAPTER_NAME:-decision} # --dtype bfloat16: composed checkpoints carry torch_dtype null (granite-switch#126). -# flash_attn_version=2: avoids the FA3 schedule bug (granite-switch#139, still open). +# flash_attn_version=2: avoids the FA3 schedule bug (granite-switch#139). # --max-logprobs 256: lets chat clients that read labels by top-K ask for more than 20. # --enable-prompt-tokens-details: reports prefix-cache hits in usage.input_tokens_details. # No global temperature override: the plugin sends temperature 0 on every decision diff --git a/src/granite_switch_decision/plugin.py b/src/granite_switch_decision/plugin.py index d781226..105648a 100644 --- a/src/granite_switch_decision/plugin.py +++ b/src/granite_switch_decision/plugin.py @@ -41,7 +41,7 @@ # jev: request body <= 32 MiB. MAX_BODY_BYTES = 32 * 1024 * 1024 -# jev: nesting <= 64 levels anywhere in the request body. +# jev: at most 64 nested levels inside each of `state`, `instructions` and `criteria`. MAX_NESTING_DEPTH = 64 # jev's context limits in tokens, raised by its long-context header; also capped by --max-model-len. DEFAULT_PER_QUESTION_TOKEN_LIMIT = 32_000 @@ -90,7 +90,7 @@ def _resolve_adapter_name(chat) -> str: return _DEFAULT_ADAPTER_NAME raise RuntimeError( f"{_ADAPTER_NAME_ENV} is not set, and this checkpoint's adapters {adapter_names!r} " - f"include no {_DEFAULT_ADAPTER_NAME!r} -- set {_ADAPTER_NAME_ENV} to the decision adapter's name." + f"include no {_DEFAULT_ADAPTER_NAME!r}. Set {_ADAPTER_NAME_ENV} to the decision adapter's name." ) @@ -154,8 +154,8 @@ def _check_joint(value: bool) -> bool: _NO_REQUEST_CONTEXT = ( - "request-level `instructions` and developer or system messages are not supported: the decision adapter reads " - "the state and each question's own instructions. Put the context in the state." + "request-level `instructions` are not supported: the decision adapter reads the state and each question's " + "own instructions. Put the context in the state." ) @@ -253,7 +253,7 @@ class DecisionQuestionImage(BaseModel): class DecisionInputJson(BaseModel): - """A JSON object, or an array of objects, rendered as compact JSON text like an object `state`.""" + """A JSON object, or an array of objects, rendered as single-line JSON text, like an object `state`.""" model_config = ConfigDict(extra="forbid") @@ -281,7 +281,7 @@ class DecisionInputImage(DecisionQuestionImage): class DecisionQuestion(BaseModel): """One OpenAI Decisions question (`predicate`/`choice`/`score`), with the field constraints of OpenAI's published spec: `instructions` required (a string, or text and - JSON parts), 2-255 unique choices, at most 10 score levels. `name` is optional, since + JSON parts), 2-255 unique choices, 2-10 score levels. `name` is optional, since answers are positional, so duplicate or missing names cannot collide (the response echoes the name or null).""" @@ -507,9 +507,8 @@ def _validation_error_response(e: PydanticValidationError) -> JSONResponse: def _error(status: int, message: str) -> JSONResponse: - """For the status codes jev's OpenAPI spec does NOT give a documented body shape - for 400/413/415 (only 422's HTTPValidationError is in the spec); see - `_validation_error_response` for 422 itself.""" + """Error body for 413 and 415, which jev's spec does not document; 422 is + `_validation_error_response`.""" return JSONResponse( content={"error": {"message": message, "code": int(status)}}, status_code=int(status), @@ -519,7 +518,7 @@ def _error(status: int, message: str) -> JSONResponse: def _openai_error(status: int, message: str, param: str | None = None, code: str | None = None) -> JSONResponse: """OpenAI's documented error body for the /v1/decisions route (`{"error": {"message", "type", "param", "code"}}`): the official SDKs parse - this shape on this route, which is the point of speaking it.""" + this shape on this route.""" return JSONResponse( content={ "error": { @@ -599,8 +598,7 @@ def _softmax(logps: list[float]) -> list[float]: def _normalize_probs(probs: list[float]) -> list[float]: - """Shared normalization step both confidence formulas below start from -- - uniform if the total is zero.""" + """First step of both confidence formulas: normalize, falling back to uniform if the total is zero.""" total = sum(probs) if total == 0: return [1.0 / len(probs)] * len(probs) @@ -608,9 +606,8 @@ def _normalize_probs(probs: list[float]) -> list[float]: def _choice_confidence(probs: list[float]) -> float: - """jev's `confidence` formula for a choice answer. It is not the top-1 probability: it - rescales the top probability so a uniform distribution gives 0 and all mass on one - option gives 1, `(max(p) - 1/n) / (1 - 1/n)`.""" + """jev's `confidence` for a choice: the top probability rescaled so a uniform distribution + gives 0 and all mass on one option gives 1, `(max(p) - 1/n) / (1 - 1/n)`.""" if len(probs) == 1: return 1.0 normalized = _normalize_probs(probs) @@ -619,10 +616,9 @@ def _choice_confidence(probs: list[float]) -> float: def _score_confidence(probs: list[float]) -> float: - """jev's actual `confidence` formula for a score answer: 1 minus the expected - distance from the modal level, divided by the mean distance every level has from - the scale's midpoint (a uniform distribution's own expected distance). With two - levels this reduces to the same formula as `_choice_confidence`.""" + """jev's `confidence` for a score: 1 minus the expected distance from the modal level over + the mean distance of a uniform distribution (levels measured from the scale's midpoint), + floored at 0. With two levels it equals `_choice_confidence`.""" if len(probs) == 1: return 1.0 normalized = _normalize_probs(probs) @@ -708,7 +704,7 @@ def _label_logprobs_by_id(top_logprobs) -> dict[int, float]: def _log_label_mass(name: str, label_logprobs: list[float]) -> float: - """Log, and return, the probability mass the option labels hold before normalising. Low mass + """Log, and return, the probability mass the option labels hold before normalizing. Low mass means the model wanted a token outside the labels (a wrong marker, a broken layout). Observation only: answers are never gated on it.""" mass = sum(math.exp(lp) for lp in label_logprobs) @@ -825,7 +821,7 @@ async def _answer_question( `reads` bounds the engine reads in flight across one request's questions.""" if q.type == "choice": keys = list(q.criteria) - # An option without a description is described by its key, as vLLM's native strategy does. + # An option without a description is described by its key. descs = [k if q.criteria[k] in (None, "") else q.criteria[k] for k in keys] elif q.type == "noul": crit = q.criteria if isinstance(q.criteria, dict) else {} @@ -837,7 +833,7 @@ async def _answer_question( msgs, label_ids = renderer(tok, state, q.instructions, descs) - # Over 128 labels: one request per chunk, same prompt, so later chunks hit the prefix cache. + # Over 128 labels: one request per chunk over the same prompt, so chunks can share the prefix cache. async def read(chunk: list[int]): async with reads: return await _fetch_label_logprobs(chat, served_model, adapter_name, msgs, chunk, options.cache_salt) @@ -975,7 +971,7 @@ async def _cancel_pending(tasks: list[asyncio.Task]) -> None: def _flatten_parts(parts: list[Any]) -> str: - """Content parts as one string, joined with blank lines: text as it is, JSON as compact + """Content parts as one string, joined with blank lines: text as it is, JSON as single-line JSON (the rule an object `state` follows on the systemone route). Image parts are rejected, as `box` questions are on the systemone route.""" texts: list[str] = [] @@ -1229,15 +1225,15 @@ def _route_exists(app: FastAPI, path: str) -> bool: class DecisionPlugin: - """`vllm.endpoint_plugins` entry point (see module docstring). Loaded only when - `granite_switch_decision` is in `VLLM_PLUGINS`; see this repo's README and Dockerfile.""" + """`vllm.endpoint_plugins` entry point. Loaded only when `granite_switch_decision` is in + `VLLM_PLUGINS`; see this repo's README and Dockerfile.""" name = "granite_switch_decision" required_tasks: tuple[str, ...] | None = ("generate",) def attach_router(self, app: FastAPI) -> None: - # vLLM also ships these routes. Whichever registers first would silently shadow - # the other, with different wire details, so refuse to start. + # Newer vLLM also ships these routes. Whichever registers first would silently shadow + # the other (the wire details differ), so refuse to start. taken = sorted(path for path in _ROUTE_PATHS if _route_exists(app, path)) if taken: raise RuntimeError( diff --git a/src/granite_switch_decision/preflight.py b/src/granite_switch_decision/preflight.py index d7fc680..c092831 100644 --- a/src/granite_switch_decision/preflight.py +++ b/src/granite_switch_decision/preflight.py @@ -38,7 +38,8 @@ def _offline(env: Mapping[str, str]) -> bool: def hub_cache_dir(env: Mapping[str, str]) -> Path: - """Where huggingface_hub looks for cached models, by its own precedence.""" + """Where huggingface_hub looks for cached models: HF_HUB_CACHE, HUGGINGFACE_HUB_CACHE, then HF_HOME + (default ~/.cache/huggingface). XDG_CACHE_HOME is not consulted.""" for name in ("HF_HUB_CACHE", "HUGGINGFACE_HUB_CACHE"): if env.get(name): return Path(env[name]) diff --git a/src/granite_switch_decision/renderer.py b/src/granite_switch_decision/renderer.py index 352f7ae..01c4947 100644 --- a/src/granite_switch_decision/renderer.py +++ b/src/granite_switch_decision/renderer.py @@ -53,7 +53,7 @@ def check_settings() -> None: def text_of(x: object) -> str: """`state`/a description may be a dict/list (jev allows `string | object | array` - for both); render JSON compactly, pass strings through, "" for None.""" + for both); render JSON on one line, pass strings through, "" for None.""" if isinstance(x, (dict, list)): return json.dumps(x, ensure_ascii=False) return "" if x is None else str(x)