diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b46b346 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +.git +.github +tests +docs +scripts +examples +build +dist +**/__pycache__ +**/*.egg-info +*.webp diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..1eab524 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,167 @@ +name: Few-shot benchmark + +# Compares adaptive-classifier with SetFit and logistic regression on public datasets. +# Run it by hand: Actions -> Few-shot benchmark -> Run workflow. It is not part of the test +# suite because it downloads datasets, trains SetFit on a CPU, and takes a long time. +# Results appear in the run summary and as downloadable artifacts. See docs/benchmark.md. + +on: + workflow_dispatch: + inputs: + datasets: + description: "Datasets, comma separated (ag_news, emotion, banking77)" + default: "ag_news,emotion" + shots: + description: "Training examples per class, space separated" + default: "4 16" + seeds: + description: "Random seeds (different few-shot samples), 1-10" + default: "3" + methods: + description: "Methods to run (adaptive adaptive-proto logreg setfit)" + default: "adaptive adaptive-proto logreg setfit" + model: + description: "Sentence encoder shared by every method" + default: "sentence-transformers/all-MiniLM-L6-v2" + max_test: + description: "Test examples per dataset (a fixed random subset)" + default: "1000" + +permissions: + contents: read + +jobs: + plan: + # Validates the inputs and that each dataset loads, in seconds, before the long jobs start. + # Inputs reach the shell only as environment variables, never interpolated into a command. + runs-on: ubuntu-latest + timeout-minutes: 15 + outputs: + datasets: ${{ steps.plan.outputs.datasets }} + shots: ${{ steps.plan.outputs.shots }} + seeds: ${{ steps.plan.outputs.seeds }} + methods: ${{ steps.plan.outputs.methods }} + model: ${{ steps.plan.outputs.model }} + max_test: ${{ steps.plan.outputs.max_test }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Validate inputs + id: plan + env: + IN_DATASETS: ${{ inputs.datasets }} + IN_SHOTS: ${{ inputs.shots }} + IN_SEEDS: ${{ inputs.seeds }} + IN_METHODS: ${{ inputs.methods }} + IN_MODEL: ${{ inputs.model }} + IN_MAX_TEST: ${{ inputs.max_test }} + run: | + pip install numpy + python scripts/benchmark_fewshot.py plan \ + --datasets "$IN_DATASETS" --shots "$IN_SHOTS" --seeds "$IN_SEEDS" \ + --methods "$IN_METHODS" --model "$IN_MODEL" --max-test "$IN_MAX_TEST" > plan.json + cat plan.json + python - <<'PY' >> "$GITHUB_OUTPUT" + import json + plan = json.load(open("plan.json")) + print("datasets=" + json.dumps(plan["datasets"])) + for key in ("shots", "seeds", "methods", "model", "max_test"): + print(f"{key}={plan[key]}") + PY + + - name: Check that every dataset loads + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + DATASETS: ${{ steps.plan.outputs.datasets }} + run: | + pip install datasets + python scripts/benchmark_fewshot.py check $(python -c "import json,os; print(' '.join(json.loads(os.environ['DATASETS'])))") + + benchmark: + needs: plan + runs-on: ubuntu-latest + timeout-minutes: 350 + strategy: + fail-fast: false + matrix: + dataset: ${{ fromJSON(needs.plan.outputs.datasets) }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + + - name: Cache Hugging Face downloads + uses: actions/cache@v4 + with: + path: ~/.cache/huggingface + key: hf-benchmark-${{ needs.plan.outputs.model }}-${{ matrix.dataset }} + + - name: Install + run: | + python -m pip install --upgrade pip + pip install torch --index-url https://download.pytorch.org/whl/cpu + pip install -e ".[benchmark]" + pip list 2>/dev/null | grep -iE "^(torch|transformers|sentence-transformers|setfit|datasets|scikit-learn|adaptive-classifier) " + + - name: Run benchmark + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + DATASET: ${{ matrix.dataset }} + SHOTS: ${{ needs.plan.outputs.shots }} + SEEDS: ${{ needs.plan.outputs.seeds }} + METHODS: ${{ needs.plan.outputs.methods }} + MODEL: ${{ needs.plan.outputs.model }} + MAX_TEST: ${{ needs.plan.outputs.max_test }} + run: | + # SHOTS and METHODS are space-separated lists validated by the plan job. + python scripts/benchmark_fewshot.py run \ + --dataset "$DATASET" --shots $SHOTS --seeds "$SEEDS" --methods $METHODS \ + --model "$MODEL" --max-test "$MAX_TEST" --out "results/$DATASET.json" + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.dataset }} + path: results/*.json + if-no-files-found: warn + + summary: + needs: [plan, benchmark] + if: always() && needs.plan.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - uses: actions/download-artifact@v4 + with: + pattern: results-* + path: results + merge-multiple: true + + - name: Build the report + run: | + pip install numpy + if ls results/*.json >/dev/null 2>&1; then + python scripts/benchmark_fewshot.py report results/*.json --out benchmark-report.md > /dev/null + cat benchmark-report.md >> "$GITHUB_STEP_SUMMARY" + else + echo "No result files were produced; see the benchmark jobs." | tee benchmark-report.md >> "$GITHUB_STEP_SUMMARY" + fi + + - uses: actions/upload-artifact@v4 + with: + name: benchmark-report + path: benchmark-report.md diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6a27b8d..19e2c1f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -21,16 +21,15 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel + pip install build - name: Run tests env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - pip install pytest pytest-cov pytest-randomly psutil - pip install -e . + pip install -e ".[test]" pytest tests/ - name: Build package run: | - python setup.py sdist bdist_wheel + python -m build - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8193114..7bbdd2b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,28 +5,83 @@ on: branches: [ main ] pull_request: branches: [ main ] + workflow_dispatch: + schedule: + # Weekly, so breakage on the Hugging Face Hub or in new transformers/torch + # releases shows up without waiting for the next PR. + - cron: '17 5 * * 1' + +# A newer push to the same PR supersedes the run in progress. +concurrency: + group: tests-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + models: + # Download the Hugging Face models once, with retries. The test jobs run + # offline from the cache, so a Hub rate limit (HTTP 429) or outage fails + # this one step instead of dozens of tests across the matrix. + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Cache Hugging Face models + id: hf-cache + uses: actions/cache@v4 + with: + path: ~/.cache/huggingface + key: hf-models-${{ hashFiles('scripts/prefetch_test_models.py') }} + + - name: Download models + if: steps.hf-cache.outputs.cache-hit != 'true' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + pip install huggingface_hub + python scripts/prefetch_test_models.py + unit-tests: + needs: models runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + # The two ends of the supported range (requires-python >=3.10). + python-version: ['3.10', '3.13'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - name: Set up Python 3.12 - uses: actions/setup-python@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: ${{ matrix.python-version }} + cache: pip + + - name: Restore Hugging Face models + uses: actions/cache/restore@v4 + with: + path: ~/.cache/huggingface + key: hf-models-${{ hashFiles('scripts/prefetch_test_models.py') }} + fail-on-cache-miss: true - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e . - pip install pytest pytest-cov psutil + pip install -e ".[test]" - - name: Run unit tests + # Everything except tests marked `integration`, run offline against the + # models prefetched above. Tests marked `integration` run in the job below. + - name: Run tests env: HF_TOKEN: ${{ secrets.HF_TOKEN }} + HF_HUB_OFFLINE: '1' run: | pytest tests/ -v --cov=adaptive_classifier --cov-report=xml --cov-report=term -m "not integration" @@ -35,7 +90,7 @@ jobs: with: file: ./coverage.xml flags: unittests - name: codecov-umbrella + name: codecov-${{ matrix.python-version }} fail_ci_if_error: false integration-tests: @@ -44,24 +99,31 @@ jobs: timeout-minutes: 30 # Generous timeout for model downloads steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.12 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.12' + cache: pip + + - name: Restore Hugging Face models + uses: actions/cache/restore@v4 + with: + path: ~/.cache/huggingface + key: hf-models-${{ hashFiles('scripts/prefetch_test_models.py') }} + fail-on-cache-miss: true - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e . - pip install pytest psutil + pip install -e ".[test]" - name: Run integration tests env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - pytest tests/test_enterprise_classifiers_integration.py -v -m "integration" --tb=short + pytest tests/ -v -m "integration" --tb=short - name: Integration test summary if: always() diff --git a/.gitignore b/.gitignore index 3d255ad..f0797e7 100644 --- a/.gitignore +++ b/.gitignore @@ -173,3 +173,6 @@ cython_debug/ **/.DS_store demo_classifier/ scripts/benchmark_results/ + +# Training checkpoints written by the benchmark harness +checkpoints/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..882f966 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,40 @@ +# Changelog + +## 0.3.0 + +### Added +- `SklearnAdaptiveClassifier`: scikit-learn estimator (`fit`, `partial_fit`, `predict`, `predict_proba`, `score`) usable in `Pipeline`, `cross_val_score` and `GridSearchCV`. +- `forget(labels)` and `remove_examples(texts, label=None, retrain=True)` to delete classes or fix wrong labels. +- `ood_score()` / `is_ood()` out-of-distribution detection, and `predict(..., abstain_below=, abstain_ood=)`. New config keys `ood_threshold` and `ood_min_radius`. +- `docs/deployment.md` and `examples/portable_inference.py`: run a saved classifier without PyTorch or FAISS (multilingual models, .NET, JavaScript). +- `save()` now writes the tokenizer files and the resolved pooling mode, so a saved directory is self-contained. + +- `calibrate(texts, labels)` fits one temperature on held-out labelled data so confidences match how often the model is right, and `calibration_report(...)` measures accuracy, expected calibration error and NLL. `predict`, `predict_batch` and `abstain_below` then use calibrated probabilities. Calibration is saved with the model and discarded when the set of classes changes. +- `predict_set(text, alpha=0.1)`: split-conformal prediction sets that contain the true label with probability at least `1 - alpha`. + +- `suggest_labels(texts, n, strategy, diverse)`: active learning. Ranks unlabeled texts by how much a label would help (`margin`, `entropy`, `least_confidence`, or `ood` to discover unseen classes), optionally spreading the picks so near-duplicates are not all chosen. +- `drift_report(texts)`: tests whether a window of incoming texts has moved away from the known classes (share of out-of-distribution texts against an expected rate, one-sided binomial test). + +- **Serving.** `python -m adaptive_classifier.serving ./model` runs a FastAPI server (`/predict`, `/predict_batch`, `/predict_set`, `/ood`, and optional authenticated `/examples`, `/forget`, `/remove_examples`); `create_app(classifier)` returns the app for embedding. Install with `pip install "adaptive-classifier[serve]"`. See `docs/serving.md`. `docker/Dockerfile` is provided but has not been built in the environment it was written in. +- Async methods `apredict`, `apredict_batch` and `aadd_examples` that run in a worker thread. +- A saved directory is self-contained for loading: the tokenizer saved next to the model is used instead of fetching the base model's repository, so an ONNX-saved classifier starts without Hub access. + + +### Changed (default behaviour: predictions change) +- **The neural head is now actually trained.** It used to get about ten optimiser steps and stop at the first loss plateau, so it could not learn even a 2-D XOR from 120 examples. It now trains for at least `head_steps` (default 300) steps with a cosine-decayed `head_learning_rate` (default 0.003). Small memories take roughly 1-2 s per `add_examples` call; lower `head_steps` if you add examples very frequently. +- **Prototype scores are sharper.** They are now `softmax(-distance² / prototype_temperature)` (default 0.25) instead of `softmax(exp(-distance))`, which barely separated the nearest class from the rest (about 0.45 vs 0.21), so a confident head could always outvote it. `prototype_temperature=None` restores the old scoring. +- **Classes with few examples lean on their prototypes.** Below `new_class_example_threshold` examples the head's weight ramps up from zero, instead of the old fixed 0.3 prototype / 0.7 head split. On separable synthetic data the old default scored 0.55-0.73 with fewer than 10 examples per class where prototypes alone scored 0.96-1.0; accuracy no longer drops off a cliff at the threshold. Trade-off: a class added with a single example is recognised less often than before, because the old split biased the head toward new classes at the expense of existing ones. Set both `new_class_prototype_weight` and `new_class_neural_weight` to restore a fixed split. +- Classifiers saved by an earlier version keep their old scoring when loaded, so their predictions do not change. +- **Python 3.10 or newer is required** (3.8 and 3.9 are end-of-life). +- Packaging moved to a `[project]` table in `pyproject.toml`; `setup.py` and `requirements.txt` are gone. Install extras: `[test]`, `[scripts]`. `tqdm` is no longer a core dependency. +- `clear_memory(labels=[...])` now removes those classes completely (it used to leave them in the label map and neural head, so they could still be predicted). + +### Fixed +- **Thread safety.** Predicting while another thread added or forgot classes crashed with errors such as "selected index k out of range", and concurrent `add_examples` calls corrupted each other. Every public operation now holds a per-instance lock, so one classifier can be shared between threads. Throughput scales with processes, not threads. +- `predict_batch(k=...)` scored only `k` classes before blending, so with more than `k` classes its scores differed from `predict` for the same text. It now scores every class and trims to `k`. +- Accuracy on small training sets (see above): the default configuration got only 75-92% of its own training data right on perfectly separable data with fewer than 10 examples per class. +- Integer (or other non-string) labels corrupted a classifier on save/reload, duplicating classes. `add_examples` now rejects non-string labels with a clear error; convert with `str(label)`. +- `predict(k=-1)` returned a silently truncated list; negative or non-integer `k` now raises `ValueError`. +- `None` as a text or label, and mixed label types, now raise a clear `ValueError` before anything is changed. +- Loading a directory without `config.json` now says so instead of reporting an invalid Hub repo id. + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d104beb..3f95e1f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,8 +61,7 @@ source venv/bin/activate # On Windows: venv\Scripts\activate 3. Install dependencies: ```bash -pip install -r requirements.txt -pip install -e . +pip install -e ".[test]" ``` 4. Run tests: diff --git a/README.md b/README.md index d884410..b1a5e5f 100644 --- a/README.md +++ b/README.md @@ -108,11 +108,8 @@ pip install adaptive-classifier git clone https://github.com/codelion/adaptive-classifier.git cd adaptive-classifier -# Install in development mode -pip install -e . - -# Install test dependencies (optional) -pip install pytest pytest-cov pytest-randomly +# Install in development mode with test dependencies +pip install -e ".[test]" ``` --- @@ -147,16 +144,9 @@ Classifiers saved before 0.2.0 keep CLS pooling when reloaded, so upgrading does not change their predictions. Retrain, or pass `pooling` explicitly, to pick up the new behaviour. -## Prototype and neural weights - -A prediction mixes two signals: similarity to the class prototypes, and the -trained neural head. `prototype_weight` and `neural_weight` control the mix. +## How a prediction is made -Both keys existed before 0.2.0 but were ignored. Every prediction path used a -hardcoded 0.7 / 0.3, so `predict` and `predict_batch` could also disagree once a -caller changed the config. They work now, and the best split depends on how well -your encoder separates your classes: an encoder that bunches everything together -produces weak prototypes and is better off leaning on the head. +A prediction mixes two signals: the **prototype score** (how close the text is to each class's mean embedding) and the **neural head** (a small trained network on the same embeddings). `prototype_weight` and `neural_weight` (default 0.7 / 0.3) control the mix. ```python classifier = AdaptiveClassifier( @@ -165,10 +155,15 @@ classifier = AdaptiveClassifier( ) ``` -A class with few examples has an unreliable prototype, so it leans on the head -until it is established. That is configurable too, via -`new_class_example_threshold` (default 10), `new_class_prototype_weight` (0.3) -and `new_class_neural_weight` (0.7). +The best split depends on how well your encoder separates your classes: an encoder that bunches everything together produces weak prototypes and is better off leaning on the head. + +**Classes with few examples.** Until a class has `new_class_example_threshold` examples (default 10), the head's weight ramps up linearly from zero to `neural_weight` and the prototype takes the rest, so a brand-new class is judged almost entirely by its prototype. (Before 0.3.0 such classes used a fixed 0.3 / 0.7 split in favour of the head, and a head fitted to a handful of points could confidently outvote a correct prototype. To get a fixed split back, set both `new_class_prototype_weight` and `new_class_neural_weight`.) + +**Prototype sharpness.** Prototype scores are `softmax(-distance² / prototype_temperature)` with a default temperature of 0.25. Lower values make the nearest class stand out more. Set `prototype_temperature` to `None` for the scoring used before 0.3.0, which barely separated the nearest class from the rest. + +**Head training.** The head is trained for at least `head_steps` optimiser steps (default 300) with a cosine-decayed `head_learning_rate` (default 0.003). That takes roughly 1-2 seconds on a small memory; lower `head_steps` (for example to 100) if you add examples very frequently. Before 0.3.0 the head got about ten steps, too few to learn even a simple XOR pattern, which prototypes alone cannot separate. + +Classifiers saved before 0.3.0 keep their old scoring when reloaded, so upgrading does not change their predictions. Retrain, or set the keys above, to adopt the new behaviour. ## ⚡ Quick Start @@ -315,8 +310,96 @@ ONNX: 2.1ms/query (4.0x faster) ✓ > **Note:** ONNX optimization is included by default. For GPU inference, PyTorch is automatically used for best performance. +#### Multilingual models and other runtimes (.NET, JavaScript, ...) + +Any multilingual encoder works as the base model, and a saved classifier can be run from C#, Node or the browser through an ONNX runtime. See the [deployment guide](docs/deployment.md) and the dependency-light reference implementation in [`examples/portable_inference.py`](examples/portable_inference.py). + ## Advanced Usage +### scikit-learn Interface + +`SklearnAdaptiveClassifier` works with `Pipeline`, `cross_val_score` and `GridSearchCV`. `X` is a list of strings. + +```python +from adaptive_classifier import SklearnAdaptiveClassifier +from sklearn.model_selection import cross_val_score + +clf = SklearnAdaptiveClassifier("sentence-transformers/all-MiniLM-L6-v2") +clf.fit(texts, labels) +clf.predict(["Where is my refund?"]) # class labels +clf.predict_proba(["Where is my refund?"]) # columns follow clf.classes_ +cross_val_score(clf, texts, labels, cv=5) + +# The library's strength: keep learning, including brand-new classes +clf.partial_fit(["App crashes on login"], ["bug"]) + +clf.classifier_.save("./model") # the underlying AdaptiveClassifier +``` + +`fit` starts over each time; `partial_fit` adds to what is already learned. A fitted estimator holds a FAISS index and can't be pickled, so save `classifier_` instead. + +### Calibrated Confidence and Prediction Sets + +Raw scores are not probabilities: a model can say 0.9 and be right 70% of the time. Fit a calibration on labelled examples the model was *not* trained on: + +```python +info = classifier.calibrate(held_out_texts, held_out_labels) +info["ece_before"], info["ece_after"] # expected calibration error, lower is better + +classifier.predict(text) # confidences now match how often the model is right +classifier.predict(text, abstain_below=0.8) # so thresholds like this mean what they say + +# A set of labels that contains the true one at least 90% of the time +classifier.predict_set(text, alpha=0.1) # [("billing", 0.55), ("refunds", 0.31)] +classifier.calibration_report(test_texts, test_labels) # check it on separate data +``` + +One easy input gives a one-label set; an ambiguous one gives several. Calibration is saved with the model and discarded if you add or forget a class; re-run it after substantial new data. + +### Active Learning and Drift + +```python +# Which of these 5,000 unlabeled texts should a person label next? +for index, score in classifier.suggest_labels(unlabeled, n=20, strategy="margin", diverse=True): + print(unlabeled[index]) + +classifier.suggest_labels(unlabeled, n=20, strategy="ood") # hunt for classes you have not defined yet + +# Has this week's traffic moved away from what the model knows? +report = classifier.drift_report(recent_texts) +report["drifted"], report["ood_rate"], report["p_value"] +``` + +`margin` (the default) finds texts on a decision boundary; `ood` finds texts unlike any known class. Uncertainty sampling is not guaranteed to beat labeling at random: when classes overlap so much that the confusion is irreducible noise, labeling the confusing texts teaches the model little (it did not in a synthetic test with heavily overlapping classes). It earns its keep when ambiguity comes from missing data, and `ood` is the reliable way to find classes you have not defined yet. + +### Serving + +```bash +pip install "adaptive-classifier[serve]" +python -m adaptive_classifier.serving ./my_classifier --port 8000 +curl -s localhost:8000/predict -H 'content-type: application/json' -d '{"text": "where is my refund?"}' +``` + +Predictions, batches, conformal sets and out-of-distribution checks over HTTP, plus optional API-key-protected endpoints for adding examples and classes while it runs. One classifier is safe to share between threads, and `await classifier.apredict(...)` keeps an event loop responsive. See [docs/serving.md](docs/serving.md) and `docker/Dockerfile`. + +### Fixing Mistakes and Knowing When to Abstain + +```python +# Remove a wrongly labelled example, or a whole class +classifier.remove_examples(["Refund my order"], label="technical") +classifier.forget("obsolete_class") + +# Detect inputs unlike anything the classifier has seen +classifier.is_ood("completely unrelated text") # True / False +classifier.ood_score("completely unrelated text") # ~1 or below = familiar, higher = further out + +# Abstain instead of guessing (an empty list means "don't know") +classifier.predict(text, abstain_below=0.6) # low confidence +classifier.predict(text, abstain_ood=True) # out of distribution +``` + +Out-of-distribution scores compare a text's distance from the nearest class prototype to that class's own spread, so no per-model tuning of absolute distances is needed. The default `ood_threshold` of 1.25 is a starting point; check it on held-out data. + ### Adding New Classes Dynamically ```python diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..8481f14 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,40 @@ +# Serve a saved AdaptiveClassifier over HTTP. +# +# docker build -f docker/Dockerfile -t adaptive-classifier . +# docker run --rm -p 8000:8000 -v "$PWD/my_classifier:/model:ro" adaptive-classifier +# curl -s localhost:8000/predict -H 'content-type: application/json' -d '{"text": "where is my refund?"}' +# +# Updates are off by default. To allow them (and persist what is learned), mount the +# model read-write and pass flags, for example: +# docker run --rm -p 8000:8000 -v "$PWD/my_classifier:/model" \ +# -e ADAPTIVE_CLASSIFIER_API_KEY=change-me adaptive-classifier \ +# /model --host 0.0.0.0 --allow-updates --save-dir /model +# +# A classifier saved with the default `save()` includes an ONNX copy of the encoder and the +# tokenizer, so it starts without reaching the Hugging Face Hub. One saved with +# `include_onnx=False` fetches its base model from the Hub on startup; mount a populated +# Hugging Face cache at /cache/huggingface (or bake the model in) for offline use. +FROM python:3.12-slim + +ENV PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + HF_HOME=/cache/huggingface + +WORKDIR /app + +# CPU-only PyTorch keeps the image a few GB smaller than the default CUDA build. +RUN pip install --index-url https://download.pytorch.org/whl/cpu torch + +COPY pyproject.toml README.md LICENSE ./ +COPY src ./src +RUN pip install ".[serve]" + +RUN useradd --create-home --uid 1000 app && mkdir -p /cache/huggingface && chown -R app /cache +USER app + +EXPOSE 8000 +HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ + CMD python -c "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=4).status == 200 else 1)" + +ENTRYPOINT ["python", "-m", "adaptive_classifier.serving"] +CMD ["/model", "--host", "0.0.0.0", "--port", "8000"] diff --git a/docs/API.md b/docs/API.md index 1ba82ac..0416958 100644 --- a/docs/API.md +++ b/docs/API.md @@ -128,7 +128,86 @@ def clear_memory(labels: Optional[List[str]] = None) Clear stored examples and prototypes. Parameters: -- `labels`: Optional list of labels to clear. If None, clears all memory +- `labels`: Optional list of labels to clear. If given, this is the same as `forget(labels)` (unknown labels are ignored). If None, clears all stored examples and prototypes + +#### calibrate + +```python +def calibrate(texts: List[str], labels: List[str]) -> Dict[str, float] +``` + +Fit confidence calibration (temperature scaling) on labelled examples the model was not trained on. Afterwards `predict`, `predict_batch` and `abstain_below` use calibrated probabilities; the winning class never changes. Returns `temperature` (above 1 means the model was over-confident), `n`, `accuracy`, and `ece_before` / `ece_after` / `nll_before` / `nll_after`. Adding a new class or forgetting one discards the calibration. + +#### calibration_report + +```python +def calibration_report(texts, labels, calibrated=True, bins=10) -> Dict[str, float] +``` + +Accuracy, mean confidence, expected calibration error (`ece`) and `nll` on labelled data. Run it on a test set separate from the calibration set. + +#### predict_set + +```python +def predict_set(text: str, alpha: float = 0.1) -> List[Tuple[str, float]] +``` + +Split-conformal prediction set: contains the true label with probability at least `1 - alpha` for inputs that look like the calibration data (an average guarantee, not per input). Never empty. Requires `calibrate` first. + +#### suggest_labels + +```python +def suggest_labels(texts, n=10, strategy="margin", diverse=False) -> List[Tuple[int, float]] +``` + +Rank unlabeled texts by how useful a label would be. Strategies: `margin` (top two classes close together), `entropy`, `least_confidence`, and `ood` (far from every known class, for discovering new classes). With `diverse=True` texts similar to an earlier pick are discounted so the picks are spread out. Returns up to `n` `(index into texts, score)` pairs, best first; uses calibrated probabilities if `calibrate` was run. + +#### drift_report + +```python +def drift_report(texts, expected_ood_rate=0.05, alpha=0.01, threshold=None) -> Dict[str, Any] +``` + +Tests whether the share of out-of-distribution texts in a window of incoming data is higher than `expected_ood_rate` (one-sided binomial test). Returns `n`, `ood_rate`, `mean_ood_score`, `p_value` and `drifted` (`p_value < alpha`). A window of a few dozen texts cannot detect small shifts; use a stricter `alpha` when checking repeatedly. + +#### forget + +```python +def forget(labels: Union[str, List[str]]) +``` + +Remove whole classes: their examples, prototype, label ids, training history and neural-head output. The remaining classes keep their learned head weights and no retraining happens. Raises `ValueError` for an unknown label. + +#### remove_examples + +```python +def remove_examples(texts: List[str], label: Optional[str] = None, retrain: bool = True) -> int +``` + +Remove training examples by exact text, for example to fix a wrong label. Prototypes are recomputed; a class left with no examples is removed. Returns the number of examples removed. + +Parameters: +- `texts`: Texts to remove +- `label`: Restrict removal to one class; searches all classes if None +- `retrain`: Fine-tune the neural head on the remaining examples. This is fine-tuning, not retraining from scratch, so the head may keep some influence of the removed text + +A classifier loaded from disk keeps only a few representative examples per class, so removing from it recomputes the prototype from those and shifts it more than removing from a live classifier (a warning is logged). + +#### ood_score + +```python +def ood_score(text: str) -> float +``` + +How far `text` is from the known classes: the distance to the nearest class prototype divided by that class's radius (the distance from its prototype to its farthest training example). Around 1 or below looks like known data; larger is further out. Returns `inf` for a classifier with no classes. Class radii are saved with the model. + +#### is_ood + +```python +def is_ood(text: str, threshold: Optional[float] = None) -> bool +``` + +True when `ood_score(text)` exceeds `threshold`, which defaults to the `ood_threshold` config value (1.25). `ood_min_radius` (default 0.05) sets a floor on class radii so single-example classes do not divide by zero. The right threshold depends on your encoder and data, so check it on held-out examples. #### merge_classifiers @@ -314,3 +393,26 @@ classifier2 = AdaptiveClassifier("bert-base-uncased") # ... train both classifiers ... classifier1.merge_classifiers(classifier2) ``` + +## SklearnAdaptiveClassifier + +```python +SklearnAdaptiveClassifier(model_name="sentence-transformers/all-MiniLM-L6-v2", + config=None, device=None, use_onnx="auto", + trust_remote_code=False) +``` + +scikit-learn estimator wrapping `AdaptiveClassifier`. `X` is a 1-D sequence of strings (a single column is also accepted); `y` holds string or integer labels, which `predict` returns unchanged. + +| Method / attribute | Description | +|---|---| +| `fit(X, y)` | Fit from scratch; returns `self` | +| `partial_fit(X, y, classes=None)` | Add examples and any new classes to what is already learned. If `classes` is given, `y` must stay within it | +| `predict(X)` | Most likely class per text | +| `predict_proba(X)` | Probabilities, columns ordered as `classes_` | +| `predict_log_proba(X)` | Log of `predict_proba` | +| `score(X, y)` | Accuracy | +| `classes_` | Sorted labels seen so far | +| `classifier_` | The underlying fitted `AdaptiveClassifier` (`save`, `push_to_hub`, `forget`, `remove_examples`, `is_ood`, ...) | + +Hyperparameters such as `prototype_weight` go inside `config`, so grid searches take `{"config": [{...}, {...}]}`. Fitted estimators cannot be pickled (they hold a FAISS index); persist `classifier_` with `save`. diff --git a/docs/benchmark.md b/docs/benchmark.md new file mode 100644 index 0000000..8c657d2 --- /dev/null +++ b/docs/benchmark.md @@ -0,0 +1,96 @@ +# Few-shot benchmark: adaptive-classifier vs SetFit vs logistic regression + +`scripts/benchmark_fewshot.py` measures how well each method does when it only has a handful +of labelled examples per class, which is the situation this library is built for. It runs as a +manually triggered GitHub Actions workflow, or on your own machine. + +## What is compared + +All methods use the **same sentence encoder** (default `sentence-transformers/all-MiniLM-L6-v2`), +the **same stratified training sample**, and the **same test examples**. + +| method | what it is | +|---|---| +| `adaptive` | this library with default settings | +| `adaptive-proto` | this library with the neural head switched off (prototypes only) | +| `logreg` | logistic regression on the frozen, L2-normalised embeddings | +| `setfit` | [SetFit](https://github.com/huggingface/setfit): contrastive fine-tuning of the encoder, then a logistic-regression head | + +`adaptive-proto` and `logreg` are included so that a difference between `adaptive` and `setfit` +can be traced: is it the encoder fine-tuning, or the classifier on top? + +## Metrics + +| metric | meaning | +|---|---| +| accuracy, macro-F1 | on a fixed random subset of the test split (`--max-test`, default 1000) | +| ECE | expected calibration error of the reported confidences: do 80% confidences come true 80% of the time? Lower is better. Reported **without** any calibration step | +| train (s) | time to train from raw text, including embedding the training examples (model loading is not counted) | +| predict test set (s), latency p50 (ms) | time for the whole test set from raw text, and the median time for a single query | +| add one example (s) | the cost of learning one more labelled example. `adaptive` updates in place. `logreg` refits on cached features. **SetFit has no incremental update, so its full training time is reported** and marked "(full retrain)" | + +Each (dataset, shots, seed) draws a stratified sample; a smaller sample for a seed is always a +subset of a larger one, so results across shot counts are comparable. Results are mean ± standard +deviation over seeds. The best value in each column is bold. + +## Running it on GitHub + +Actions → **Few-shot benchmark** → **Run workflow**. The inputs are validated before any long job +starts, and each dataset is checked to load, so a typo fails in seconds. Each dataset runs as its +own job (in parallel); the combined Markdown report appears in the run summary and as the +`benchmark-report` artifact, and the raw numbers as `results-` artifacts. + +Defaults (`ag_news,emotion`, 4 and 16 shots, 3 seeds) take a while on GitHub's CPU runners, mostly +because SetFit fine-tunes the encoder. `banking77` (77 classes) is much slower: use few shots and +one seed first. + +## Running it locally + +```bash +pip install -e ".[benchmark]" +python scripts/benchmark_fewshot.py check ag_news emotion banking77 # do the datasets load? +python scripts/benchmark_fewshot.py run --dataset ag_news --shots 4 16 --seeds 3 --out results/ag_news.json +python scripts/benchmark_fewshot.py report results/*.json --out results/summary.md +``` + +`--methods` selects a subset (for example `adaptive logreg` to skip SetFit), `--model` changes the +encoder, and `--dataset synthetic` runs a tiny generated dataset with no network access, only to +check that the harness works. + +## Datasets + +| name | source | classes | +|---|---|---| +| `ag_news` | `fancyzhx/ag_news` | 4 | +| `emotion` | `dair-ai/emotion` | 6 | +| `banking77` | `mteb/banking77` | 77 | + +`PolyAI/banking77` is not used: it is a Hub loading script, which current `datasets` releases +refuse to run. The `mteb` copy has the same data as parquet. To add a dataset, add an entry to +`DATASETS` in the script (Hub id, text column, label column, test split name) and run `check`. + +## Limits of what this shows + +Read the numbers with these in mind: + +- **One encoder.** Everything shares MiniLM-L6. A different encoder can change the ranking, and + SetFit is usually run with larger ones. +- **Untuned settings.** SetFit uses its common defaults (1 epoch, 20 pair iterations, batch 16); + logistic regression uses `C=10`; adaptive-classifier uses its defaults. None was tuned per dataset. +- **Few seeds, subsampled test set.** Differences of a point or two are within noise. With 3 seeds + the standard deviations are themselves rough. +- **Three classification datasets of short English text.** Nothing here speaks to long documents, + other languages, or very many classes beyond `banking77`. +- **CPU timings are indicative.** They depend on the machine and on whether a GPU is present + (the report records the device). +- **No LLM baseline.** Few-shot prompting of a language model is a real alternative and is not + included. +- **The first full runs have not been analysed yet.** Check the report before quoting any number + from it, and look at the run's `error` fields for failed cells (shown as `failed`). + +## Checking the harness itself + +The parts that can be tested offline (sampling, metrics, aggregation, the report, the command line, +input validation, and a smoke run of every method on a tiny local model) are covered by +`tests/test_benchmark_harness.py`, so a surprising number is more likely to be about the methods than +about the harness. diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..943b9f5 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,148 @@ +# Deployment guide: multilingual models and non-Python runtimes + +This guide covers two questions that come up often: + +1. [Can I use it for languages other than English?](#multilingual-classification) +2. [Can I train in Python and run inference in C#, JavaScript or anything else with an ONNX runtime?](#running-outside-python) + +Short answers: yes, and yes. You train and update in Python. Inference needs only the encoder in ONNX plus a few lines of arithmetic, which this guide spells out and `examples/portable_inference.py` implements. + +--- + +## Multilingual classification + +Nothing in the library is English-specific. The language support comes entirely from the encoder you pick, so swap the base model: + +```python +from adaptive_classifier import AdaptiveClassifier + +classifier = AdaptiveClassifier("intfloat/multilingual-e5-small") + +classifier.add_examples( + ["Das Produkt ist großartig", "Producto terrible, no lo recomiendo", "商品很好用"], + ["positive", "negative", "positive"], +) +classifier.predict("Muy buen servicio") +``` + +Encoders worth trying (all published on the Hugging Face Hub): + +| Model | Notes | +|---|---| +| `intfloat/multilingual-e5-small` / `-base` | Small and fast, about 100 languages. Mean pooling. | +| `sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2` | Small, trained for sentence similarity. Mean pooling. | +| `BAAI/bge-m3` | Larger and stronger, long inputs, 100+ languages. CLS pooling. | +| `xlm-roberta-base` | Multilingual, but not trained for sentence similarity; expect weaker prototypes than the models above. | + +Things that matter: + +- **Pooling.** The default `pooling: "auto"` reads the model's own sentence-transformers config, so the table above is handled for you. The result is saved with the classifier. See the "Pooling" section of the README. +- **Prefixes.** Some models expect a prefix on every input (E5 wants `"query: "`). Add it yourself to training and prediction texts, consistently. +- **Mixed-language data works.** Good multilingual encoders place translations of the same sentence close together, so a class trained mostly on English examples will often recognise other languages. Add a few examples in each language you care about to firm it up. +- **Always evaluate on your own data.** Multilingual quality varies a lot by language and domain; the library does not ship per-language benchmarks. + +--- + +## Running outside Python + +### What gets saved + +`classifier.save("./my_classifier")` (or `push_to_hub`) writes: + +| File | Contents | +|---|---| +| `config.json` | Label maps (`id_to_label`), per-class `training_history`, and the `config` block: resolved `pooling`, `max_length`, blend weights | +| `tokenizer.json` and friends | The base model's tokenizer | +| `model.safetensors` | `prototype_