0.3.0: better small-data accuracy, calibration, active learning, serving, sklearn API, and a hardened test suite

.dockerignore

@@ -0,0 +1,11 @@
+.git
+.github
+tests
+docs
+scripts
+examples
+build
+dist
+**/__pycache__
+**/*.egg-info
+*.webp

.github/workflows/benchmark.yml

@@ -0,0 +1,167 @@
+name: Few-shot benchmark
+
+# Compares adaptive-classifier with SetFit and logistic regression on public datasets.
+# Run it by hand: Actions -> Few-shot benchmark -> Run workflow. It is not part of the test
+# suite because it downloads datasets, trains SetFit on a CPU, and takes a long time.
+# Results appear in the run summary and as downloadable artifacts. See docs/benchmark.md.
+
+on:
+  workflow_dispatch:
+    inputs:
+      datasets:
+        description: "Datasets, comma separated (ag_news, emotion, banking77)"
+        default: "ag_news,emotion"
+      shots:
+        description: "Training examples per class, space separated"
+        default: "4 16"
+      seeds:
+        description: "Random seeds (different few-shot samples), 1-10"
+        default: "3"
+      methods:
+        description: "Methods to run (adaptive adaptive-proto logreg setfit)"
+        default: "adaptive adaptive-proto logreg setfit"
+      model:
+        description: "Sentence encoder shared by every method"
+        default: "sentence-transformers/all-MiniLM-L6-v2"
+      max_test:
+        description: "Test examples per dataset (a fixed random subset)"
+        default: "1000"
+
+permissions:
+  contents: read
+
+jobs:
+  plan:
+    # Validates the inputs and that each dataset loads, in seconds, before the long jobs start.
+    # Inputs reach the shell only as environment variables, never interpolated into a command.
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    outputs:
+      datasets: ${{ steps.plan.outputs.datasets }}
+      shots: ${{ steps.plan.outputs.shots }}
+      seeds: ${{ steps.plan.outputs.seeds }}
+      methods: ${{ steps.plan.outputs.methods }}
+      model: ${{ steps.plan.outputs.model }}
+      max_test: ${{ steps.plan.outputs.max_test }}
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: Validate inputs
+      id: plan
+      env:
+        IN_DATASETS: ${{ inputs.datasets }}
+        IN_SHOTS: ${{ inputs.shots }}
+        IN_SEEDS: ${{ inputs.seeds }}
+        IN_METHODS: ${{ inputs.methods }}
+        IN_MODEL: ${{ inputs.model }}
+        IN_MAX_TEST: ${{ inputs.max_test }}
+      run: |
+        pip install numpy
+        python scripts/benchmark_fewshot.py plan \
+          --datasets "$IN_DATASETS" --shots "$IN_SHOTS" --seeds "$IN_SEEDS" \
+          --methods "$IN_METHODS" --model "$IN_MODEL" --max-test "$IN_MAX_TEST" > plan.json
+        cat plan.json
+        python - <<'PY' >> "$GITHUB_OUTPUT"
+        import json
+        plan = json.load(open("plan.json"))
+        print("datasets=" + json.dumps(plan["datasets"]))
+        for key in ("shots", "seeds", "methods", "model", "max_test"):
+            print(f"{key}={plan[key]}")
+        PY
+
+    - name: Check that every dataset loads
+      env:
+        HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        DATASETS: ${{ steps.plan.outputs.datasets }}
+      run: |
+        pip install datasets
+        python scripts/benchmark_fewshot.py check $(python -c "import json,os; print(' '.join(json.loads(os.environ['DATASETS'])))")
+
+  benchmark:
+    needs: plan
+    runs-on: ubuntu-latest
+    timeout-minutes: 350
+    strategy:
+      fail-fast: false
+      matrix:
+        dataset: ${{ fromJSON(needs.plan.outputs.datasets) }}
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+        cache: pip
+
+    - name: Cache Hugging Face downloads
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/huggingface
+        key: hf-benchmark-${{ needs.plan.outputs.model }}-${{ matrix.dataset }}
+
+    - name: Install
+      run: |
+        python -m pip install --upgrade pip
+        pip install torch --index-url https://download.pytorch.org/whl/cpu
+        pip install -e ".[benchmark]"
+        pip list 2>/dev/null | grep -iE "^(torch|transformers|sentence-transformers|setfit|datasets|scikit-learn|adaptive-classifier) "
+
+    - name: Run benchmark
+      env:
+        HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        DATASET: ${{ matrix.dataset }}
+        SHOTS: ${{ needs.plan.outputs.shots }}
+        SEEDS: ${{ needs.plan.outputs.seeds }}
+        METHODS: ${{ needs.plan.outputs.methods }}
+        MODEL: ${{ needs.plan.outputs.model }}
+        MAX_TEST: ${{ needs.plan.outputs.max_test }}
+      run: |
+        # SHOTS and METHODS are space-separated lists validated by the plan job.
+        python scripts/benchmark_fewshot.py run \
+          --dataset "$DATASET" --shots $SHOTS --seeds "$SEEDS" --methods $METHODS \
+          --model "$MODEL" --max-test "$MAX_TEST" --out "results/$DATASET.json"
+
+    - name: Upload results
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: results-${{ matrix.dataset }}
+        path: results/*.json
+        if-no-files-found: warn
+
+  summary:
+    needs: [plan, benchmark]
+    if: always() && needs.plan.result == 'success'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - uses: actions/download-artifact@v4
+      with:
+        pattern: results-*
+        path: results
+        merge-multiple: true
+
+    - name: Build the report
+      run: |
+        pip install numpy
+        if ls results/*.json >/dev/null 2>&1; then
+          python scripts/benchmark_fewshot.py report results/*.json --out benchmark-report.md > /dev/null
+          cat benchmark-report.md >> "$GITHUB_STEP_SUMMARY"
+        else
+          echo "No result files were produced; see the benchmark jobs." | tee benchmark-report.md >> "$GITHUB_STEP_SUMMARY"
+        fi
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: benchmark-report
+        path: benchmark-report.md

.github/workflows/publish.yml

@@ -21,16 +21,15 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install setuptools wheel
+          pip install build
       - name: Run tests
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
         run: |
-          pip install pytest pytest-cov pytest-randomly psutil
-          pip install -e .
+          pip install -e ".[test]"
           pytest tests/
       - name: Build package
         run: |
-          python setup.py sdist bdist_wheel
+          python -m build
       - name: Publish package distributions to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/test.yml

@@ -5,28 +5,83 @@ on:
     branches: [ main ]
   pull_request:
     branches: [ main ]
+  workflow_dispatch:
+  schedule:
+    # Weekly, so breakage on the Hugging Face Hub or in new transformers/torch
+    # releases shows up without waiting for the next PR.
+    - cron: '17 5 * * 1'
+
+# A newer push to the same PR supersedes the run in progress.
+concurrency:
+  group: tests-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
+  models:
+    # Download the Hugging Face models once, with retries. The test jobs run
+    # offline from the cache, so a Hub rate limit (HTTP 429) or outage fails
+    # this one step instead of dozens of tests across the matrix.
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: Cache Hugging Face models
+      id: hf-cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/huggingface
+        key: hf-models-${{ hashFiles('scripts/prefetch_test_models.py') }}
+
+    - name: Download models
+      if: steps.hf-cache.outputs.cache-hit != 'true'
+      env:
+        HF_TOKEN: ${{ secrets.HF_TOKEN }}
+      run: |
+        pip install huggingface_hub
+        python scripts/prefetch_test_models.py
+
   unit-tests:
+    needs: models
     runs-on: ubuntu-latest
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        # The two ends of the supported range (requires-python >=3.10).
+        python-version: ['3.10', '3.13']
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
-    - name: Set up Python 3.12
-      uses: actions/setup-python@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.12'
+        python-version: ${{ matrix.python-version }}
+        cache: pip
+
+    - name: Restore Hugging Face models
+      uses: actions/cache/restore@v4
+      with:
+        path: ~/.cache/huggingface
+        key: hf-models-${{ hashFiles('scripts/prefetch_test_models.py') }}
+        fail-on-cache-miss: true
 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -e .
-        pip install pytest pytest-cov psutil
+        pip install -e ".[test]"
 
-    - name: Run unit tests
+    # Everything except tests marked `integration`, run offline against the
+    # models prefetched above. Tests marked `integration` run in the job below.
+    - name: Run tests
       env:
         HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        HF_HUB_OFFLINE: '1'
       run: |
         pytest tests/ -v --cov=adaptive_classifier --cov-report=xml --cov-report=term -m "not integration"
 
@@ -35,7 +90,7 @@ jobs:
       with:
         file: ./coverage.xml
         flags: unittests
-        name: codecov-umbrella
+        name: codecov-${{ matrix.python-version }}
         fail_ci_if_error: false
 
   integration-tests:
@@ -44,24 +99,31 @@ jobs:
     timeout-minutes: 30  # Generous timeout for model downloads
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python 3.12
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.12'
+        cache: pip
+
+    - name: Restore Hugging Face models
+      uses: actions/cache/restore@v4
+      with:
+        path: ~/.cache/huggingface
+        key: hf-models-${{ hashFiles('scripts/prefetch_test_models.py') }}
+        fail-on-cache-miss: true
 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -e .
-        pip install pytest psutil
+        pip install -e ".[test]"
 
     - name: Run integration tests
       env:
         HF_TOKEN: ${{ secrets.HF_TOKEN }}
       run: |
-        pytest tests/test_enterprise_classifiers_integration.py -v -m "integration" --tb=short
+        pytest tests/ -v -m "integration" --tb=short
 
     - name: Integration test summary
       if: always()

.gitignore

@@ -173,3 +173,6 @@ cython_debug/
 **/.DS_store
 demo_classifier/
 scripts/benchmark_results/
+
+# Training checkpoints written by the benchmark harness
+checkpoints/

CHANGELOG.md

@@ -0,0 +1,40 @@
+# Changelog
+
+## 0.3.0
+
+### Added
+- `SklearnAdaptiveClassifier`: scikit-learn estimator (`fit`, `partial_fit`, `predict`, `predict_proba`, `score`) usable in `Pipeline`, `cross_val_score` and `GridSearchCV`.
+- `forget(labels)` and `remove_examples(texts, label=None, retrain=True)` to delete classes or fix wrong labels.
+- `ood_score()` / `is_ood()` out-of-distribution detection, and `predict(..., abstain_below=, abstain_ood=)`. New config keys `ood_threshold` and `ood_min_radius`.
+- `docs/deployment.md` and `examples/portable_inference.py`: run a saved classifier without PyTorch or FAISS (multilingual models, .NET, JavaScript).
+- `save()` now writes the tokenizer files and the resolved pooling mode, so a saved directory is self-contained.
+
+- `calibrate(texts, labels)` fits one temperature on held-out labelled data so confidences match how often the model is right, and `calibration_report(...)` measures accuracy, expected calibration error and NLL. `predict`, `predict_batch` and `abstain_below` then use calibrated probabilities. Calibration is saved with the model and discarded when the set of classes changes.
+- `predict_set(text, alpha=0.1)`: split-conformal prediction sets that contain the true label with probability at least `1 - alpha`.
+
+- `suggest_labels(texts, n, strategy, diverse)`: active learning. Ranks unlabeled texts by how much a label would help (`margin`, `entropy`, `least_confidence`, or `ood` to discover unseen classes), optionally spreading the picks so near-duplicates are not all chosen.
+- `drift_report(texts)`: tests whether a window of incoming texts has moved away from the known classes (share of out-of-distribution texts against an expected rate, one-sided binomial test).
+
+- **Serving.** `python -m adaptive_classifier.serving ./model` runs a FastAPI server (`/predict`, `/predict_batch`, `/predict_set`, `/ood`, and optional authenticated `/examples`, `/forget`, `/remove_examples`); `create_app(classifier)` returns the app for embedding. Install with `pip install "adaptive-classifier[serve]"`. See `docs/serving.md`. `docker/Dockerfile` is provided but has not been built in the environment it was written in.
+- Async methods `apredict`, `apredict_batch` and `aadd_examples` that run in a worker thread.
+- A saved directory is self-contained for loading: the tokenizer saved next to the model is used instead of fetching the base model's repository, so an ONNX-saved classifier starts without Hub access.
+
+
+### Changed (default behaviour: predictions change)
+- **The neural head is now actually trained.** It used to get about ten optimiser steps and stop at the first loss plateau, so it could not learn even a 2-D XOR from 120 examples. It now trains for at least `head_steps` (default 300) steps with a cosine-decayed `head_learning_rate` (default 0.003). Small memories take roughly 1-2 s per `add_examples` call; lower `head_steps` if you add examples very frequently.
+- **Prototype scores are sharper.** They are now `softmax(-distance² / prototype_temperature)` (default 0.25) instead of `softmax(exp(-distance))`, which barely separated the nearest class from the rest (about 0.45 vs 0.21), so a confident head could always outvote it. `prototype_temperature=None` restores the old scoring.
+- **Classes with few examples lean on their prototypes.** Below `new_class_example_threshold` examples the head's weight ramps up from zero, instead of the old fixed 0.3 prototype / 0.7 head split. On separable synthetic data the old default scored 0.55-0.73 with fewer than 10 examples per class where prototypes alone scored 0.96-1.0; accuracy no longer drops off a cliff at the threshold. Trade-off: a class added with a single example is recognised less often than before, because the old split biased the head toward new classes at the expense of existing ones. Set both `new_class_prototype_weight` and `new_class_neural_weight` to restore a fixed split.
+- Classifiers saved by an earlier version keep their old scoring when loaded, so their predictions do not change.
+- **Python 3.10 or newer is required** (3.8 and 3.9 are end-of-life).
+- Packaging moved to a `[project]` table in `pyproject.toml`; `setup.py` and `requirements.txt` are gone. Install extras: `[test]`, `[scripts]`. `tqdm` is no longer a core dependency.
+- `clear_memory(labels=[...])` now removes those classes completely (it used to leave them in the label map and neural head, so they could still be predicted).
+
+### Fixed
+- **Thread safety.** Predicting while another thread added or forgot classes crashed with errors such as "selected index k out of range", and concurrent `add_examples` calls corrupted each other. Every public operation now holds a per-instance lock, so one classifier can be shared between threads. Throughput scales with processes, not threads.
+- `predict_batch(k=...)` scored only `k` classes before blending, so with more than `k` classes its scores differed from `predict` for the same text. It now scores every class and trims to `k`.
+- Accuracy on small training sets (see above): the default configuration got only 75-92% of its own training data right on perfectly separable data with fewer than 10 examples per class.
+- Integer (or other non-string) labels corrupted a classifier on save/reload, duplicating classes. `add_examples` now rejects non-string labels with a clear error; convert with `str(label)`.
+- `predict(k=-1)` returned a silently truncated list; negative or non-integer `k` now raises `ValueError`.
+- `None` as a text or label, and mixed label types, now raise a clear `ValueError` before anything is changed.
+- Loading a directory without `config.json` now says so instead of reporting an invalid Hub repo id.
+

CONTRIBUTING.md

@@ -61,8 +61,7 @@ source venv/bin/activate  # On Windows: venv\Scripts\activate
 
 3. Install dependencies:
 ```bash
-pip install -r requirements.txt
-pip install -e .
+pip install -e ".[test]"
 ```
 
 4. Run tests: