diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..0a7fde7 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,54 @@ +name: Docker image + +# Builds the serving image and queries it. It only runs when something the image +# depends on changes (and weekly, to catch breakage in upstream packages), so +# ordinary PRs do not pay for it. +on: + pull_request: + paths: + - 'docker/**' + - '.dockerignore' + - 'pyproject.toml' + - 'src/adaptive_classifier/serving.py' + - 'scripts/docker_smoke.py' + - '.github/workflows/docker.yml' + push: + branches: [ main ] + paths: + - 'docker/**' + - '.dockerignore' + - 'pyproject.toml' + - 'src/adaptive_classifier/serving.py' + - 'scripts/docker_smoke.py' + - '.github/workflows/docker.yml' + workflow_dispatch: + schedule: + - cron: '43 5 * * 1' + +concurrency: + group: docker-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + smoke: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + + - name: Build image + run: docker build -f docker/Dockerfile -t adaptive-classifier . + + # A tiny local encoder, so no Hugging Face access is needed. + - name: Install library (to create a model to serve) + run: | + python -m pip install --upgrade pip + pip install -e . + + - name: Serve and query + run: python scripts/docker_smoke.py adaptive-classifier diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 19e2c1f..b347c10 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,12 +4,20 @@ on: types: [created] jobs: + # The full test workflow (model prefetch with retries, both Python versions, the + # Node.js example, integration tests). Nothing is published unless it passes. + tests: + uses: ./.github/workflows/test.yml + secrets: inherit + pypi-publish: name: Publish release to PyPI + needs: tests runs-on: ubuntu-latest environment: name: pypi url: https://pypi.org/p/adaptive-classifier + # Only this job can publish; the test code above never sees these credentials. permissions: id-token: write steps: @@ -18,16 +26,21 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.x" - - name: Install dependencies + + - name: Check the release tag matches the package version + run: | + version=$(sed -n 's/^__version__ = "\(.*\)"/\1/p' src/adaptive_classifier/__init__.py) + tag="${{ github.event.release.tag_name }}" + echo "package version: $version, release tag: $tag" + if [ "v$version" != "$tag" ] && [ "$version" != "$tag" ]; then + echo "The release tag $tag does not match the package version $version" >&2 + exit 1 + fi + + - name: Install build tools run: | python -m pip install --upgrade pip pip install build - - name: Run tests - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: | - pip install -e ".[test]" - pytest tests/ - name: Build package run: | python -m build diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7bbdd2b..94a945e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,6 +6,8 @@ on: pull_request: branches: [ main ] workflow_dispatch: + # The release workflow runs the same tests before publishing. + workflow_call: schedule: # Weekly, so breakage on the Hugging Face Hub or in new transformers/torch # releases shows up without waiting for the next PR. @@ -76,6 +78,10 @@ jobs: python -m pip install --upgrade pip pip install -e ".[test]" + - name: Install the Node.js example's dependencies + run: npm install --no-audit --no-fund + working-directory: examples/javascript + # Everything except tests marked `integration`, run offline against the # models prefetched above. Tests marked `integration` run in the job below. - name: Run tests @@ -128,4 +134,4 @@ jobs: - name: Integration test summary if: always() run: | - echo "Integration tests completed. Check logs above for detailed results." \ No newline at end of file + echo "Integration tests completed. Check logs above for detailed results." diff --git a/.gitignore b/.gitignore index f0797e7..ede2a38 100644 --- a/.gitignore +++ b/.gitignore @@ -176,3 +176,4 @@ scripts/benchmark_results/ # Training checkpoints written by the benchmark harness checkpoints/ +examples/javascript/node_modules/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 882f966..4c09479 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,43 @@ # Changelog +## 0.3.1 + +### Fixed +- `MultiLabelAdaptiveClassifier`: `forget()` and `remove_examples()` no longer crash; `load(..., use_onnx=...)` works and the multi-label settings and per-label thresholds are saved and restored; a bare-string label (which was split into one class per character) is rejected; `max_labels=0` returns nothing and `max_labels` caps the `min_predictions` top-up; the head is trained for `head_steps` steps and retrained when labels are added, instead of ~10 steps with a softmax loss that did not apply; the public methods hold the instance lock. + +- Strategic mode: `cost_coefficients` is validated (a dict of named features, as the README once showed, or a list of the wrong length used to disable strategic mode with a cryptic message; the reason now says what is wrong and how many entries are needed); the best response explored only the first few embedding dimensions and used unseeded randomness, so it now samples the cheapest and a random spread of dimensions with a fixed seed; `predict`, `predict_strategic` and `predict_robust` scores no longer change with `k`; `evaluate_strategic_robustness` no longer divides by zero, rejects unknown labels with `ValueError`, and is repeatable and thread-safe. +- Strategic-mode predictions are different from 0.3.0 because of the candidate and `k` fixes. + +- Copying a classifier (`copy.deepcopy`, `pickle`) failed with "cannot pickle RLock" since the instance lock was added in 0.3.0; copies now get their own lock. +- `predict_batch` disagreed with `predict` in strategic mode (it skipped the strategic blend) and for `MultiLabelAdaptiveClassifier` (it softmaxed sigmoid scores). Both now return what `predict` returns, which also fixes the server's `/predict_batch` for those classifiers. + +- `merge_classifiers`: classes taken from the other classifier were not searchable afterwards (the prototype index was not rebuilt, so the merged classifier got 0% on them); merging into an empty classifier raised; training counts were not combined, so merged classes got no head weight; stale calibration was kept; example objects were shared with the source. Merging a classifier into itself is now a no-op. + +- `add_examples` now rejects blank texts and blank labels (with the index of the first offender) instead of learning them; a blank text carries no information but still counted as an example of its class and moved its prototype. **Behaviour change:** code that passes empty rows now gets a `ValueError`. `select_representative_examples(k=0)` returns an empty list instead of failing inside scikit-learn. + +- `add_examples` is now all-or-nothing. An error (or Ctrl-C) while training used to leave the label map ahead of the neural head, so every later prediction raised "selected index k out of range" and every retry failed; or left the search index stale, so predictions silently returned the wrong class. Texts are encoded before any state changes, and on failure everything is restored. +- Elastic Weight Consolidation did nothing: the penalty compared the old head with its own stored copy, so it was always zero and never reached the head being trained. It now penalises how far the live head's existing rows move. With the default strength (5.0) the effect is small: re-running the class-incremental experiment from the README (MiniLM, ag_news and emotion, 3 seeds) moved final accuracy by -0.004 and +0.016, old-class loss by -0.002 and +0.002 and new-class recall by -0.012 and 0.000, all within noise. `EWC.ewc_loss` takes the model being trained as `model=`. +- `clear_memory()` with no labels cleared only the prototypes; the label map, training counts, calibration and neural head stayed, so `predict` kept answering from the head alone. It now empties the classifier. +- A classifier loaded from disk kept only a few examples per class and re-averaged its prototype from them on the first `add_examples`, jerking the prototype away from where it was saved (0.32 versus 0.03 for a 30-example class). Prototypes now fold in new examples with the weight of the examples they were saved from. The neural head is still retrained from the examples held in memory. +- Saving a classifier that was loaded from ONNX wrote the ONNX folder as `model_name`, so the copy could not be loaded once that folder was gone. It keeps the original encoder name. +- `merge_classifiers` could deadlock when two threads merged each other; it never holds two classifiers' locks at once. It also keeps the other classifier's recorded class spread (OOD radii). +- `add_examples` encoded every text in one forward pass, ignoring `batch_size`; large lists could run out of memory. +- `remove_examples` given a bare string treated it as a set of characters and removed nothing; it now raises. +- `MultiLabelAdaptiveClassifier`: an explicit `threshold` in `predict_multilabel` was ignored for every label that had a learned threshold; a saved multi-label classifier loaded through `AdaptiveClassifier.load` (or the server) came back as single-label with softmax scores; `calibrate`, `calibration_report` and `predict_set` ran but produced meaningless numbers and now raise `NotImplementedError`; `predict` lacked the `abstain_*` options; thresholds went stale after `merge_classifiers`/`remove_examples`; strategic mode, which does not apply to sigmoid outputs, is refused. +- Server: updates, `/info` and saving no longer block the event loop (a save with ONNX export froze `/health` for seconds); a multi-label classifier is refused with a clear message instead of failing every request with 500. +- Strategic mode: `predict_strategic` and `predict_robust` accept empty text and negative `k` that `predict` rejects; `evaluate_strategic_robustness` raised `KeyError` unless the levels included 0.0 and 1.0, and accepted levels outside [0, 1]. +- scikit-learn wrapper: `partial_fit(classes=[...])` now lists every declared class in `classes_`; labels that would become the same class name (`1` and `"1"`), `None` and `NaN` are rejected; the docs wrongly said a fitted estimator cannot be pickled. +- `PrototypeMemory.get_strategic_prototypes` raised `NameError` (a missing import) whenever strategic prototypes existed. + +- `examples/basic_usage.py` crashed on a default CPU install (it called `.model.eval()` on an ONNX Runtime model). The docs no longer recommend a `--workers` flag the server does not have, say the Docker image is unverified, or omit parameters (`use_onnx`, `abstain_*`, `include_onnx`, `pooling`, ...) that exist. +- The release workflow ran its own cut-down test step with PyPI credentials in scope. It now calls the full test workflow, only the publish job holds credentials, and the release tag must match the package version. + +### Added +- Tests for strategic mode (`tests/test_strategic.py`); README and `docs/API.md` document it and `MultiLabelAdaptiveClassifier`, and the settings added in 0.3.0. +- `benchmark` and `scripts` extras pin `datasets<4` and `huggingface-hub<1.0`, which keeps `transformers` working. +- `examples/javascript/portable_inference.mjs`: a tested Node.js port of the portable inference, and a Docker smoke test (`scripts/docker_smoke.py`, `.github/workflows/docker.yml`) that builds the serving image and queries it. +- README: measured guidance on tuning `new_class_example_threshold` for new-class recall. + ## 0.3.0 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3f95e1f..62aaa15 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -76,7 +76,7 @@ We use: - [isort](https://github.com/PyCQA/isort) for import sorting - [flake8](https://github.com/PyCQA/flake8) for linting -Before submitting a PR, please run: +Before submitting a PR, please run (install the tools first: `pip install black isort flake8`): ```bash black . isort . diff --git a/README.md b/README.md index b1a5e5f..bb24bc8 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,10 @@ Adaptive Classifier is a PyTorch-based machine learning library that revolutioni - **📈 Continuous Learning** - Add new examples without catastrophic forgetting - **🔄 Dynamic Classes** - Add new classes at runtime without retraining - **⏱️ Zero Downtime** - Update models in production without service interruption +- **✅ Knows What It Doesn't Know** - Calibrated confidence, prediction sets, out-of-distribution detection and abstention +- **🧹 Fixable** - Forget a wrong example or a whole class without retraining +- **🎣 Active Learning & Drift** - Suggest which texts to label next and detect when traffic has shifted +- **🔌 Easy to Adopt** - scikit-learn interface, HTTP server, async methods, and ONNX inference outside Python ### 🛡️ **Advanced Defense** - **🎮 Strategic Classification** - Game-theoretic defense against adversarial manipulation @@ -159,6 +163,17 @@ The best split depends on how well your encoder separates your classes: an encod **Classes with few examples.** Until a class has `new_class_example_threshold` examples (default 10), the head's weight ramps up linearly from zero to `neural_weight` and the prototype takes the rest, so a brand-new class is judged almost entirely by its prototype. (Before 0.3.0 such classes used a fixed 0.3 / 0.7 split in favour of the head, and a head fitted to a handful of points could confidently outvote a correct prototype. To get a fixed split back, set both `new_class_prototype_weight` and `new_class_neural_weight`.) +**Recognising a new class sooner.** The ramp is the dial between "protect the classes you already have" and "pick up a new class quickly". Lowering `new_class_example_threshold` gives the head a say earlier. Measured with MiniLM, adding classes one at a time with 4 examples each (3 seeds, 600 test texts; averages over the added classes): + +| `new_class_example_threshold` | ag_news: final accuracy / new-class recall / old-class loss | emotion: final accuracy / new-class recall / old-class loss | +|---|---|---| +| 10 (default) | 0.76 / 0.77 / 0.06 | 0.35 / 0.43 / 0.09 | +| 5 | 0.74 / 0.85 / 0.12 | 0.35 / 0.59 / 0.14 | +| 3 | 0.72 / 0.87 / 0.15 | 0.34 / 0.59 / 0.15 | +| fixed 0.3 / 0.7 (pre-0.3.0 split) | 0.66 / 0.91 / 0.22 | 0.32 / 0.68 / 0.18 | + +Each step buys new-class recall with old-class accuracy. If a new class that is missed matters more to you than one that disturbs the others, try 5; the default favours stability. Two datasets and one encoder, so check it on your own data. + **Prototype sharpness.** Prototype scores are `softmax(-distance² / prototype_temperature)` with a default temperature of 0.25. Lower values make the nearest class stand out more. Set `prototype_temperature` to `None` for the scoring used before 0.3.0, which barely separated the nearest class from the rest. **Head training.** The head is trained for at least `head_steps` optimiser steps (default 300) with a cosine-decayed `head_learning_rate` (default 0.003). That takes roughly 1-2 seconds on a small memory; lower `head_steps` (for example to 100) if you add examples very frequently. Before 0.3.0 the head got about ten steps, too few to learn even a simple XOR pattern, which prototypes alone cannot separate. @@ -336,7 +351,7 @@ clf.partial_fit(["App crashes on login"], ["bug"]) clf.classifier_.save("./model") # the underlying AdaptiveClassifier ``` -`fit` starts over each time; `partial_fit` adds to what is already learned. A fitted estimator holds a FAISS index and can't be pickled, so save `classifier_` instead. +`fit` starts over each time; `partial_fit` adds to what is already learned. A fitted estimator can be pickled, but that copies the whole encoder, so save `classifier_` instead. ### Calibrated Confidence and Prediction Sets @@ -473,17 +488,17 @@ print(f"Label-specific thresholds: {stats['label_thresholds']}") config = { 'enable_strategic_mode': True, 'cost_function_type': 'linear', - 'cost_coefficients': { - 'sentiment_words': 0.5, # Cost to change sentiment-bearing words - 'length_change': 0.1, # Cost to modify text length - 'word_substitution': 0.3 # Cost to substitute words - }, + # One cost per embedding dimension (384 for all-MiniLM-L6-v2, 768 for BERT-base): + # how expensive it is for an adversary to push that dimension upward. + 'cost_coefficients': [0.3] * 768, 'strategic_blend_regular_weight': 0.6, # Weight for regular predictions 'strategic_blend_strategic_weight': 0.4 # Weight for strategic predictions } classifier = AdaptiveClassifier("bert-base-uncased", config=config) classifier.add_examples(texts, labels) +# If cost_coefficients has the wrong length, strategic mode is switched off and the +# reason is logged; check classifier.strategic_mode. # Robust predictions that consider potential manipulation text = "This product has amazing quality features!" diff --git a/docs/API.md b/docs/API.md index 0416958..b43fb48 100644 --- a/docs/API.md +++ b/docs/API.md @@ -11,7 +11,9 @@ AdaptiveClassifier( model_name: str, device: Optional[str] = None, config: Optional[Dict[str, Any]] = None, - seed: int = 42 + seed: int = 42, + use_onnx: Union[bool, str, None] = "auto", + trust_remote_code: bool = False ) ``` @@ -22,6 +24,8 @@ Parameters: - `device`: Device to run the model on ("cuda" or "cpu"). If None, automatically detects GPU availability - `config`: Optional configuration dictionary (see ModelConfig for details) - `seed`: Random seed for initialization (default: 42) +- `use_onnx`: Run the encoder with ONNX Runtime: `"auto"` (default) uses ONNX on CPU and PyTorch on GPU, `True`/`False` force it. With ONNX, `classifier.model` is an ONNX Runtime model, not a `torch.nn.Module` +- `trust_remote_code`: Allow models that ship custom code (default: False) ### Methods @@ -43,7 +47,8 @@ Raises: #### predict ```python -def predict(text: str, k: int = 5) -> List[Tuple[str, float]] +def predict(text: str, k: int = 5, abstain_below: Optional[float] = None, + abstain_ood: bool = False) -> List[Tuple[str, float]] ``` Predict labels for a single text input. @@ -51,6 +56,8 @@ Predict labels for a single text input. Parameters: - `text`: Input text to classify - `k`: Number of top predictions to return (default: 5) +- `abstain_below`: Return an empty list when the top confidence is below this value +- `abstain_ood`: Return an empty list when the text is out of distribution (see `is_ood`) Returns: - List of (label, confidence) tuples, sorted by confidence @@ -81,26 +88,39 @@ Returns: #### save ```python -def save(save_dir: str) +def save(save_dir: str, include_onnx: bool = True, quantize_onnx: bool = True) ``` Save the classifier state to disk. Parameters: - `save_dir`: Directory to save the model state +- `include_onnx`: Also export the encoder as ONNX (needed to load without PyTorch and for non-Python runtimes) +- `quantize_onnx`: Also write an INT8 copy (`onnx/model_quantized.onnx`) + +What a saved classifier keeps: every class prototype (averaged over all the examples the class was trained on), the neural head, label maps, training counts, class spreads and calibration, plus `num_representative_examples` examples per class (default 5). Not the full training set. + +Consequences when you load and keep learning: +- Prototypes keep their saved position. New examples are folded in with the weight of the examples the prototype was saved from, so adding one example to a class trained on 30 barely moves it. +- The neural head is retrained from the examples held in memory, which after a load are only the representative ones. If the head matters for your data, keep the original training data and re-add it, or raise `num_representative_examples` before saving. +- `remove_examples` on a loaded class recomputes its prototype from the examples that are held. #### load ```python @classmethod -def load(cls, save_dir: str, device: Optional[str] = None) -> 'AdaptiveClassifier' +def load(cls, save_dir: str, device: Optional[str] = None, use_onnx: Union[bool, str, None] = "auto", + prefer_quantized: bool = True, trust_remote_code: bool = False) -> 'AdaptiveClassifier' ``` -Load a saved classifier from disk. +Load a saved classifier from disk or from the Hugging Face Hub (`save_dir` may be a repo id). A saved `MultiLabelAdaptiveClassifier` loads as one. Parameters: -- `save_dir`: Directory containing the saved model state +- `save_dir`: Directory (or Hub repo id) containing the saved model state - `device`: Optional device to load the model onto +- `use_onnx`: `"auto"` (default), `True` or `False` +- `prefer_quantized`: Use the INT8 ONNX copy when there is one (default: True) +- `trust_remote_code`: Allow models that ship custom code (default: False) Returns: - Loaded AdaptiveClassifier instance @@ -283,10 +303,26 @@ Training Settings: - `early_stopping_patience`: Patience for early stopping (default: 3) - `min_examples_per_class`: Minimum examples required per class (default: 3) +General Settings: +- `pooling`: How token embeddings become one vector: `"auto"` (default; the model's own sentence-transformers setting, falling back to mean), `"mean"` or `"cls"`. The resolved value is saved with the classifier + Prediction Settings: - `prototype_weight`: Weight for prototype predictions (default: 0.7) - `neural_weight`: Weight for neural network predictions (default: 0.3) - `min_confidence`: Minimum confidence threshold (default: 0.1) +- `prototype_temperature`: Prototype scores are `softmax(-distance² / temperature)` (default: 0.25; `None` restores the scoring used before 0.3.0) +- `new_class_example_threshold`: Examples a class needs before the head has its full weight (default: 10) +- `new_class_prototype_weight` / `new_class_neural_weight`: Fixed split below that threshold (default: `None`, meaning the head's weight ramps up linearly; set both to use a fixed split) + +Head Training Settings: +- `head_steps`: Minimum optimiser steps each time the head is trained (default: 300) +- `head_learning_rate`: Peak learning rate of the cosine schedule (default: 0.003) + +Out-of-distribution Settings: +- `ood_threshold`: Distance-to-radius ratio above which `is_ood` is true (default: 1.25) +- `ood_min_radius`: Smallest class radius used for that ratio (default: 0.05) + +Strategic Mode Settings (see Strategic mode below): `enable_strategic_mode`, `cost_function_type`, `cost_coefficients`, `strategic_lambda`, `strategic_training_frequency`, `strategic_blend_regular_weight`, `strategic_blend_strategic_weight`, `strategic_robust_proto_weight`, `strategic_robust_head_weight`, `strategic_prediction_proto_weight`, `strategic_prediction_head_weight`. Device Settings: - `device_map`: Device mapping strategy (default: 'auto') @@ -415,4 +451,51 @@ scikit-learn estimator wrapping `AdaptiveClassifier`. `X` is a 1-D sequence of s | `classes_` | Sorted labels seen so far | | `classifier_` | The underlying fitted `AdaptiveClassifier` (`save`, `push_to_hub`, `forget`, `remove_examples`, `is_ood`, ...) | -Hyperparameters such as `prototype_weight` go inside `config`, so grid searches take `{"config": [{...}, {...}]}`. Fitted estimators cannot be pickled (they hold a FAISS index); persist `classifier_` with `save`. +Hyperparameters such as `prototype_weight` go inside `config`, so grid searches take `{"config": [{...}, {...}]}`. Fitted estimators can be pickled, but that copies the whole encoder; persist `classifier_` with `save` instead. `partial_fit(X, y, classes=[...])` lists every declared class in `classes_` even before an example of it has been seen, and labels that would become the same class name (`1` and `"1"`), `None` and `NaN` are rejected. + +## MultiLabelAdaptiveClassifier + +A text can have several labels. Subclass of `AdaptiveClassifier`; the head uses sigmoid outputs and is trained with binary cross-entropy. + +```python +from adaptive_classifier import MultiLabelAdaptiveClassifier + +clf = MultiLabelAdaptiveClassifier( + "sentence-transformers/all-MiniLM-L6-v2", + default_threshold=0.5, # between 0 and 1; lowered automatically as the number of labels grows + min_predictions=1, # return at least this many labels, even below the threshold + max_predictions=None, # hard cap on labels returned (None = no cap) +) +clf.add_examples(["new GPU benchmarks", "vaccine trial results"], + [["technology", "hardware"], ["health", "science"]]) # one list of labels per text +clf.predict_multilabel("a study of AI in medicine", threshold=0.3, max_labels=5) +``` + +- `add_examples(texts, labels)`: `labels` holds one list of strings per text. A bare string instead of a list raises `ValueError`; a label repeated within a text counts once; a text with an empty list is skipped. +- `predict_multilabel(text, threshold=None, max_labels=None)`: labels whose score reaches their threshold, best first. `max_labels` is a hard cap (0 returns nothing) and also bounds the `min_predictions` top-up. Per-label thresholds are adjusted for how common each label is. +- `predict(text, k=5)`: `predict_multilabel` limited to `k` labels. +- `forget`, `remove_examples`, `save` and `load` work as on `AdaptiveClassifier`; the settings above and the per-label thresholds are saved and restored. +- Adding a new label retrains the head on all stored examples, so a classifier loaded from disk (which keeps only representative examples per class) retrains on those. + +## Strategic mode + +Optional defence against inputs that are edited to game the classifier. Enable it with a config where `cost_coefficients` has one entry per embedding dimension: + +```python +config = { + "enable_strategic_mode": True, + "cost_function_type": "linear", # or "separable" + "cost_coefficients": [0.3] * 768, # embedding size of the encoder + "strategic_blend_regular_weight": 0.6, + "strategic_blend_strategic_weight": 0.4, +} +clf = AdaptiveClassifier("bert-base-uncased", config=config) +clf.strategic_mode # False if the coefficients were unusable; the reason is logged +``` + +- `predict(text)`: blends the regular and strategic predictions with the two blend weights. +- `predict_strategic(text)`: predicts on the input as an adversary would most profitably move it. +- `predict_robust(text)`: assumes the input may already have been moved; leans on the prototypes (`strategic_robust_proto_weight` 0.8, `strategic_robust_head_weight` 0.2). +- `evaluate_strategic_robustness(texts, labels, gaming_levels=[0.0, 0.5, 1.0])`: accuracy when that share of inputs is gamed. `relative_robustness` is `nan` when accuracy without gaming is zero. + +With strategic mode off, `predict_strategic` and `predict_robust` are the same as `predict`. Strategic scoring is not part of the portable reference in `docs/deployment.md`. diff --git a/docs/deployment.md b/docs/deployment.md index 943b9f5..e0f3343 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -116,7 +116,7 @@ The scoring is a few dozen lines of array maths. Read `model.safetensors` with a ### JavaScript / Node / browser -Use `onnxruntime-node` (server) or `onnxruntime-web` (browser) for the encoder and `@huggingface/transformers` for tokenization. +Use `onnxruntime-node` (server) or `onnxruntime-web` (browser) for the encoder and `@huggingface/tokenizers` (a small, dependency-free package) for tokenization. ```js import * as ort from "onnxruntime-node"; @@ -132,7 +132,14 @@ const hidden = out.last_hidden_state; // Float32Array data, dims [1, tokens, d For the browser, check the size of `model_quantized.onnx` first: multilingual encoders carry a large vocabulary embedding and can be well over 100 MB even when quantized. -The C# and JavaScript snippets above are sketches of the API calls and have **not** been run; the Python reference script is the tested artifact. +A complete, tested Node.js port is in `examples/javascript/portable_inference.mjs` (tokenizer, safetensors reader, pooling, scoring): + +```bash +cd examples/javascript && npm install +node portable_inference.mjs ./my_classifier "text to classify" +``` + +`tests/test_js_example.py` checks it against `AdaptiveClassifier.predict` for both pooling modes and every scoring branch, including inputs longer than `max_length`. It uses `onnxruntime-node` when installed and falls back to `onnxruntime-web` (WASM). The C# snippet above is a sketch of the API calls and has **not** been run; the Python and JavaScript ports are the tested artifacts. ### Updating a deployed model diff --git a/docs/serving.md b/docs/serving.md index 5c39328..4f1383b 100644 --- a/docs/serving.md +++ b/docs/serving.md @@ -50,7 +50,7 @@ Without an API key anyone who can reach the server can change the model, and the Every classifier operation takes a lock, so one process can safely serve concurrent requests while learning. That means requests are processed one at a time inside a process. For more throughput run -several **processes** (`--workers`, or several containers behind a load balancer); with a read-only +several **processes** (for example several containers behind a load balancer); with a read-only server this scales cleanly. Each process has its own copy of the model, so updates only reach the worker that received them: run a single process if you enable updates. @@ -65,7 +65,4 @@ docker build -f docker/Dockerfile -t adaptive-classifier . docker run --rm -p 8000:8000 -v "$PWD/my_classifier:/model:ro" adaptive-classifier ``` -The image runs as a non-root user, has a health check on `/health`, and uses CPU-only PyTorch. The -Dockerfile has not been built or run in the environment this was written in (no Docker daemon was -available); the application it runs is covered by the server tests, but treat the image build as -unverified until you have run it once. +The image runs as a non-root user, has a health check on `/health`, and uses CPU-only PyTorch. CI builds it and queries it (`scripts/docker_smoke.py`, workflow `docker.yml`) whenever the Dockerfile, `pyproject.toml` or the server change, and weekly. diff --git a/examples/basic_usage.py b/examples/basic_usage.py index ccc4879..e71fef0 100644 --- a/examples/basic_usage.py +++ b/examples/basic_usage.py @@ -64,8 +64,6 @@ def main(): ] print("\nTesting predictions:") - classifier.model.eval() - with torch.no_grad(): for text in test_texts: predictions = classifier.predict(text) @@ -101,8 +99,6 @@ def main(): print("\nTesting technical classification:") technical_test = "API giving null pointer exception" - loaded_classifier.model.eval() - with torch.no_grad(): predictions = loaded_classifier.predict(technical_test) print(f"\nText: {technical_test}") diff --git a/examples/javascript/package.json b/examples/javascript/package.json new file mode 100644 index 0000000..ec373e6 --- /dev/null +++ b/examples/javascript/package.json @@ -0,0 +1,13 @@ +{ + "name": "adaptive-classifier-portable-inference", + "private": true, + "type": "module", + "description": "Run a saved AdaptiveClassifier from Node.js with no Python.", + "dependencies": { + "@huggingface/tokenizers": "^0.2.0" + }, + "optionalDependencies": { + "onnxruntime-node": "^1.22.0", + "onnxruntime-web": "^1.22.0" + } +} diff --git a/examples/javascript/portable_inference.mjs b/examples/javascript/portable_inference.mjs new file mode 100644 index 0000000..1b5da95 --- /dev/null +++ b/examples/javascript/portable_inference.mjs @@ -0,0 +1,207 @@ +// Run a saved AdaptiveClassifier from Node.js, with no Python, PyTorch or FAISS. +// +// npm install +// node portable_inference.mjs ./my_classifier "text to classify" +// +// This is a port of examples/portable_inference.py; docs/deployment.md explains each +// step. It reads config.json, model.safetensors, tokenizer.json, tokenizer_config.json +// and onnx/ from a directory written by AdaptiveClassifier.save(). + +import { readFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { Tokenizer } from "@huggingface/tokenizers"; + +// onnxruntime-node is the fast server runtime; onnxruntime-web (WASM) also runs in +// Node and is the fallback where the native binaries are unavailable. +async function loadRuntime() { + for (const name of ["onnxruntime-node", "onnxruntime-web"]) { + try { + return await import(name); + } catch { + // try the next one + } + } + throw new Error("Install onnxruntime-node or onnxruntime-web"); +} + +// safetensors: 8-byte little-endian header length, a JSON header, then raw data. +export function readSafetensors(path) { + const buffer = readFileSync(path); + const headerLength = Number(buffer.readBigUInt64LE(0)); + const header = JSON.parse(buffer.subarray(8, 8 + headerLength).toString("utf8")); + const base = 8 + headerLength; + const tensors = {}; + for (const [name, info] of Object.entries(header)) { + if (name === "__metadata__") continue; + if (info.dtype !== "F32") throw new Error(`${name}: unsupported dtype ${info.dtype}`); + const [start, end] = info.data_offsets; + // Copy: the file buffer is not guaranteed to be 4-byte aligned. + const bytes = buffer.subarray(base + start, base + end); + const data = new Float32Array(bytes.byteLength / 4); + new Uint8Array(data.buffer).set(bytes); + tensors[name] = { shape: info.shape, data }; + } + return tensors; +} + +function softmax(values) { + const max = Math.max(...values); + const exps = values.map((v) => Math.exp(v - max)); + const total = exps.reduce((a, b) => a + b, 0); + return exps.map((e) => e / total); +} + +function squaredDistance(a, b) { + let sum = 0; + for (let i = 0; i < a.length; i++) sum += (a[i] - b[i]) ** 2; + return sum; +} + +// out = W x + b with W stored (out, in). +function affine({ shape, data }, bias, x) { + const [rows, cols] = shape; + const out = new Float32Array(rows); + for (let r = 0; r < rows; r++) { + let sum = bias.data[r]; + for (let c = 0; c < cols; c++) sum += data[r * cols + c] * x[c]; + out[r] = sum; + } + return out; +} + +export class PortableClassifier { + static async load(directory, { pooling, preferQuantized = true } = {}) { + const self = new PortableClassifier(); + const cfg = JSON.parse(readFileSync(join(directory, "config.json"), "utf8")); + const settings = cfg.config ?? {}; + self.cfg = cfg; + self.maxLength = settings.max_length ?? 512; + self.prototypeWeight = settings.prototype_weight ?? 0.7; + self.neuralWeight = settings.neural_weight ?? 0.3; + self.newThreshold = settings.new_class_example_threshold ?? 10; + self.newPrototypeWeight = settings.new_class_prototype_weight ?? null; + self.newNeuralWeight = settings.new_class_neural_weight ?? null; + self.temperature = settings.prototype_temperature ?? null; + self.pooling = pooling ?? settings.pooling; + if (self.pooling !== "mean" && self.pooling !== "cls") { + throw new Error("config.json does not fix a pooling mode; pass { pooling: 'mean' | 'cls' }"); + } + + const count = Object.keys(cfg.id_to_label).length; + self.labels = Array.from({ length: count }, (_, i) => cfg.id_to_label[String(i)]); + const tensors = readSafetensors(join(directory, "model.safetensors")); + self.prototypes = new Map(); + for (const label of self.labels) { + const tensor = tensors[`prototype_${label}`]; + if (tensor) self.prototypes.set(label, tensor.data); + } + // Linear-ReLU-Dropout x2 then Linear: affine layers at indices 0, 3 and 6. + self.head = tensors["adaptive_head_model.0.weight"] + ? [0, 3, 6].map((i) => [tensors[`adaptive_head_model.${i}.weight`], tensors[`adaptive_head_model.${i}.bias`]]) + : null; + + self.tokenizer = new Tokenizer( + JSON.parse(readFileSync(join(directory, "tokenizer.json"), "utf8")), + JSON.parse(readFileSync(join(directory, "tokenizer_config.json"), "utf8")), + ); + + self.ort = await loadRuntime(); + const quantized = join(directory, "onnx", "model_quantized.onnx"); + const full = join(directory, "onnx", "model.onnx"); + const file = preferQuantized && existsSync(quantized) ? quantized : full; + self.session = await self.ort.InferenceSession.create(file); + return self; + } + + async embed(text) { + const encoded = this.tokenizer.encode(text); + let ids = encoded.ids; + let mask = encoded.attention_mask; + if (ids.length > this.maxLength) { + // Keep the final special token (e.g. [SEP]) as the Python tokenizer does. + ids = [...ids.slice(0, this.maxLength - 1), ids[ids.length - 1]]; + mask = mask.slice(0, this.maxLength); + } + const n = ids.length; + const tensor = (values) => new this.ort.Tensor("int64", BigInt64Array.from(values.map(BigInt)), [1, n]); + const feeds = { input_ids: tensor(ids), attention_mask: tensor(mask) }; + if (this.session.inputNames.includes("token_type_ids")) { + feeds.token_type_ids = tensor(new Array(n).fill(0)); + } + const output = await this.session.run(feeds); + const hidden = output[this.session.outputNames[0]]; // (1, tokens, dim) + const dim = hidden.dims[2]; + + const vec = new Float64Array(dim); + if (this.pooling === "cls") { + for (let d = 0; d < dim; d++) vec[d] = hidden.data[d]; + } else { + let real = 0; + for (let t = 0; t < n; t++) { + if (!mask[t]) continue; + real++; + for (let d = 0; d < dim; d++) vec[d] += hidden.data[t * dim + d]; + } + for (let d = 0; d < dim; d++) vec[d] /= Math.max(real, 1e-9); + } + const norm = Math.max(Math.sqrt(vec.reduce((s, v) => s + v * v, 0)), 1e-12); + return Float32Array.from(vec, (v) => v / norm); + } + + headProbabilities(embedding) { + let x = embedding; + this.head.forEach(([weight, bias], i) => { + x = affine(weight, bias, x); + if (i < this.head.length - 1) x = x.map((v) => Math.max(v, 0)); + }); + return softmax(Array.from(x)); + } + + weights(label) { + const trained = this.cfg.training_history?.[label] ?? 0; + if (trained >= this.newThreshold) return [this.prototypeWeight, this.neuralWeight]; + if (this.newPrototypeWeight !== null && this.newNeuralWeight !== null) { + return [this.newPrototypeWeight, this.newNeuralWeight]; + } + const share = trained / this.newThreshold; + return [1 - share + share * this.prototypeWeight, share * this.neuralWeight]; + } + + async predict(text, k = 5) { + const embedding = await this.embed(text); + + const names = [...this.prototypes.keys()]; + const distances = names.map((name) => squaredDistance(this.prototypes.get(name), embedding)); + const prototypeScores = new Map( + names.map((name, i) => [ + name, + (this.temperature + ? softmax(distances.map((d) => -d / this.temperature)) + : softmax(distances.map((d) => Math.exp(-d))))[i], + ]), + ); + const headScores = this.head ? this.headProbabilities(embedding) : null; + + const combined = this.labels.map((label, i) => { + const [wp, wn] = this.weights(label); + return [label, (prototypeScores.get(label) ?? 0) * wp + (headScores ? headScores[i] : 0) * wn]; + }); + const total = combined.reduce((s, [, score]) => s + score, 0); + return combined + .sort((a, b) => b[1] - a[1]) + .slice(0, k) + .map(([label, score]) => [label, score / total]); + } +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + if (process.argv.length < 4) { + console.error("usage: node portable_inference.mjs "); + process.exit(2); + } + const classifier = await PortableClassifier.load(process.argv[2]); + for (const [label, score] of await classifier.predict(process.argv.slice(3).join(" "))) { + console.log(`${label}\t${score.toFixed(4)}`); + } +} diff --git a/pyproject.toml b/pyproject.toml index 90e91bd..26cc973 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,15 +53,19 @@ serve = [ "uvicorn>=0.23.0", ] # Benchmark harness: scripts/benchmark_fewshot.py (see docs/benchmark.md). +# transformers 4.x needs huggingface-hub<1.0, but datasets 4+ pulls in a newer hub and +# breaks the install, so these extras keep both on compatible releases. benchmark = [ "setfit>=1.1.0", - "datasets>=2.14.0", + "datasets>=2.14.0,<4", + "huggingface-hub>=0.34,<1.0", ] # Only needed to run the scripts/ benchmarks and data generators. scripts = [ "tqdm>=4.65.0", "textdistance", - "datasets", + "datasets<4", + "huggingface-hub>=0.34,<1.0", "sentence-transformers", "openai", ] diff --git a/scripts/adaptive_router/config.json b/scripts/adaptive_router/config.json deleted file mode 100644 index 405d3f3..0000000 --- a/scripts/adaptive_router/config.json +++ /dev/null @@ -1 +0,0 @@ -{"model_name": "distilbert/distilbert-base-cased", "embedding_dim": 768, "label_to_id": {"HIGH": 0, "LOW": 1}, "id_to_label": {"0": "HIGH", "1": "LOW"}, "train_steps": 20, "config": {"max_length": 512, "batch_size": 32, "learning_rate": 0.0005, "warmup_steps": 0, "max_examples_per_class": 500, "prototype_update_frequency": 50, "similarity_threshold": 0.7, "ewc_lambda": 100.0, "num_representative_examples": 5, "epochs": 10, "early_stopping_patience": 3, "min_examples_per_class": 3, "prototype_weight": 0.8, "neural_weight": 0.2, "min_confidence": 0.1, "device_map": "auto", "quantization": null, "gradient_checkpointing": false}, "examples": {"LOW": [{"text": "Is drinking coffee every day bad for my teeth? If so, what can I do to prevent problems?", "label": "LOW", "embedding": [0.02039445750415325, 0.009329110383987427, -0.0063858344219625, -0.0022812490351498127, -0.02323310077190399, -0.013027237728238106, 0.01961006224155426, -0.013638310134410858, 0.005428168922662735, -0.07121948897838593, -0.014629247598350048, -0.0024634592700749636, -0.002002640860155225, -0.008783119730651379, -0.028760455548763275, 6.424890307243913e-05, 0.012376553378999233, 0.0038553052581846714, -0.002131714718416333, -0.016042346134781837, 0.004439534619450569, -0.005865334998816252, 0.031842831522226334, -0.014386935159564018, 0.015017320401966572, 0.0002791396400425583, 0.013109290972352028, 0.010686184279620647, -0.012168671004474163, 0.019312020391225815, 0.0009414437226951122, 0.008297912776470184, -0.000900810060556978, 0.00730891153216362, -0.007535258773714304, 0.00919526070356369, -0.006669522728770971, -0.019071796908974648, -0.010579504072666168, -0.016308315098285675, -0.04010802507400513, 0.00792109314352274, 0.04441574215888977, -0.010729307308793068, 0.002213916275650263, -0.025585148483514786, 0.007240836974233389, -0.004635443910956383, -0.005535494070500135, 0.006190856918692589, 0.0025153581518679857, 0.00625216169282794, -0.00983637198805809, -0.0028825534973293543, 0.01950671896338463, 0.009769483469426632, -0.005437754560261965, 0.006399035453796387, -0.03175976872444153, 0.021063653752207756, 0.0013231730554252863, -0.0019202150870114565, 0.017947625368833542, -0.00458945520222187, -0.0017249748343601823, 0.008747135289013386, 0.0008959486149251461, 0.00858580507338047, -0.013003678992390633, -0.014394905418157578, 0.004457827191799879, 0.01405587699264288, 0.004601889289915562, 0.06408803910017014, 0.02073892392218113, -0.012503165751695633, 0.028974877670407295, -0.0019089512061327696, -0.006704524625092745, 0.004076198674738407, 0.00115435931365937, 0.005870849825441837, -0.004858851432800293, -0.014478415250778198, -0.00970640778541565, -0.009062430821359158, 0.013542640022933483, -0.0020952257327735424, -0.004976946860551834, 0.013095272704958916, 0.01613515056669712, -0.007884802296757698, -0.018110938370227814, 0.0033188823144882917, 0.0013254889054223895, 0.011479954235255718, 0.005350329913198948, -0.004205691162496805, 0.42081305384635925, -0.00968922395259142, -0.008573644794523716, 0.0009014643728733063, 0.00906637404114008, -0.004837092477828264, 0.02269764430820942, -0.03025415912270546, -0.012866695411503315, -0.028694601729512215, 0.003883116412907839, 0.031925003975629807, 0.030822059139609337, -0.002144597005099058, 0.0022426871582865715, -4.5188437070464715e-05, -0.006833445746451616, -0.030236119404435158, 0.01121692918241024, 0.001971140503883362, 0.017980923876166344, -0.011958480812609196, 0.004468109924346209, -0.017697839066386223, 0.07248935103416443, 0.00786749366670847, -0.009477674961090088, -0.01135212741792202, -0.003371140221133828, -0.012895338237285614, 0.001460296451114118, -0.02349063754081726, -0.03485414758324623, -0.009475907310843468, -0.00048241938930004835, -0.0015780527610331774, -0.0010878484463319182, 0.0011548349866643548, -0.008919508196413517, -0.0025622970424592495, -0.05595393851399422, 0.01394527219235897, 0.012133805081248283, -0.006726194638758898, 0.005455933045595884, -0.01264666672796011, -0.022755097597837448, 0.18685847520828247, -0.01691720262169838, 0.0023538663517683744, -0.004938594531267881, 0.0018003035802394152, 0.0008885525166988373, -0.020330961793661118, -0.012031910009682178, 0.012911496683955193, -0.011570140719413757, -0.011590506881475449, 0.01575644500553608, 0.004554626997560263, 0.00662964629009366, 0.0004100624064449221, -0.06498976796865463, 0.010730903595685959, -0.008107719011604786, 0.030820028856396675, 0.009474949911236763, -0.01906215026974678, 0.009458707645535469, -0.043944187462329865, 0.005206570029258728, 0.01424614991992712, 0.001170583302155137, 0.0005184027832001448, -0.14996354281902313, 0.021765295416116714, 0.009286255575716496, 0.0072013926692306995, -0.004659716039896011, 0.011929381638765335, -0.003190129529684782, -0.015722446143627167, -0.01815064623951912, 0.01834193430840969, 0.006084519904106855, 0.016180403530597687, -0.032099515199661255, 0.011430078186094761, 0.003294322406873107, -0.008923242799937725, 0.0003509738598950207, 0.009325523860752583, -0.0019701814744621515, -0.028425632044672966, 0.003964909352362156, 0.015864670276641846, 0.010673252865672112, 0.011225402355194092, 0.005043562036007643, -0.009630661457777023, -0.004088860470801592, 0.010778944939374924, -0.006817656569182873, 0.00015835711383260787, -0.018961118534207344, 0.016901995986700058, 0.021244479343295097, -0.003804424311965704, -0.005433924496173859, 0.006842302158474922, 0.016494009643793106, -0.0024014578666538, -0.010637672618031502, -7.727232878096402e-05, 0.0053171431645751, 0.03094189427793026, 0.0353022962808609, -0.021167855709791183, -0.0008689461974427104, 0.012743168510496616, 0.008250226266682148, 0.007845299318432808, -0.013233593665063381, 0.008673184551298618, -0.02156921476125717, -0.009463030844926834, -0.008926155045628548, -0.006732558365911245, 0.012439075857400894, 0.0009144321666099131, 0.005373652093112469, -0.009167078882455826, 0.008026966825127602, 0.0063455733470618725, 0.00894924532622099, -0.015243635512888432, -0.002839545253664255, 0.0048542930744588375, -0.006756486836820841, 0.004971047397702932, -0.001208290341310203, 0.01429009810090065, 0.005963467061519623, -0.007884562946856022, 0.02749212644994259, -0.0013566984562203288, 0.005307959858328104, 0.0012359292013570666, 0.01852298155426979, 0.011950000189244747, 0.0072205038741230965, 0.005552733317017555, 0.043303485959768295, 0.0081065334379673, -0.021918853744864464, -0.016146095469594002, -0.029025882482528687, -0.010243748314678669, -0.002525107702240348, -0.0796574279665947, -0.017740987241268158, -0.010356458835303783, 0.010222683660686016, -0.19163382053375244, 0.003947861026972532, 0.013203560374677181, 0.008440962061285973, -0.008878355845808983, 0.025226140394806862, 0.0013747744960710406, -0.01895708218216896, 0.011175540275871754, -0.02654990740120411, -0.003266084473580122, -0.041313450783491135, 0.009877652861177921, -0.0038216696120798588, 0.007183769252151251, -0.01046864315867424, -0.006100723519921303, -0.011098721995949745, -0.021343091502785683, -0.02041606418788433, 0.0031096464954316616, -0.019416995346546173, 0.019631320610642433, -0.013745995238423347, 0.024470513686537743, 0.01762683317065239, 0.009671872481703758, 0.0028081329073756933, 0.2604740262031555, 0.026210034266114235, -0.013867592439055443, 0.008933997713029385, -0.016630325466394424, -0.005856370087713003, -0.010458548553287983, -0.029652513563632965, 0.006278388202190399, 0.004829395096749067, 0.009914031252264977, -0.014958598650991917, 0.0012652042787522078, -0.039437148720026016, -0.007311483845114708, -0.021462853997945786, 0.008077370002865791, 0.003967227879911661, 0.005063635762780905, -0.03678053617477417, -0.0049983724020421505, -0.0028740540146827698, -0.0013815644197165966, -0.0020935693755745888, 0.004799794405698776, -0.009738544002175331, -0.023781854659318924, -0.015738803893327713, 0.032626233994960785, -0.007238231133669615, -0.0537262000143528, 0.012643839232623577, -0.0016240023542195559, -0.009904581122100353, -0.007598008960485458, 0.003405306488275528, -0.0008665909990668297, -0.004028125666081905, -0.025651630014181137, -0.022545790299773216, -0.0007770610391162336, -0.007444023620337248, 0.003744380082935095, -0.02425505220890045, -0.011070209555327892, -0.006295229308307171, -0.01313671562820673, -0.020499657839536667, 0.01205499842762947, 0.005852322559803724, 0.007805896457284689, -0.01712975837290287, -0.01067646499723196, -0.018875913694500923, 0.014298168942332268, 0.019405683502554893, 0.0063184695318341255, 0.017186330631375313, 0.0016592043684795499, 0.01577926054596901, -0.001817985437810421, 0.006715130060911179, -0.00014420766092371196, -0.00024471114738844335, 0.016705675050616264, -0.005719838198274374, -0.0023667258210480213, -0.01015920378267765, -0.019472772255539894, 0.0006595418090000749, 0.00735819898545742, 0.003003645921126008, -0.13396503031253815, 0.011126355268061161, 0.009786608628928661, -0.005150869023054838, -0.0010131056187674403, -0.01121619250625372, 0.0070387390442192554, -0.006282364018261433, 0.029067253693938255, 0.01522061601281166, 0.00017981980636250228, 0.016120480373501778, 0.006321825552731752, -0.02651059441268444, 0.0006518823211081326, 0.002931169467046857, 0.04065092280507088, 0.005657408386468887, -0.022772766649723053, -0.01689116097986698, 0.005222965497523546, 0.015482182614505291, -0.006359671242535114, 0.005627564154565334, 0.006165416445583105, 0.0031208707951009274, -0.018388239666819572, -0.008571879006922245, -0.00942636001855135, 0.018839146941900253, -0.012159918434917927, -0.0062193116173148155, -0.004445856902748346, -0.006011628080159426, 0.0021532855462282896, 0.0010279976995661855, 0.007652680389583111, 0.011531555093824863, -0.014759029261767864, -0.006943935062736273, 0.005930377170443535, -0.002639238955453038, 0.013245626352727413, -0.019623612985014915, -0.008578967303037643, 0.001572834444232285, -0.010334818623960018, -0.10554677248001099, -0.00977408979088068, -0.006572606973350048, -0.01331594679504633, -0.012126442976295948, 0.019541308283805847, 0.003057927591726184, 0.006769230589270592, -0.001669080462306738, -0.004076978657394648, 0.020721081644296646, 0.017214080318808556, 0.020775336772203445, -0.009672624990344048, 0.001611808082088828, -0.020856143906712532, -0.012448630295693874, 0.01653185859322548, 0.006088856607675552, -0.01404204685240984, 0.0021502559538930655, 0.013774893246591091, -0.04752389341592789, -0.010632338002324104, -0.0003475138801150024, -0.03574530407786369, -0.01821925677359104, -0.017107384279370308, 0.0007779861916787922, -0.013850036077201366, -0.004089724272489548, 0.3403048813343048, -0.014394883997738361, 0.009628048166632652, -0.0025859910529106855, -0.01923978328704834, -0.011382333002984524, 0.015602130442857742, -0.003190044779330492, 0.04598904401063919, 0.0028009344823658466, -0.009925877675414085, -0.02182849869132042, -0.004793766885995865, 0.008026781491935253, -0.03747987002134323, 0.021869324147701263, 0.007670832797884941, -0.0064276233315467834, -0.001845433609560132, -0.011233883909881115, 0.0031099149491637945, 0.004901505075395107, 0.007599821779876947, -0.012925037182867527, 0.016049401834607124, 0.0050430018454790115, 0.00239281146787107, -0.008275839500129223, -0.003632206004112959, -0.008740216493606567, -0.01720230095088482, 0.012146754190325737, 0.005852756556123495, 0.015462875366210938, -0.004518353845924139, -0.00517266197130084, 0.012092678807675838, 0.006908233277499676, 0.008768774569034576, 0.004185569006949663, 0.01363169215619564, 0.00423558009788394, 0.06088092923164368, -0.001793655683286488, 0.014455302618443966, 0.010724241845309734, -0.0052613383159041405, -0.009440657682716846, 0.029123345389962196, 0.0062365783378481865, -0.014326999895274639, 0.011866627261042595, 0.015416090376675129, -0.008981081657111645, -0.0025332290679216385, 0.0051352581940591335, 0.00821018684655428, -0.011693744920194149, -0.011451606638729572, -0.005426885560154915, -0.013806660659611225, -0.006607064511626959, -0.009153145365417004, -0.002933175303041935, 0.01713304966688156, -0.015997933223843575, 0.0014536705566570163, -0.007618360221385956, -0.014526145532727242, -0.008658366277813911, -0.018422337248921394, 0.0011256936704739928, -0.02748172916471958, 0.012479417957365513, 0.021452156826853752, 0.001562531921081245, -0.025555524975061417, 0.0065613193437457085, -0.00209504971280694, -0.013913575559854507, -0.012075789272785187, -0.01455396693199873, -0.006788528990000486, -0.010492048226296902, -0.003972693346440792, 0.009717912413179874, -0.025845356285572052, -0.026431582868099213, -0.0048737614415585995, 0.017324643209576607, -0.0013998278882354498, -0.018030479550361633, -0.008311547338962555, -0.007493891753256321, -0.0035244368482381105, 0.00771515304222703, 0.0021924739703536034, -0.005751968361437321, -0.006604904308915138, 0.005347672384232283, 0.019293809309601784, -0.0003495284472592175, 0.010960870422422886, 0.004829931538552046, -0.003964467439800501, -0.008894444443285465, -0.0016397202853113413, 0.011930232867598534, -0.010505779646337032, -0.003346486249938607, 0.015857968479394913, 0.014665059745311737, 0.014732534997165203, -0.004206720273941755, -0.0019176763016730547, 0.01332307793200016, 0.0021988777443766594, -0.02403758279979229, -0.4469744861125946, 0.01972227357327938, 0.004987025633454323, 0.014963134191930294, -0.008944636210799217, -0.005878764670342207, 0.02129177562892437, 0.013982802629470825, 0.018398547545075417, 0.011725765652954578, -0.02574373222887516, -0.021991107612848282, -0.12666742503643036, 0.011377723887562752, -0.013467676937580109, 0.016786223277449608, -0.01608293317258358, -0.04973398521542549, 0.00016399554442614317, 0.03954412788152695, -0.013195888139307499, 0.012831269763410091, 0.013146511279046535, 0.009329939261078835, 0.014414996840059757, 0.0021092528477311134, 0.002863264875486493, 0.005193709395825863, 0.008082847110927105, -0.009050048887729645, -0.0022750503849238157, -0.001845191465690732, 0.030253732576966286, -0.009286943823099136, -0.005152231547981501, -0.010391759686172009, 0.004703886806964874, -0.004108126275241375, 0.007455661427229643, -0.019625799730420113, -0.0037084040232002735, 0.002227721968665719, 0.019071374088525772, 0.014121511951088905, -0.03018501214683056, -0.01767222210764885, -0.03424135223031044, -0.15107634663581848, 0.00812471192330122, -0.009383570402860641, 0.014023326337337494, 0.0047807348892092705, 0.009023885242640972, -0.0035885018296539783, 0.0006727855652570724, 0.01547104399651289, -0.00184194918256253, -0.006005006842315197, 0.02434641867876053, -0.005449032410979271, -0.005406939424574375, -0.003070979146286845, 0.028605615720152855, 0.0045657409355044365, -0.008447639644145966, 0.014060555025935173, -0.02609773352742195, -0.002231627469882369, -0.016093777492642403, -0.02187499962747097, -0.01729179173707962, 2.487706660758704e-05, -0.016653643921017647, 0.007122368551790714, 0.011934424750506878, 0.02483328804373741, 0.005048356484621763, 0.004156915005296469, -0.008086747489869595, 0.002900171559303999, -0.014238164760172367, -0.01110175158828497, 0.03273740038275719, 0.022436799481511116, 0.007743579801172018, -0.007951783947646618, -0.0028308737091720104, 0.019021708518266678, 0.004402468912303448, -0.0026788879185914993, -0.0028126374818384647, 0.0008484298014082015, 0.010685634799301624, -0.009817673824727535, -0.01054972130805254, 0.011133360676467419, -0.013031776994466782, 0.020091768354177475, -0.01239350251853466, -0.00947580300271511, -0.007215695921331644, -0.0034883450716733932, 0.012784423306584358, -0.026235120370984077, -0.004409985151141882, -0.008968875743448734, 0.16672198474407196, 0.0025160759687423706, -0.005737856961786747, -0.001338813453912735, -0.0018787229200825095, 0.012143617495894432, 0.004520316142588854, 0.007079395931214094, 0.10781430453062057, 0.009063338860869408, -0.0029431963339447975, 0.008303149603307247, -0.00031500306795351207, -0.004589234944432974, 0.0059718182310462, 0.02588905766606331, -0.005579439923167229, 0.01689429022371769, -0.006689496338367462, 0.006833633873611689, 0.016785891726613045, -0.010677634738385677, 0.01190104242414236, -0.002533396240323782, -0.007275957148522139, -0.00253050634637475, -0.019088461995124817, -0.018875621259212494, 0.02016613818705082, -0.021909620612859726, 0.0032349645625799894, 0.01767919957637787, -0.012697185389697552, 0.05635295808315277, -0.01611725613474846, 0.0064377738162875175, 0.013404080644249916, -0.002685476327314973, 0.0030814805068075657, -0.016239970922470093, -0.006281547714024782, 0.0017259657615795732, -0.02607986330986023, 0.030810900032520294, -0.006045927759259939, 0.0010585401905700564, -0.014656913466751575, -0.016642896458506584, 0.010925422422587872, -0.0038384597282856703, 0.004876702092587948, -0.017458079382777214, -0.016415869817137718, 0.01680702343583107, -0.012250321917235851, 0.006089995615184307, -0.053506746888160706, -0.011852365918457508, 0.015827203169465065, -0.0075766462832689285, -0.001035819062963128, 0.011731131933629513, 0.0026399760972708464, -0.006009269040077925, 0.0010348598007112741, -0.004818199202418327, 0.00935276411473751, -0.004817518871277571, -0.009980720467865467, -0.01086097490042448, -0.012204722501337528, 0.01532853115350008, -0.15799018740653992, -0.0017619041027501225, 0.009771771728992462, -0.018511716276407242, -0.018712328746914864, -0.018102288246154785, 0.021576520055532455, 0.010633010417222977, 0.004200779367238283, 0.010237096808850765, -0.004062600899487734, 0.1258859634399414, -0.014047365635633469, 0.005082272458821535, -0.009407381527125835, -0.00940761063247919, 0.005881977267563343, 0.015061646699905396, -0.0035507441498339176, -0.02356540411710739, -0.010422644205391407, 0.10631720721721649, 0.004390729125589132, 0.023114612326025963, 0.017772773280739784, -0.010049737058579922, 0.003284045960754156, 0.010080733336508274, 0.014630740508437157, -0.013631023466587067, -0.004977407865226269, 0.005198233760893345, 0.0011560205603018403]}, {"text": "how do i filter two power bi tables by the same slicer. 1 candidate table and 1 job title. filter both by a slicer of locations", "label": "LOW", "embedding": [0.03489350527524948, 0.003136225277557969, 0.00013345353363547474, -0.011975273489952087, -0.02074385993182659, -0.005287150386720896, 0.01184588298201561, -0.0007837472949177027, -0.006606614217162132, -0.07512196153402328, -0.017726726830005646, 0.014086160808801651, 0.0010294229723513126, -0.0022694075014442205, -0.03574155271053314, 0.004082104656845331, 0.020955543965101242, 0.010155308991670609, -0.013656181283295155, -0.02212485671043396, 0.007035127840936184, -0.002324436791241169, 0.044143032282590866, -0.015844857320189476, 0.011229412630200386, -0.0037326342426240444, 0.017111873254179955, 0.004489642567932606, -0.004428504966199398, 0.02440137229859829, 0.007222115062177181, 0.019538085907697678, 2.766246552710072e-06, -0.0012459639692679048, -0.018124686554074287, 0.0055198767222464085, -0.006065991707146168, -0.025108763948082924, -0.005404314957559109, -0.0040809460915625095, -0.031273603439331055, -0.00144443497993052, 0.04749973863363266, -0.007255310658365488, 0.0025544781237840652, -0.025226157158613205, 0.003992315847426653, 0.007360522169619799, -0.0014234466943889856, 0.013607335276901722, 0.00034176817280240357, -0.00927402637898922, 0.007617424242198467, 0.00899854488670826, 0.00728200189769268, -0.0002672528789844364, -0.010344168171286583, 0.011964177712798119, -0.02139199525117874, 0.009078671224415302, -0.0051537249237298965, -0.0028415480628609657, 0.02342689409852028, -0.004179775714874268, -0.008705653250217438, 0.0016872896812856197, 0.005423786584287882, 0.0037572099827229977, -0.015002180822193623, -0.0061349933966994286, -0.0016117453342303634, 0.005676937289535999, 0.02021627686917782, 0.06540247052907944, 0.025565925985574722, -0.013750833459198475, 0.0392581969499588, 0.003642213763669133, 0.006029071286320686, 0.00815823394805193, 0.014882792718708515, 0.012616971507668495, -0.017966290935873985, 0.00885151419788599, -0.00962687935680151, -0.010121607221662998, 0.014971887692809105, -0.013193878345191479, 0.004754995461553335, 0.02909468114376068, 0.010229131206870079, -0.009076261892914772, -0.0012980976607650518, -0.012467609718441963, -0.0027021674904972315, 0.0002686473890207708, -0.0001295141555601731, 0.00538935512304306, 0.41036632657051086, -0.0066476380452513695, -0.007657352369278669, 0.005912592634558678, -0.002291856799274683, -0.016884392127394676, 0.02269548550248146, -0.015102856792509556, -0.004088236950337887, -0.02306906320154667, 0.00247234757989645, 0.0271415114402771, 0.0331827849149704, -0.011971925385296345, 0.0098364083096385, -0.007346632890403271, 0.0032067298889160156, -0.009451387450098991, 0.01090692263096571, 0.007905088365077972, 0.014253782108426094, -0.0055229319259524345, 0.0031783091835677624, -0.0012642020592465997, 0.06168726086616516, -9.591568232281134e-05, -0.0025045701768249273, -0.006161816418170929, -0.001115142717026174, -0.01022313255816698, 0.021723931655287743, -0.009461077861487865, -0.05117661878466606, -0.006779432762414217, -0.01318911463022232, -0.0009745320421643555, 0.005076568108052015, 0.009836756624281406, -0.018359294161200523, -0.009561577811837196, -0.042748529464006424, 0.015251007862389088, 0.003996780142188072, -0.019485151395201683, -0.0007838592282496393, -0.020275862887501717, -0.007949027232825756, 0.18964573740959167, -0.003784745465964079, -0.009647030383348465, -0.014201602898538113, -0.00131988525390625, -0.00010317263513570651, -0.021907025948166847, -0.007356033194810152, 0.009314908646047115, -0.016134347766637802, -0.01190085057169199, 0.006837719120085239, 0.007835608907043934, -0.005627134814858437, -0.008309704251587391, -0.0602143332362175, 0.001339669106528163, -0.013408614322543144, 0.02334732748568058, 0.0009237417252734303, -0.004179289098829031, -0.0027809380553662777, -0.04592915624380112, 0.017191432416439056, 0.02761569619178772, 0.00822413805872202, 0.002736262744292617, -0.14652352035045624, 0.020730726420879364, 0.013813169673085213, 0.0065575637854635715, 0.013008414767682552, -0.0026660249568521976, -0.003483609762042761, -0.006917054299265146, -0.016157833859324455, 0.0070387511514127254, 0.0008516897796653211, -0.0038414092268794775, -0.02936074323952198, 0.003852104302495718, 0.012792065739631653, -0.008459504693746567, 0.013004801236093044, 0.010313869453966618, 0.009642493911087513, -0.005682583898305893, -0.002350213471800089, 0.005366537719964981, 0.0008792912121862173, -0.007902107201516628, 0.012484262697398663, -0.012378242798149586, -0.0026265489868819714, -0.004418476950377226, -0.024170750752091408, 0.002578709041699767, -0.008525709621608257, -0.001846909523010254, 0.024708494544029236, 0.007528435438871384, -0.012419942766427994, 0.0039374809712171555, 0.022937869653105736, -0.004780701361596584, -0.006968658417463303, -0.008415778167545795, 0.0004020902269985527, 0.028293462470173836, 0.04243874177336693, -0.018094142898917198, -0.008926065638661385, -0.0071911984123289585, 0.005644379183650017, -0.006335211452096701, -0.007322000805288553, 0.0044991634786129, -0.010952010750770569, -0.010070051066577435, -0.002717274008318782, 0.010766347870230675, 0.01990627869963646, -0.0035804216749966145, 0.012688722461462021, -0.022736918181180954, 0.010980888269841671, 0.01299348846077919, -0.0009025471517816186, -0.01123711559921503, -0.01616118848323822, -0.012390727177262306, -0.001714723533950746, 0.0009573912248015404, -0.0027056888211518526, 0.021031886339187622, -0.00867998693138361, 0.002538442611694336, 0.021973442286252975, -0.004673432558774948, -0.00544340955093503, 0.02622324787080288, 0.027193289250135422, 0.0101301409304142, -0.008740915916860104, -0.005426518619060516, 0.05181683599948883, 0.013538281433284283, -0.025189928710460663, -0.012999259866774082, -0.02151332050561905, 0.005334408022463322, -0.003319022012874484, -0.09284604340791702, -0.026883859187364578, -0.012372216209769249, 0.023541737347841263, -0.18498896062374115, 0.0072428821586072445, 0.0011587415356189013, 0.002605344634503126, -0.021187203004956245, 0.015067552216351032, -0.001675029401667416, -0.014065100811421871, 0.009593685157597065, -0.027208862826228142, 0.001530650770291686, -0.04078829288482666, 0.00651888782158494, 0.0050700632855296135, 0.010110794566571712, -0.011914215050637722, 0.006320086773484945, -0.006854797247797251, 0.0018558050505816936, -0.018411457538604736, -0.005382256116718054, -0.009711542166769505, 0.02761133573949337, -0.024388471618294716, 0.0123403025791049, 0.00800800696015358, 0.00919865071773529, 0.007729295175522566, 0.25240248441696167, 0.01097842026501894, 0.007411558646708727, 0.015599502250552177, -0.01991003006696701, -0.0035277048591524363, 0.0017453861655667424, -0.03897488862276077, 0.012365113943815231, 0.00020124511502217501, 0.01608363538980484, -0.01230367086827755, -0.0012331007746979594, -0.03084763139486313, 0.0035359826870262623, -0.02687348797917366, 0.011445889249444008, -0.008723679929971695, 0.010068788193166256, -0.042385317385196686, -0.0003403893206268549, 0.0350307822227478, -0.001854040427133441, 0.004519979003816843, 0.003987665753811598, -0.007801594212651253, -0.029964245855808258, -0.020831545814871788, 0.0290522500872612, 0.004824759904295206, -0.06082393601536751, 0.011426862329244614, 0.0020574775990098715, -0.01189474482089281, -0.006421489175409079, -0.004679100587964058, -0.0010148432338610291, -0.006593542639166117, -0.016848351806402206, -0.022352872416377068, -0.006653049029409885, -0.0018254532478749752, 0.012821638956665993, -0.0159252118319273, -0.003243707586079836, 0.003305541118606925, -0.0009972199331969023, 0.01254630833864212, 0.004152042791247368, -0.006415905896574259, 0.011264394037425518, -0.016317404806613922, 0.0017327364766970277, -0.010765401646494865, 0.016060832887887955, 0.017356321215629578, 0.004629525355994701, 0.015559028834104538, -0.009205066598951817, 0.005914720706641674, 0.005188140086829662, -0.012476516887545586, -0.0021531949751079082, -0.01725919544696808, 0.015073457732796669, 0.01845303177833557, -0.016382357105612755, -0.004425075836479664, -0.0173539649695158, 0.0060491301119327545, 0.010679347440600395, 0.005394438747316599, -0.1340526044368744, 0.00863219890743494, 0.0038497268687933683, -0.008710339665412903, 0.006677952129393816, -0.015056823380291462, -0.003315312322229147, 0.0031753908842802048, 0.029113877564668655, 0.011080587282776833, 0.002320552011951804, 0.01931845396757126, 0.014308829791843891, -0.020096998661756516, 0.00567164272069931, 0.0026119013782590628, 0.042072538286447525, 0.019103877246379852, -0.025333352386951447, -0.01284492202103138, 0.013236301951110363, 0.01775035448372364, -0.004607044626027346, -0.0017861351370811462, 0.004510845057666302, -0.009691034443676472, -0.028329715132713318, 0.004135138355195522, 0.003144781570881605, 0.029054181650280952, 0.007423423696309328, -0.009466951712965965, -0.005953717045485973, -0.00858550239354372, -0.001516006770543754, -0.0019972948357462883, 0.01263347640633583, 0.010178273543715477, -0.016717251390218735, -0.012670782394707203, 0.002490847371518612, 0.0005430335877463222, -0.009636507369577885, -0.017666757106781006, -0.009260745719075203, 0.015040842816233635, -0.02130410633981228, -0.0943947583436966, 0.004737202078104019, -0.00856553204357624, -0.020302556455135345, -0.007108778227120638, 0.011324901133775711, 0.016321448609232903, -0.0032582732383161783, 0.010336773470044136, -0.005004193168133497, 0.02124362252652645, -0.0014505223371088505, 0.011753576807677746, 0.0067431931383907795, 0.004421417135745287, -0.02450242079794407, -0.0010102188680320978, 0.009934723377227783, -0.0020738786552101374, -0.01860458217561245, 0.005377030465751886, -0.003058214206248522, -0.04875076934695244, -0.0072264294140040874, 0.0020145855378359556, -0.01569741778075695, -0.019948750734329224, -0.023663898929953575, 0.006632380653172731, -0.01276591420173645, -0.008654098957777023, 0.33793506026268005, -0.015704138204455376, 0.021552788093686104, 0.006733962334692478, -0.013085402548313141, 0.005039595067501068, 0.014720221981406212, -0.00965897087007761, 0.05943867191672325, 0.0058715068735182285, -0.021828019991517067, -0.018740007653832436, 0.010086439549922943, 0.0007075808825902641, -0.023008570075035095, 0.02946694940328598, -0.003728704061359167, -0.009074573405086994, 0.006820289418101311, -0.0019728182815015316, -0.001360346213914454, -0.011618522927165031, 0.010857504792511463, -0.015327819623053074, 0.01585414819419384, 0.017435139045119286, 0.013905618339776993, -0.0021413995418697596, -0.0003165137895848602, -0.018046429380774498, -0.010896043851971626, 0.021377796307206154, 0.00798007007688284, 0.008790670894086361, 0.004094349220395088, -0.009578037075698376, 0.012415286153554916, 0.0010678753023967147, 0.007617679424583912, 0.01822771690785885, 0.003930150996893644, -0.006726063787937164, 0.06144767999649048, -0.00422295555472374, 0.012881600297987461, 0.018554652109742165, 0.00010200912220170721, -0.011009606532752514, 0.023077400401234627, 0.008675302378833294, -0.00468615535646677, 0.009468049742281437, 0.017467651516199112, -0.01936344802379608, 0.008646710775792599, 0.004596613813191652, 0.011698278598487377, 0.005212021060287952, 0.0005336372996680439, 0.014455361291766167, -0.014320742338895798, -0.01153484359383583, -0.014126122929155827, -0.008116602897644043, 0.002168098231777549, -0.012662140652537346, 0.00565036665648222, -0.016244441270828247, -0.028341395780444145, -0.004490296356379986, -0.008218486793339252, 0.013727612793445587, -0.02716231718659401, 0.016405394300818443, 0.014315237291157246, 0.0008104253793135285, -0.02502485364675522, 0.0019569380674511194, 0.0059456052258610725, -0.024564282968640327, -0.009556181728839874, -0.0209627915173769, -0.001051364466547966, -0.023019609972834587, 0.005734703503549099, 0.0025941915810108185, -0.023694239556789398, -0.016781622543931007, -0.000793750979937613, 0.004647925030440092, -0.0018065025797113776, -0.005727069452404976, -0.008392387069761753, -0.00039135711267590523, 0.003294403897598386, 0.022617654874920845, -0.003877888899296522, 0.0009011466172523797, -0.00644189678132534, 0.0029323077760636806, 0.023241916671395302, 0.008961361832916737, 0.0018858362454921007, -0.0058393776416778564, 0.0013933161972090602, -0.006949798669666052, 0.001753459102474153, 0.01340426504611969, -0.002735417103394866, -0.00797993317246437, 0.017810635268688202, 0.006618174724280834, 0.02216007187962532, -0.015597355552017689, 0.0045047602616250515, 0.030001724138855934, 0.00015865691239014268, -0.026099342852830887, -0.44930461049079895, 0.007250750903040171, -0.005528052803128958, 0.013061963953077793, -0.024329595267772675, -0.006794680375605822, 0.03133322298526764, 0.012338776141405106, 0.016395624727010727, -0.005758542567491531, -0.01922575756907463, -0.009432356804609299, -0.14546319842338562, 0.017504429444670677, -0.016465600579977036, -0.003736065700650215, -0.010625826194882393, -0.07051914930343628, -0.003342571435496211, 0.023106249049305916, -0.005833889357745647, 0.020698867738246918, 0.01889161206781864, 0.011523611843585968, 0.014739547856152058, -0.0009696630295366049, 0.005834754556417465, -0.00619840482249856, 0.012872541323304176, 0.004132158122956753, 0.004181017633527517, -0.010223970748484135, 0.03565081208944321, 0.0032651021610945463, -0.003252684138715267, -0.006327474489808083, -0.005220359191298485, -0.001558981486596167, 0.011387650854885578, -0.01906585320830345, -0.006231805309653282, -0.004777147900313139, 0.010010097175836563, 0.011364196427166462, -0.03575512394309044, -0.014719055034220219, -0.021945249289274216, -0.15378914773464203, 0.0009750770987011492, -0.015257589519023895, 0.0069380286149680614, 0.014768303371965885, -0.013931108638644218, -0.010922227054834366, -0.016931962221860886, 0.011851380579173565, 0.006281539797782898, 0.005613933317363262, 0.019927816465497017, -0.02298779971897602, -0.014188777655363083, -0.012134605087339878, 0.032970260828733444, 0.0018828498432412744, -0.01914048381149769, 0.0047907340340316296, -0.025417614728212357, -0.012875007465481758, -0.020905744284391403, -0.016441741958260536, -0.003318347269669175, -0.01381655503064394, -0.004414008930325508, 0.00791921652853489, 0.021895097568631172, 0.027420450001955032, 0.011240720748901367, -0.006539409048855305, 0.011839570477604866, 0.0101950503885746, -0.02490006387233734, -0.014599321410059929, 0.01928459107875824, 0.014013205654919147, 0.007938386872410774, -0.008626536466181278, -0.004591081757098436, 0.005954395048320293, -0.022290261462330818, 0.002334084128960967, 0.002876383950933814, 0.011544688604772091, -0.005610486026853323, -0.01666816882789135, -0.011346370913088322, -0.0011619427241384983, -0.01569129340350628, -0.0029112736228853464, -0.035083889961242676, -0.004116900265216827, -0.003334674285724759, 0.010524335317313671, 0.014860863797366619, -0.02679758332669735, -0.011025146581232548, -0.01771407574415207, 0.16937431693077087, 0.005695281084626913, -0.016888050362467766, -0.007332572713494301, -0.0027092837262898684, 0.006453078240156174, -0.0012794117210432887, 0.01524088904261589, 0.11179395765066147, -0.005055903922766447, -0.017007235437631607, 0.007990255020558834, 0.0021452398505061865, 0.0008146287291310728, 0.005208081100136042, 0.034080252051353455, 0.011706477962434292, 0.013624854385852814, -0.0032176831737160683, 0.010031750425696373, 0.012511772103607655, -0.014216778799891472, 6.898899300722405e-05, -0.005686796735972166, 0.012578018940985203, -0.008595922961831093, -0.017523448914289474, -0.008871931582689285, 0.0073803192935884, -0.02266545407474041, 0.0015746266581118107, -0.014443025924265385, -0.016635186970233917, 0.05729164183139801, -0.01529026497155428, 0.013618448749184608, 0.022922197356820107, 0.011116747744381428, 0.0006602559005841613, -0.02280248887836933, -0.008121669292449951, 0.014244169928133488, -0.034028418362140656, 0.034336693584918976, -0.0017677516443654895, -0.008115175180137157, -0.007907604798674583, -0.008570886217057705, 0.005416251718997955, -0.008143365383148193, -0.006494828034192324, -0.021416710689663887, -0.014334254898130894, 0.006336995400488377, 0.0008985999156720936, -0.0015707724960520864, -0.05117341876029968, -0.008252335712313652, 0.01367854978889227, -0.004904999863356352, 0.0024623065255582333, -0.006511757615953684, 0.001568328938446939, 0.003755437210202217, -0.006121892016381025, 0.0007552523748017848, 0.00407383730635047, -0.033182721585035324, -0.008839397691190243, -0.013622280210256577, -0.006673557683825493, 0.00877341441810131, -0.13768140971660614, -0.0013742416631430387, 0.00843155849725008, -0.039339710026979446, -0.015977364033460617, -0.01924612559378147, 0.014812890440225601, 0.010088124312460423, -0.00045744553790427744, 0.01315030362457037, 0.01037298608571291, 0.11654022336006165, -0.004852269310504198, -0.007917158305644989, -0.016654713079333305, -0.008315951563417912, 0.01723969541490078, 0.010663866065442562, -0.006389958783984184, -0.025507966056466103, -0.019618062302470207, 0.10039320588111877, 0.005359656177461147, 0.021398251876235008, 0.011016220785677433, -0.010477172210812569, -0.0008988783229142427, 0.004391006659716368, -0.007086579222232103, 0.0009222133085131645, -0.016631904989480972, 0.005759488791227341, 0.013994482345879078]}, {"text": "You are AoControl, a helpful and knowledgeable agent. To achieve your goal of answering complex questions correctly, you have access to the following tools:\n AoInnovus: An agent with knowledge of the Cadence Innovus User Guide and corresponding Tcl command usage\n AoTcl: An agent with knowledge of the Tcl language\nTo answer questions, you'll need to go through multiple steps involving step-by-step thinking and selecting appropriate tools and their inputs; tools will respond with observations.\nWhen you are ready for a final answer, respond with the `Final Answer:`\nUse the following format:\nQuestion: The question to be answered\nTHought: Reason if you have the final answer. If yes, anwer the question. If not, find out the missing information needed to answer it.\nTool: Pick one of {AoInnovus, AoTcl}\nTool Input: The input for the tool\nObservation: The tool will respond with the result\n...\nFinal Answer: The final answer to the question, make it short (50-100 words)\nThought, Tool, Tool Input, and Observation steps can be repeated multiple times, but sometimes we can find an answer in the first pass.\n---\nQuestion: How do I identify the impact of metal fill on my timing violations?\nThought: Let's think step-by-step, I first need to", "label": "LOW", "embedding": [0.028481466695666313, -0.0032100745011121035, -0.0036680581979453564, -0.012064126320183277, -0.014738199301064014, -0.008851398713886738, 0.01305114571005106, -0.007984080351889133, 0.006234572269022465, -0.07052527368068695, -0.01605345867574215, 0.0028764274902641773, 0.006010148208588362, -0.005135666579008102, -0.03757606819272041, 0.011544846929609776, 0.013252086006104946, -0.008494338020682335, -0.012022257782518864, -0.01595737598836422, 0.005728313699364662, -0.003944289870560169, 0.04078539088368416, -0.019679097458720207, 0.013171138241887093, -0.0033915722742676735, 0.013431847095489502, -0.002831314457580447, -0.012022067792713642, 0.03594043105840683, -0.003874531714245677, 0.016333891078829765, -0.007171308621764183, -0.014637019485235214, -0.0127987926825881, -0.0014821848599240184, -0.003763703629374504, -0.023475799709558487, -0.01440843753516674, -0.011180538684129715, -0.024014534428715706, -9.284873522119597e-05, 0.030041739344596863, -0.023861469700932503, -0.0041903662495315075, -0.028050566092133522, -0.008066064678132534, 0.004128975793719292, 0.006282537244260311, 0.011226043105125427, -0.00032679576543159783, -0.006346442271023989, 4.8125351895578206e-05, 0.011022440157830715, -0.00018652713333722204, 0.008158954791724682, 0.004160143435001373, -0.007040262222290039, -0.014989886432886124, 0.008227295242249966, -0.005556535441428423, -0.0031042799819260836, 0.010290107689797878, 0.008744004182517529, -0.01597421243786812, -0.0016717870021238923, 0.0063789780251681805, -0.0012683117529377341, -0.0039807530120015144, -0.011786128394305706, -0.005436820909380913, 0.0018739051884040236, 0.007266359869390726, 0.06621047109365463, 0.021580353379249573, -0.005671683233231306, 0.034882523119449615, 0.01079897303134203, 0.007216220255941153, 0.006121003534644842, 0.010424097068607807, -0.0035014578606933355, -0.010838386602699757, 0.002388108056038618, -0.005598912946879864, -0.0006575909210368991, 0.013120759278535843, -0.011220728978514671, -0.005976819898933172, 0.01715632900595665, 0.006815479602664709, -0.004414496943354607, -0.034402087330818176, -0.008330200798809528, -0.007713593076914549, 0.016911979764699936, 0.006792599800974131, 0.001579583971761167, 0.4029686152935028, 0.006133169401437044, -0.006269508507102728, 0.0009417503024451435, -0.0047039189375936985, -0.0023076010402292013, 0.019042614847421646, -0.016674473881721497, -0.01143437810242176, -0.022738857194781303, -0.0013461931375786662, 0.033254340291023254, 0.03403940424323082, -0.004657887853682041, 0.010137207806110382, 0.003191462717950344, 0.00423059007152915, -0.014854967594146729, 0.00187963608186692, 0.0033404480200260878, 0.008774741552770138, -0.004102198872715235, 0.013924067839980125, -0.002166380872949958, 0.06777045875787735, 0.01696203276515007, -0.009100930765271187, -0.013235462829470634, 0.003540582722052932, -0.009110520593822002, 0.010211421176791191, -0.01178856659680605, -0.04253000393509865, -0.014148823916912079, -0.009067322127521038, 0.0002958943950943649, 0.009686066769063473, 0.0011271099792793393, -0.022043123841285706, 0.004045727197080851, -0.04135211184620857, 0.011710057966411114, 0.007540761027485132, -0.021476654335856438, 0.011550378985702991, -0.01543466653674841, -0.018416501581668854, 0.18628528714179993, -0.010102721862494946, -0.009657307527959347, -0.0007819949532859027, -0.002873751102015376, -0.010273076593875885, -0.018142543733119965, -0.01662677340209484, 0.0031405820045620203, -0.0047050523571670055, -0.012725889682769775, 0.0184447281062603, -0.0031997181940823793, -0.006230160593986511, -0.0003932864929083735, -0.05243059620261192, 0.002467630198225379, -0.006740711163729429, 0.041581891477108, 0.00135491241235286, -0.0043099713511765, 0.00013487310206983238, -0.0531163327395916, 0.01094493642449379, 0.0371587909758091, 0.009056678041815758, -0.0022147393319755793, -0.15529899299144745, 0.012336480431258678, 0.012374239973723888, -0.0013503319351002574, 0.008443396538496017, -0.00022072401770856231, -0.00296808616258204, -0.010984603315591812, -0.017146727070212364, 0.01517662312835455, 0.0016807044157758355, -0.008087344467639923, -0.020684365183115005, 0.00844782218337059, 0.004575537052005529, -0.02506963536143303, -0.003397218883037567, 0.002653817180544138, 0.0016820784658193588, -0.013694286346435547, 0.002471347339451313, 0.01588994637131691, 0.020769501104950905, -0.0029095239005982876, 0.015043907798826694, -0.017222166061401367, -0.01522522047162056, -0.001344539923593402, -0.010885421186685562, 0.005388220772147179, -0.01165734138339758, 0.002077261218801141, 0.015180230140686035, 0.0065882327035069466, -0.004321780521422625, 0.0065977550111711025, 0.024087296798825264, 0.004317300394177437, -0.01362061221152544, 0.000977547257207334, -0.009196503087878227, 0.020808910951018333, 0.0413227453827858, -0.014660883694887161, -0.005173196084797382, 0.008529740385711193, 0.0005162470042705536, -0.0051298500038683414, 0.01033002883195877, 0.012104791589081287, -0.0182360652834177, -0.01297315489500761, 0.0014281488256528974, 0.003872287692502141, 0.007588055916130543, -0.014890051446855068, -0.0001284214376937598, -0.016883866861462593, 0.00034463449264876544, 0.011394888162612915, -0.0013301551807671785, -6.5691776399035e-05, 0.0027407945599406958, -0.014958030544221401, -0.00879379827529192, 0.017592739313840866, 0.008886925876140594, 0.024556633085012436, -0.0017414436442777514, -0.0027629819232970476, 0.023790931329131126, -0.013575847260653973, 0.0021446903701871634, 0.013762501999735832, 0.024049069732427597, 0.00280263414606452, -0.004523663781583309, -0.0053471969440579414, 0.061793193221092224, 0.01940036751329899, -0.01367171574383974, -0.010136540979146957, -0.028502998873591423, 0.00020018868963234127, -0.009876967407763004, -0.08737751841545105, -0.02316276729106903, -0.011066901497542858, 0.008019373752176762, -0.19544245302677155, 0.012813440524041653, 0.010146625339984894, 0.003991636913269758, -0.008251442573964596, 0.01881340891122818, 0.007313342299312353, -0.005583884660154581, -0.002973964437842369, -0.029929479584097862, 0.009119047783315182, -0.028870416805148125, 0.0014640770386904478, -0.0021524184849113226, 0.02146441489458084, 0.0005779755883850157, -0.0027765659615397453, 0.008556170389056206, 0.0009732022881507874, -0.019156960770487785, 0.006540260743349791, -0.015143433585762978, 0.03906533122062683, -0.02133104018867016, 0.02247505635023117, 0.009192245081067085, 0.002206547185778618, 0.0037197815254330635, 0.25034400820732117, 0.02491210587322712, 0.0064856004901230335, 0.0181731004267931, -0.026356995105743408, -0.003594128880649805, -0.0026246323250234127, -0.03161114081740379, 0.017528334632515907, -0.0027028783224523067, 0.004975538235157728, -0.01684742048382759, -0.0060270666144788265, -0.03920280933380127, 0.008427784778177738, -0.022113759070634842, 0.008520949631929398, -0.0029229901265352964, 0.011051123961806297, -0.04535704478621483, -0.009062517434358597, 0.027849270030856133, -7.125990669010207e-05, 0.008085643872618675, 0.003574596717953682, -0.0058377450332045555, -0.043075814843177795, -0.024250848218798637, 0.03552256152033806, 0.008264884352684021, -0.05097333341836929, 0.006994653958827257, 0.00281835556961596, -0.022983506321907043, -0.012356574647128582, 0.0008327520336024463, -0.0020424528047442436, 0.0013315633404999971, -0.018399694934487343, -0.019879095256328583, 0.00553104979917407, -0.0048089600168168545, 0.02499583177268505, -0.01640998385846615, -0.008041865192353725, 0.006009378004819155, -0.008389551192522049, 0.0005091773928143084, 0.013637284748256207, 0.008504332043230534, 0.00012893107486888766, -0.015254270285367966, -0.0037575499154627323, -0.02032279036939144, 0.011838273145258427, 0.01785503700375557, 0.009433496743440628, 0.01658165641129017, -0.01113493088632822, 0.019913066178560257, -0.0025226385332643986, -0.0071386778727173805, 0.003986249677836895, -0.006006501615047455, 0.019915560260415077, 0.006187689024955034, -0.016032641753554344, -0.0052686394192278385, -0.01946667581796646, 0.008946087211370468, 0.007346574682742357, 0.008798312395811081, -0.13482312858104706, 0.019759267568588257, 0.01634576916694641, -0.010625009424984455, 0.0030150171369314194, -0.005240299738943577, 0.011385271325707436, -0.0049846479669213295, 0.021069277077913284, 0.011931764893233776, -0.007639411836862564, 0.0031936154700815678, 0.01148471049964428, -0.021793212741613388, -0.0005502455751411617, 0.01754508726298809, 0.04001929238438606, 0.016804732382297516, -0.014178791083395481, -0.014155114069581032, 0.009545539505779743, 0.015728304162621498, -0.002480913884937763, -0.0037433793768286705, -0.004303444176912308, 0.0036718042101711035, -0.027051599696278572, 0.0005708340322598815, -0.0045464555732905865, 0.018299678340554237, -0.005026747472584248, -0.007229410577565432, -0.004213571548461914, -0.0010581830283626914, -0.0009691077866591513, -8.765130769461393e-05, 0.01213790662586689, 0.011448062025010586, -0.015211221762001514, -0.014225609600543976, 0.005352279171347618, 0.002950174966827035, 0.0018103790935128927, -0.01874382235109806, 0.004209776408970356, 0.007550138048827648, -0.01867091841995716, -0.10517671704292297, -0.014730149880051613, -0.011389746330678463, -0.014610673300921917, -0.00628548301756382, 0.0066108000464737415, 0.014477737247943878, 0.003205421380698681, -0.005561803467571735, -0.005047264043241739, 0.015232146717607975, 0.0026111097540706396, 0.008841216564178467, 0.0013381035532802343, 0.01354994997382164, -0.02971905656158924, 0.002546435920521617, 0.019230207428336143, 0.001415462582372129, -0.02076919749379158, 0.007205471396446228, 0.00804719515144825, -0.05084868520498276, -0.004378694109618664, 0.0028741599526256323, -0.029340794309973717, -0.006062444765120745, -0.01728450506925583, 0.004825662821531296, -0.017383180558681488, -0.013110457919538021, 0.3406284749507904, -0.011256606318056583, 0.015743611380457878, 0.007575451396405697, -0.014195769093930721, 0.00389893283136189, 0.019256463274359703, -0.009480844251811504, 0.05363301932811737, 0.0031888687517493963, -0.021639686077833176, -0.024615388363599777, 0.016033759340643883, 0.0012751492904499173, -0.01906251348555088, 0.021517276763916016, 0.004980066791176796, -0.003369574435055256, -0.006726786494255066, 0.002780372742563486, -0.0071771773509681225, -0.015407818369567394, 0.016232172027230263, -0.009401519782841206, 0.002734348876401782, 0.018209725618362427, 0.00510728545486927, -0.0025584683753550053, -0.00753369415178895, -0.013244487345218658, -0.018637966364622116, 0.018083272501826286, 0.009038437157869339, 0.018527138978242874, 0.001672680489718914, -0.014017142355442047, 0.001983570633456111, 0.0003638377820607275, 0.013502437621355057, 0.015476514585316181, 0.01534595899283886, -0.005360999144613743, 0.05548416078090668, -0.0022822082974016666, 0.016051001846790314, 0.009160741232335567, -0.0007279389537870884, -0.01568947173655033, 0.022993922233581543, 0.01002364233136177, 0.0009458106360398233, 0.021198786795139313, 0.008258165791630745, -0.01263381727039814, 0.009771843440830708, 0.010516144335269928, 0.0033517556730657816, -0.008542624302208424, 0.007836688309907913, 0.00304834870621562, 0.0016635021893307567, -0.0173858143389225, -0.004054618533700705, -0.004043348599225283, 0.015762904658913612, -0.004780821036547422, 0.005086285527795553, -0.016211949288845062, -0.027660345658659935, -0.012494189664721489, -0.017237087711691856, 0.00409947382286191, -0.01701396517455578, 0.013591744005680084, 0.018766677007079124, 0.009997373446822166, -0.02899988740682602, 0.005956615786999464, 0.0018688042182475328, -0.012759780511260033, -0.01984594017267227, -0.01342670526355505, -0.007871230132877827, -0.017757218331098557, -0.010107247158885002, -0.005552962888032198, -0.042343057692050934, -0.01442392822355032, 0.004624031018465757, 0.009142719209194183, -0.0024364530108869076, -0.00779950013384223, -0.012429878115653992, 0.009014386683702469, -0.005744538269937038, 0.011449596844613552, 2.4167520678020082e-05, -0.010021339170634747, -0.005453824531286955, 0.007501058280467987, 0.023800330236554146, 0.000507863936945796, 0.0009390436462126672, -0.009046148508787155, 0.002733289496973157, -0.0023227271158248186, -0.004945544991642237, 0.008424296043813229, -0.012587535195052624, -0.0005831593298353255, 0.0115160858258605, -0.005984772462397814, 0.012134176678955555, 0.0018935505067929626, -0.007110421545803547, 0.020056268200278282, -0.006530118174850941, -0.02069927752017975, -0.4561271667480469, 0.006033765152096748, 0.006408394780009985, 0.004936664830893278, -0.021691415458917618, 0.0016890342812985182, 0.026843173429369926, 0.004279727581888437, 0.015640810132026672, 0.00015536749560851604, -0.022893333807587624, -0.009500727988779545, -0.13548730313777924, 0.013845794834196568, -0.008374335244297981, 0.002331565599888563, -0.0059048766270279884, -0.05896355211734772, 0.0010769212385639548, 0.03999321162700653, -0.004881581291556358, 0.010504393838346004, 0.01977461576461792, 0.014779353514313698, 0.029292156919836998, -0.01331364642828703, -0.00449405238032341, 0.005207533482462168, 0.00795875396579504, -0.0009343190467916429, -0.00478253373876214, -0.021151812747120857, 0.027697231620550156, 0.0037257433868944645, -0.0008393958560191095, -0.009514912962913513, -0.002861006883904338, -0.0014223129255697131, 0.011421425268054008, -0.02395738661289215, -0.006004301365464926, 0.007238397840410471, 0.013602610677480698, 0.013419140130281448, -0.030369361862540245, -0.01429602038115263, -0.03642832860350609, -0.148605614900589, 0.002944819163531065, -0.008960862644016743, 0.017539873719215393, 0.0005102307186461985, 0.0022721323184669018, -0.006210957188159227, -0.011270688846707344, 0.011764095164835453, 0.007209755480289459, 0.010177290998399258, 0.019597399979829788, -0.014854487963020802, -0.01290462352335453, -0.006237701512873173, 0.02643461711704731, 0.002075844444334507, 0.004868035204708576, 0.008638898842036724, -0.017200855538249016, -0.0013375000562518835, -0.018687915056943893, -0.02052062377333641, -0.0008527587633579969, -0.007883081212639809, -0.004736979026347399, 0.010171622037887573, 0.018002936616539955, 0.021635418757796288, 1.7149042832897976e-05, -0.003462042659521103, 0.007129318546503782, -0.001986239105463028, -0.025379445403814316, -0.006045063491910696, 0.015646036714315414, 0.025369713082909584, 0.01897394098341465, -0.004390508867800236, -0.0018207089742645621, 0.015384625643491745, -0.011445743031799793, 0.009166812524199486, 0.0001588084560353309, 0.01275301817804575, 0.0068309111520648, -0.00376890879124403, -0.0017735871952027082, 0.00171623972710222, -0.019546490162611008, 0.008847960270941257, -0.020602576434612274, -0.008028321899473667, -0.007949132472276688, 0.005732436198741198, 0.008815097622573376, -0.029176734387874603, -0.00013534327445086092, -0.01748441345989704, 0.17584189772605896, 0.014477892778813839, -0.0020427689887583256, 0.004106832668185234, -0.008773569017648697, 0.005818760488182306, -0.0002685371437110007, 0.009639648720622063, 0.09532323479652405, 0.0035690173972398043, -0.018146900460124016, 0.013089788146317005, -0.0009885039180517197, 0.008971941657364368, 0.008153371512889862, 0.024393999949097633, 0.002158266259357333, 0.014087831601500511, -0.015712009742856026, 0.0044474187307059765, 0.011503162793815136, -0.012693719938397408, -0.00025247258599847555, -0.011811968870460987, -0.0024292569141834974, 0.002557281870394945, -0.03024854138493538, -0.01596192829310894, 0.01634310372173786, -0.02085294760763645, 0.005664019379764795, 0.004347300156950951, -0.02302495948970318, 0.06019114702939987, -0.01967056654393673, -0.0006525053177028894, 0.02158287726342678, -0.004441113211214542, -0.0018470429349690676, -0.02695525996387005, -0.010261321440339088, -0.010808425955474377, -0.03391271084547043, 0.03491147980093956, 0.0033113835379481316, -0.0026383099611848593, -0.015767330303788185, -0.009497586637735367, 0.002648768713697791, -0.013696894980967045, -0.017453767359256744, -0.013554755598306656, -0.006575877778232098, 0.007614815607666969, 0.003164459951221943, -0.008089522831141949, -0.04084805026650429, -0.015493734739720821, 0.012476840987801552, -0.015358414500951767, 0.008400984108448029, -0.0027257942128926516, 0.0037222024984657764, -0.004754429683089256, -0.0017123222351074219, -0.005012303590774536, 0.0025338344275951385, -0.025273513048887253, 0.0007854397990740836, -0.01523483358323574, -0.009258589707314968, 0.008368309587240219, -0.1480589210987091, 0.002570760902017355, 0.010927931405603886, -0.03319719806313515, -0.023863675072789192, -0.01907525770366192, 0.02151910774409771, 0.013949736021459103, -0.0012292651226744056, 0.01792049966752529, 0.004961602855473757, 0.12112898379564285, 0.00016630523896310478, -0.002727924380451441, -0.0075034270994365215, -0.013858512043952942, 0.015004047192633152, 0.010907307267189026, -0.013387860730290413, -0.014549978077411652, -0.015193810686469078, 0.1108766570687294, 0.003858026582747698, 0.02428543195128441, 0.006991077214479446, -0.0043445914052426815, -0.0035389061085879803, 0.006504585966467857, 0.008052886463701725, -0.003911263309419155, -0.01721899025142193, 0.01042222697287798, 0.010188529267907143]}, {"text": "May you write an article about 'Global Warming' in 250 words?", "label": "LOW", "embedding": [0.026489121839404106, 0.00400069123134017, -0.014728345908224583, -0.010334095917642117, -0.023692931979894638, 0.0008241761825047433, 0.02020481787621975, -0.005474946927279234, 0.005612898617982864, -0.07204721868038177, -0.017359837889671326, -0.002814121078699827, -0.010488013736903667, -0.0057466281577944756, -0.03895016014575958, 0.0022793805692344904, 0.012593495659530163, -0.001533782808110118, -0.0008843732648529112, -0.011035108007490635, 0.011750071309506893, -0.012735392898321152, 0.03985227644443512, -0.021356288343667984, 0.009403713047504425, -0.0031369798816740513, 0.01526428759098053, 0.007691060192883015, -0.015358931384980679, 0.026734163984656334, -0.004999430850148201, 0.014628211036324501, -0.011224731802940369, 0.001256199786439538, -0.015948021784424782, 0.009516090154647827, -0.0031164793763309717, -0.023098936304450035, -0.010067666880786419, -0.004284778144210577, -0.03537492826581001, 0.014002113603055477, 0.04402836039662361, -0.01306924782693386, 0.006599972024559975, -0.033028822392225266, -0.003748263930901885, 0.003386382944881916, -0.011534221470355988, 0.011218801140785217, -0.001477933255955577, 0.00415984820574522, -0.012463998049497604, 0.0036991441156715155, 0.010405167005956173, 0.004511087667196989, -0.007595773320645094, 0.007987024262547493, -0.034262534230947495, 0.016504181548953056, 0.005265682935714722, -0.0033367990981787443, 0.028063137084245682, 0.007109018974006176, -0.013013501651585102, -0.0035732965916395187, 0.007727885153144598, -0.002431234810501337, -0.022401906549930573, -0.006180774420499802, -0.0014240851160138845, 0.0068091582506895065, 0.011243925429880619, 0.05715678259730339, 0.018198803067207336, -0.014894126914441586, 0.033470917493104935, 0.007320176810026169, -0.0007230306509882212, 0.00030121408053673804, 0.002917248522862792, 0.006361300125718117, -0.009833664633333683, -0.0024141683243215084, -0.0024956914130598307, -0.0024197392631322145, 0.009225239045917988, 0.00046602735528722405, -0.013586809858679771, 0.015147141180932522, 0.006640798412263393, -0.003830119501799345, -0.024916373193264008, 0.0007264112937264144, -0.00666396040469408, 0.010914194397628307, 0.002241949550807476, 0.0020669098012149334, 0.4120471477508545, -0.007469322066754103, -0.011694100685417652, 0.008362552151083946, 0.010790076106786728, -0.008111247792840004, 0.029171261936426163, -0.024237334728240967, -0.006435296963900328, -0.022633958607912064, 0.004903746303170919, 0.030382033437490463, 0.028432361781597137, -0.00011019509111065418, 0.003046040190383792, -0.009058207273483276, -0.007793826516717672, -0.021961621940135956, 0.0022128876298666, -0.00289185275323689, 0.009550461545586586, -0.012613466009497643, 0.008994808420538902, -0.006424738094210625, 0.05871381610631943, 0.021361693739891052, -0.009635847993195057, -0.00976010411977768, -0.001816694624722004, -0.011921563185751438, 0.019271790981292725, -0.014263426885008812, -0.043140240013599396, -0.014404241926968098, -0.003948269411921501, -0.006233703810721636, 0.009688949212431908, 0.006052189972251654, -0.007256435230374336, 0.012071501463651657, -0.06734088808298111, 0.01144150085747242, 0.012725375592708588, -0.01372016966342926, 0.007166068069636822, -0.011385287158191204, -0.025669513270258904, 0.1845022290945053, -0.005295124836266041, 0.001330484519712627, -0.004056419711560011, -0.0036274029407650232, 0.009140648879110813, -0.013705695047974586, -0.015275265090167522, 0.007203268352895975, 0.0030933243688195944, -0.002482819138094783, 0.009870445355772972, 0.010142656974494457, 0.005700578913092613, 0.006706345360726118, -0.05801817402243614, 0.00836422573775053, -0.0031418108846992254, 0.026719436049461365, 0.003422003937885165, -0.01663697324693203, 0.014333480037748814, -0.04842846095561981, 0.009970898739993572, 0.02627507783472538, -0.002832365222275257, -0.008798259310424328, -0.15845727920532227, 0.02108398638665676, 0.005248647183179855, 0.004556745756417513, 0.006811157334595919, 0.0031765352468937635, -0.007436682935804129, -0.01715732179582119, -0.013802091591060162, 0.021496128290891647, 0.0031680732499808073, 0.003170708427205682, -0.02458208240568638, 0.004539525602012873, -0.002173606539145112, -0.004159450065344572, -0.003645094111561775, 0.011452230624854565, -0.0011853509349748492, -0.022664332762360573, 0.009061693213880062, 0.010235944762825966, 0.016389230266213417, -0.002338721649721265, 0.0073088775388896465, -0.011342614889144897, -0.003760087303817272, 0.00975928083062172, -0.005131568294018507, -0.005393259692937136, -0.014259041287004948, 0.011857581324875355, 0.02465076744556427, 0.007355081848800182, -8.686680666869506e-05, 0.010473915375769138, 0.019402310252189636, -0.0014300253242254257, -0.016695745289325714, 0.009161530993878841, 0.0018877959810197353, 0.03230073302984238, 0.028618942946195602, -0.01540833804756403, 0.0014513954520225525, 0.008468933403491974, 0.010029951110482216, -0.0006899710278958082, -0.01325999666005373, 0.007204756140708923, -0.025562508031725883, -0.012359808199107647, 0.0020073221530765295, -0.0033736301120370626, 0.009656673297286034, 0.004118646029382944, -0.0021286301780492067, -0.013094337657094002, 0.0059400442987680435, 0.009158503264188766, -0.0007123309769667685, -0.010517236776649952, 0.0040705506689846516, 0.004455859772861004, -0.01141218189150095, 0.009197242558002472, 0.0018598920432850718, 0.016502024605870247, -0.006810961756855249, -0.0031144344247877598, 0.02555752359330654, -0.007616182789206505, 0.006809026002883911, 0.012505401857197285, 0.02352095954120159, 0.00807160884141922, 0.003514694981276989, 0.001785237225703895, 0.0435749813914299, 0.012305240146815777, -0.012156525626778603, -0.01898384839296341, -0.03937593102455139, -0.0056031858548521996, -0.008025544695556164, -0.09394460171461105, -0.021436678245663643, -0.003664312418550253, 0.00817066803574562, -0.19624119997024536, 0.011035515926778316, 0.01442545372992754, 0.0021780566312372684, -0.006457013543695211, 0.01978391222655773, -0.0031551348511129618, -0.016423679888248444, 0.005288708955049515, -0.03664819523692131, 0.001127908006310463, -0.036206409335136414, 0.009074990637600422, -0.009406975470483303, 0.019579127430915833, -0.008397411555051804, -0.000896182784345001, -0.01618834212422371, -0.019384179264307022, -0.021903371438384056, 0.01028576958924532, -0.015276232734322548, 0.02062033675611019, -0.01689678244292736, 0.0354410819709301, 0.018901603296399117, 0.013266082853078842, 0.005794900469481945, 0.263570100069046, 0.015493486076593399, -0.013869833201169968, 0.02540765143930912, -0.01833352819085121, -0.006057239603251219, -0.0032085420098155737, -0.03513593226671219, -0.003236078191548586, 0.004629946313798428, 0.009108788333833218, -0.023051651194691658, -0.004151524044573307, -0.03742844611406326, 0.0022533664014190435, -0.01825842447578907, 0.0017117672832682729, 0.009571441449224949, 0.004658719524741173, -0.03654173016548157, -0.005738452076911926, 0.008396277204155922, 0.010747765190899372, -0.00039301038486883044, 0.004735198337584734, -0.007904361933469772, -0.01676197722554207, -0.02063269354403019, 0.033704325556755066, 0.009046853519976139, -0.06097785383462906, 0.010877126827836037, 0.005845207720994949, -0.02327750064432621, -0.005545967258512974, 0.010013533756136894, -0.005862153600901365, 0.00020681762543972582, -0.010481394827365875, -0.010407778434455395, -0.0012392448261380196, -0.0045167552307248116, -0.007881044410169125, -0.025964712724089622, -0.021803105250000954, -0.006348819006234407, -0.014295864850282669, -0.010702255181968212, 0.013674086891114712, 0.012365146540105343, 0.0035964976996183395, -0.015469741076231003, 0.0014134908560663462, -0.021115342155098915, 0.0018183999927714467, 0.015159988775849342, 0.006818752270191908, 0.01293661817908287, -0.004194397944957018, 0.005621636752039194, 0.0034065863583236933, 0.005182011518627405, 0.005288601387292147, 0.0028868597000837326, 0.02525697834789753, 0.00047765052295289934, -0.0038732041139155626, -0.013276292011141777, -0.02262951247394085, 0.0020099931862205267, 0.005472411401569843, 0.0008875164203345776, -0.12726417183876038, 0.018902428448200226, 0.004179652314633131, -0.0146397789940238, 0.0041646952740848064, 0.0012118781451135874, 0.00905204564332962, 0.00016267462342511863, 0.034712858498096466, 0.01649220660328865, 0.004458303097635508, 0.017856404185295105, 0.00827434565871954, -0.02606312744319439, 0.0059824916534125805, 0.010369418188929558, 0.041658684611320496, 0.017321186140179634, -0.028088023886084557, -0.02034049853682518, -0.001960211666300893, 0.01352631114423275, -0.0074627636931836605, 0.006226781755685806, -0.0004719165444839746, -0.0006160522461868823, -0.028242485597729683, -0.0035479941871017218, -0.0039598653092980385, 0.02385363169014454, -0.008917703293263912, -0.007783094886690378, -0.01507992297410965, -0.009022088721394539, -0.0025781714357435703, -0.011904158629477024, 0.012734801508486271, 0.019198037683963776, -0.01917995698750019, -0.010792753659188747, 0.006569750141352415, -0.005208136513829231, 0.005144799593836069, -0.01665710285305977, -0.007480567786842585, 0.00083803606685251, -0.011904120445251465, -0.09836255013942719, 0.003298075869679451, -0.003212673356756568, -0.016263434663414955, -0.008106068708002567, 0.015736116096377373, 0.014443574473261833, -0.005710083991289139, 0.005418268032371998, 0.0012029327917844057, 0.01682349108159542, 0.02263595536351204, 0.02220844477415085, -0.0030905287712812424, 0.003855783259496093, -0.027397746220231056, -6.730664608767256e-05, 0.019211163744330406, -0.006181809119880199, -0.01132142636924982, 0.0033223661594092846, 0.012859143316745758, -0.04025770723819733, -0.015056321397423744, 0.004215999972075224, -0.017071865499019623, -0.009144884534180164, -0.015540077351033688, 0.010416069999337196, -0.012059046886861324, -0.004272725898772478, 0.3369620442390442, -0.012361652217805386, 0.009123190306127071, -0.0035102127585560083, -0.01263075415045023, -0.005609071347862482, 0.014751040376722813, 0.0009677172638475895, 0.05564720183610916, 0.0025043305940926075, -0.013282651081681252, -0.02288430742919445, -0.0030472641810774803, 0.0065215774811804295, -0.03137397766113281, 0.022681884467601776, 0.0012484792387112975, -0.006067607086151838, -0.003286242950707674, -0.016158968210220337, 0.003837715368717909, -0.009724478237330914, 0.011182865127921104, -0.01669393479824066, 0.015052670612931252, 0.020387757569551468, 0.005549508612602949, -0.012980052269995213, 0.0024751308374106884, -0.01872844062745571, -0.013447375036776066, 0.009727680124342442, 0.008015957660973072, 0.015933414921164513, 0.015197482891380787, -0.01281366590410471, 0.010467791929841042, 0.0028484356589615345, 0.01977805234491825, 0.006998488213866949, 0.022540569305419922, 0.018094288185238838, 0.0666310265660286, 0.0045472364872694016, 0.016333643347024918, 0.014887586236000061, 0.0009581457125023007, -0.005019848700612783, 0.025787774473428726, 0.006752736866474152, -0.012283023446798325, 0.023345068097114563, 0.0121966153383255, -0.009453222155570984, 0.0033876849338412285, 0.002886032685637474, 0.010863342322409153, -0.01913195475935936, -0.011464511975646019, 0.006265547126531601, -0.004750953521579504, -0.0019060721388086677, -0.015133845619857311, -0.009565899148583412, 0.003944799304008484, -0.015879763290286064, 0.01596292480826378, -0.007246738765388727, -0.020950688049197197, -0.012268517166376114, -0.02433461882174015, 0.007083186414092779, -0.026472391560673714, 0.007763929199427366, 0.011249841190874577, 0.0037414454855024815, -0.025174640119075775, 0.008059640415012836, -0.006499662529677153, -0.013014807365834713, -0.009792481549084187, -0.011616086587309837, -0.005314661655575037, -0.01759163662791252, -0.006189344916492701, -0.002486241515725851, -0.03288041427731514, -0.03192947432398796, 0.006475719157606363, 0.018522441387176514, -0.009255980141460896, -0.012199732474982738, -0.015475400723516941, 0.006216919980943203, -0.007810049690306187, 0.01452222466468811, -0.00530023779720068, -0.0065565286204218864, -0.00945150014013052, 0.014421917498111725, 0.028107350692152977, 0.00508491788059473, 0.008919806219637394, 0.00022691916092298925, 0.0019651674665510654, -0.011769290082156658, -0.011252065189182758, 0.005651228595525026, -0.007959933951497078, 0.00272437185049057, 0.015344483777880669, 0.002161229494959116, 0.009469950571656227, -0.0036086325999349356, -0.009222696535289288, 0.007038078736513853, 0.0029803744982928038, -0.023939546197652817, -0.43966394662857056, 0.020679675042629242, -0.0006769430474378169, 0.01706642284989357, -0.01839337684214115, -0.004857504274696112, 0.02055037021636963, 0.0028386381454765797, 0.015073966234922409, 0.0031297372188419104, -0.02546306513249874, -0.008448359556496143, -0.13353441655635834, 0.020133310928940773, -0.005974474363029003, 0.009472062811255455, -0.018484238535165787, -0.049293216317892075, -0.00501802796497941, 0.030518725514411926, -0.010765121318399906, 0.011608846485614777, 0.01771598681807518, 0.015667730942368507, 0.01946093514561653, -0.0006708108121529222, -0.002259631874039769, 0.0048065707087516785, 0.008751987479627132, -0.003858563955873251, -0.000276548060355708, -0.011472363956272602, 0.024431906640529633, -0.0007201014086604118, -0.007195251528173685, -0.019575392827391624, 0.0038918298669159412, 0.0005770757561549544, 0.017144663259387016, -0.022542400285601616, -0.00516725517809391, 0.006870722863823175, 0.009057307615876198, 0.019846752285957336, -0.02490178868174553, -0.009911500848829746, -0.036297742277383804, -0.15085369348526, 0.008032470941543579, 0.0044517857022583485, 0.02317938767373562, 0.004161266144365072, -0.0011649380903691053, 0.0033345315605401993, -0.01748519018292427, 0.0028167949058115482, -0.003142379689961672, -0.002474274253472686, 0.02645733766257763, -0.010450823232531548, -0.01894858293235302, -0.0023506316356360912, 0.023196758702397346, 0.0040303668938577175, -0.009569864720106125, 0.0049675991758704185, -0.030698459595441818, -0.0004981535603292286, -0.009559322148561478, -0.022259825840592384, -0.014162871986627579, 0.0010068019619211555, -0.01268716435879469, 0.00800398737192154, 0.017168493941426277, 0.023802559822797775, 0.0043264115229249, 0.0015718027716502547, -0.00918525643646717, -0.002797643654048443, -0.027845127508044243, -0.01416602823883295, 0.029674552381038666, 0.015137976966798306, 0.005623066797852516, -0.008978047408163548, -0.008050479926168919, 0.021296046674251556, -0.0017752323765307665, 0.000736291753128171, 0.0005452001350931823, 0.008078766986727715, 0.010231397114694118, -0.003165846923366189, -0.0076051210053265095, 0.01453905738890171, -0.01371407974511385, 0.003918827977031469, -0.030010702088475227, -0.0069785634987056255, -0.010936976410448551, 0.00013863554340787232, 0.008834341540932655, -0.026449214667081833, -0.007220170926302671, -0.008527402766048908, 0.17051738500595093, 0.004614910110831261, -0.005210649687796831, -0.0013190996833145618, 0.004119134973734617, 0.011394820176064968, 0.007205550093203783, 0.017397552728652954, 0.10435468703508377, 4.4626518501900136e-05, -0.006651582662016153, 0.008833104744553566, -0.0087201576679945, 0.006388853304088116, -0.002573351375758648, 0.02692006714642048, -0.002071782248094678, 0.01578201726078987, -0.010066008195281029, 0.002425156533718109, 0.02186296321451664, -0.019258635118603706, 0.014925463125109673, -0.005661596078425646, 0.00017077445227187127, -0.0023505419958382845, -0.014679128304123878, -0.02553863264620304, 0.00949908047914505, -0.022416332736611366, -0.005413079168647528, 0.012191371992230415, -0.0024616015143692493, 0.06135515496134758, -0.009695522487163544, 0.007317144423723221, 0.026295650750398636, -0.002844767877832055, -0.0025774252135306597, -0.030404355376958847, -0.014790286310017109, -0.009962803684175014, -0.03695661574602127, 0.028453832492232323, -0.00452220905572176, -0.0029461898375302553, -0.006691392511129379, -0.007751360069960356, -0.0005985025782138109, -0.005986648146063089, -0.002986188745126128, -0.018548984080553055, -0.011098836548626423, 0.007023291662335396, -0.009728091768920422, -0.00039898656541481614, -0.04646572098135948, -0.0008864692645147443, 0.015405922196805477, -0.0061421566642820835, 0.0016143537359312177, 0.0062434375286102295, -0.009661217220127583, 0.0008592577651143074, 0.00023273637634702027, -0.0067783077247440815, 0.013595854863524437, -0.01635022833943367, -0.012279335409402847, -0.016765465959906578, -0.011593619361519814, 0.009482655674219131, -0.15158773958683014, -0.005465535447001457, -0.002380920806899667, -0.019285794347524643, -0.018937023356556892, -0.014394961297512054, 0.017416033893823624, 0.003909922204911709, 0.004340807441622019, 0.010954532772302628, -0.007657262496650219, 0.1208495944738388, -0.007315537892282009, -0.0018151404801756144, -0.0033840227406471968, -0.014714956283569336, 0.013190235011279583, 0.010578785091638565, -0.006197648588567972, -0.018092697486281395, -0.015300048515200615, 0.11893020570278168, 0.005412606988102198, 0.017531437799334526, 0.009198318235576153, -0.012129398062825203, -0.0007818111917003989, -1.1839984836115036e-05, 0.00927791092544794, -0.001361721777357161, -0.007480500731617212, 0.012923954986035824, 0.00266449898481369]}, {"text": "Can you tell me more about Technical intelligence and Emotional intelligence and how this relates with a professional development framework?", "label": "LOW", "embedding": [0.031014323234558105, -0.0015219979686662555, -0.008036484941840172, -0.012464109808206558, -0.0220953356474638, -0.0028410330414772034, 0.01745465025305748, -0.002228877041488886, 0.008885023184120655, -0.0708170011639595, -0.01518796756863594, -0.002861029002815485, -0.009421558119356632, -0.0013572081224992871, -0.03742077946662903, 0.004648472648113966, 0.01704036258161068, 0.006063147913664579, 0.002390355570241809, -0.0165437962859869, 0.0031266314908862114, -0.018081873655319214, 0.03483756259083748, -0.013540784828364849, 0.005275946110486984, -0.003553795162588358, 0.0217877384275198, 0.005859755910933018, -0.008967870846390724, 0.023120926693081856, -0.00136216776445508, 0.017621075734496117, -0.010564838536083698, 0.001798236626200378, -0.022136176005005836, 0.006633084267377853, 0.0009735809289850295, -0.016165511682629585, -0.01860644854605198, 0.0008015729254111648, -0.03126906231045723, 0.015357688069343567, 0.04310137778520584, -0.013273980468511581, 0.002769188955426216, -0.026807691901922226, 0.004310864023864269, -0.001664280309341848, -0.009683076292276382, 0.007445706985890865, -0.0037375790998339653, 0.0007010286208242178, -0.013923252001404762, 0.003698185086250305, 0.009967401623725891, 0.00942304078489542, -0.008960065431892872, 0.0013587767025455832, -0.032204095274209976, 0.01704459637403488, -0.00325791840441525, -0.015904024243354797, 0.021513111889362335, 0.009020678699016571, -0.007096382789313793, 0.0018781539984047413, 0.011505831964313984, -0.005177977494895458, -0.018963217735290527, -0.011045591905713081, 0.004621523432433605, 0.004458126612007618, 0.01821381039917469, 0.059348102658987045, 0.02290257252752781, -0.013944940641522408, 0.03622931241989136, 0.00791071355342865, 0.0021161832846701145, -0.01078722346574068, 0.008082901127636433, 0.0018913834355771542, -0.005848676431924105, -0.009815835393965244, -0.0035127773880958557, -0.00622559105977416, 0.011654782108962536, 0.006792553700506687, -0.01219211332499981, 0.01929967850446701, 0.010011824779212475, -0.004175620153546333, -0.022934969514608383, -0.004409564193338156, -0.005393462721258402, 0.014104577712714672, 0.00027879528352059424, 0.009521723724901676, 0.41536086797714233, -0.005624375306069851, -0.001505775609984994, 0.010526688769459724, 0.005513096693903208, -0.0038337549194693565, 0.028534870594739914, -0.026055922731757164, -0.008728070184588432, -0.02282865345478058, -0.008146737702190876, 0.03069387935101986, 0.03600211814045906, -0.012335298582911491, 0.004915269557386637, -0.005332157015800476, -0.0024809865280985832, -0.019853517413139343, -0.005145625211298466, -0.0023139696568250656, 0.010794163681566715, -0.010105527006089687, 0.005121386144310236, -0.00623070215806365, 0.07146872580051422, 0.015866827219724655, -0.01426840852946043, -0.0112729137763381, -0.007935892790555954, -0.011000208556652069, 0.0051419674418866634, -0.01519680954515934, -0.035130713135004044, -0.013999907299876213, -0.004459299612790346, -0.002151795895770192, 0.003585939761251211, -0.000775535823777318, -0.012474666349589825, -0.002773995976895094, -0.06196281686425209, 0.003838149830698967, 0.005834728479385376, -0.007512412965297699, 0.00902410689741373, -0.021340323612093925, -0.01916937716305256, 0.1917467564344406, -0.009062331169843674, 0.004138245712965727, -0.006178683135658503, 0.006449496373534203, 0.006814718246459961, -0.011612528003752232, -0.007678883615881205, 0.010019106790423393, 0.0015921280719339848, -0.013454328291118145, 0.01052333228290081, 0.011318000964820385, 0.0008854297921061516, 0.0024637372698634863, -0.061703745275735855, 0.0024882773868739605, -0.009498723782598972, 0.03328869491815567, 0.004698484670370817, -0.013040506280958652, 0.005718105472624302, -0.04768628627061844, 0.012590442784130573, 0.023207755759358406, -0.0071741677820682526, -0.006329873111099005, -0.15688270330429077, 0.01694559119641781, 0.009998644702136517, 0.005335027351975441, -0.0007212301134131849, 0.015064983628690243, -0.0004781639436259866, -0.018242469057440758, -0.020647022873163223, 0.019538594409823418, 0.004780765622854233, -0.005035890731960535, -0.01985616609454155, 0.010326588526368141, 0.001337299938313663, -0.0008932487107813358, 0.0028566131368279457, 0.004548371769487858, 0.014051945880055428, -0.018220333382487297, 0.00017056200886145234, 0.018054883927106857, 0.009805782698094845, 0.012035543099045753, 0.006125138606876135, -0.013799920678138733, -0.006035906728357077, 0.002744923112913966, -0.004603380803018808, -0.002649356611073017, -0.01341918297111988, 0.013785531744360924, 0.02421870455145836, 0.004861047025769949, -0.005662252195179462, 0.018602853640913963, 0.01594666577875614, -0.004272010177373886, -0.008190959692001343, 0.01757979765534401, -0.007015337701886892, 0.031171469017863274, 0.036265596747398376, -0.022333582863211632, -0.0009494042606092989, 0.0026122990529984236, 0.0050719804130494595, 0.0013329637004062533, -0.011683999560773373, 0.005394519306719303, -0.02040042355656624, -0.012790040113031864, 0.0016430914402008057, -0.0006346488953568041, 0.010924837552011013, -0.006291338708251715, -0.00016312267689500004, -0.006517800502479076, 0.006903927773237228, 0.010487519204616547, 0.002259851898998022, -0.008831791579723358, 0.0018538826843723655, -0.0057938480749726295, -0.01697409525513649, 0.01033873576670885, 0.007979677990078926, 0.0190977081656456, -0.00718597462400794, 0.0015534741105511785, 0.02564927004277706, -0.009150991216301918, -1.797179356799461e-05, 0.009681285358965397, 0.02352507784962654, 0.015285346657037735, 0.0001224071893375367, 0.006605369970202446, 0.0459592305123806, 0.016686515882611275, -0.011271223425865173, -0.016699930652976036, -0.03447325900197029, -0.007414347492158413, -0.003279824974015355, -0.08621681481599808, -0.014770956709980965, 0.0006918054423294961, 0.007307098247110844, -0.20315243303775787, 0.00801724661141634, 0.011813152581453323, 0.006430685520172119, -0.00832861103117466, 0.017946599051356316, -0.00177041906863451, -0.008722888305783272, 0.0004959347425028682, -0.028595324605703354, 0.0006601724890060723, -0.033415164798498154, 0.012231790460646152, -0.004945865832269192, 0.01681157574057579, -0.012905354611575603, -0.009818014688789845, -0.011576518416404724, -0.015560286119580269, -0.01627904176712036, 0.011119624599814415, -0.01886986382305622, 0.02206200361251831, -0.02527422457933426, 0.03287818655371666, 0.012142767198383808, 0.00892824586480856, 0.003417357336729765, 0.26573941111564636, 0.012010597623884678, -0.0108742481097579, 0.015720410272479057, -0.018937842920422554, -0.00423048110678792, -0.005261201877146959, -0.04221988841891289, 0.0043108584359288216, -0.000522670685313642, 0.013103111647069454, -0.011826681904494762, -0.001960186753422022, -0.036638859659433365, -0.002060017315670848, -0.017703497782349586, 0.0007523348904214799, 0.004760099574923515, 0.0032471215818077326, -0.037283726036548615, -0.009822346270084381, 0.01158315222710371, 0.0070524197071790695, -0.0013390614185482264, 0.009516422636806965, -0.006759187206625938, -0.02127746120095253, -0.01977335289120674, 0.03174705058336258, 0.0032567058224231005, -0.05815654620528221, 0.0059072659350931644, -0.0006257620989345014, -0.020263494923710823, -0.00789584219455719, 0.00533456914126873, -0.005046567413955927, -0.0027257006149739027, -0.017936155200004578, -0.02003198303282261, -0.0034913504496216774, -0.009800565429031849, 9.692277672002092e-05, -0.023397274315357208, -0.015008475631475449, 0.0033766240812838078, -0.005287639796733856, -0.0074705518782138824, 0.016991673037409782, 0.01242284756153822, 0.0016749342903494835, -0.01182407233864069, -0.0022531275171786547, -0.022644061595201492, 0.015316071920096874, 0.009701925329864025, 0.009779701940715313, 0.016724981367588043, -0.005749957635998726, 0.008108499459922314, -0.0015313595067709684, 9.776625665836036e-05, -0.0005194341065362096, -0.005580785218626261, 0.03199845179915428, 0.0074791633524000645, -0.005429414100944996, -0.015546698123216629, -0.017142733559012413, 0.009149173274636269, 0.011826865375041962, 0.00021987651416566223, -0.13962067663669586, 0.01504432037472725, 0.006534627638757229, -0.010522185824811459, -0.005209197290241718, -0.0014837898779660463, 0.00655137887224555, -0.0001003618017421104, 0.037407953292131424, 0.018543245270848274, 0.003779199905693531, 0.02081911452114582, 0.007095718756318092, -0.02625686675310135, 0.006213280372321606, 0.0009619317133910954, 0.035118214786052704, 0.011860662139952183, -0.01837824657559395, -0.016600070521235466, 0.00581136392429471, 0.01588510349392891, -0.004265348892658949, 0.007429823279380798, 0.00011786404502345249, 0.00908804964274168, -0.023616129532456398, -0.008413361385464668, 0.002824393566697836, 0.024342870339751244, -0.008050612173974514, -0.009041338227689266, -0.0015566368820145726, -0.011086486279964447, 0.006563240196555853, -0.00943272840231657, 0.016253041103482246, 0.01896798424422741, -0.024715110659599304, -0.00886491872370243, 0.003934646025300026, -0.003926504869014025, 0.009049619548022747, -0.014016241766512394, -0.00921617541462183, 0.0023389551788568497, -0.014498741365969181, -0.10229594260454178, 0.000524716975633055, -0.004776425659656525, -0.01715332455933094, -0.01404751930385828, 0.012279733084142208, 0.007473940495401621, -0.00026003533275797963, 0.004244427662342787, -0.007519809063524008, 0.019422564655542374, 0.02410094253718853, 0.018304379656910896, 0.0014021552633494139, 0.008921762928366661, -0.0210694819688797, -0.006901034619659185, 0.018718577921390533, 0.01389568392187357, -0.013201803900301456, 0.0026300535537302494, 0.019196458160877228, -0.048730991780757904, -0.009874911978840828, 0.006656805984675884, -0.02118648961186409, -0.013852400705218315, -0.018078234046697617, 0.016281520947813988, -0.011325058527290821, -0.0037568260449916124, 0.3449825048446655, -0.01063915528357029, 0.0115573201328516, -0.003756436752155423, -0.012679273262619972, -0.006395605392754078, 0.013338776305317879, -0.0015235032187774777, 0.050420910120010376, 0.0030866446904838085, -0.015923993661999702, -0.025214731693267822, 0.0003806738823186606, 0.010970949195325375, -0.03428088128566742, 0.01892700605094433, 0.007612967863678932, -0.0028268315363675356, -0.0034436604473739862, -0.007117799483239651, 0.0007522105006501079, -0.009045873768627644, 0.009777569212019444, -0.021894484758377075, 0.01743270829319954, 0.015432372689247131, 0.0012928372016176581, -0.0027447212487459183, -0.0013694020453840494, -0.012339852750301361, -0.021465342491865158, 0.01538163609802723, -0.00034138254704885185, 0.016282808035612106, 0.0033576490823179483, -0.014377559535205364, 0.010911591351032257, 0.002433640882372856, 0.01843658648431301, 0.0069731008261442184, 0.022101489827036858, 0.008032282814383507, 0.06679974496364594, 0.0009571866248734295, 0.01402230653911829, 0.010972562246024609, -0.0005214696866460145, -0.011891070753335953, 0.030429517850279808, 0.013457424007356167, -0.010821722447872162, 0.021963974460959435, 0.010622370056807995, -0.006938529200851917, 0.0013799016596749425, 0.0029281640890985727, 0.01066917460411787, -0.012680737301707268, -0.0008370672003366053, 0.004319929983466864, -0.012787465937435627, -0.004831480793654919, -0.011068454943597317, -0.011256158351898193, 0.007221827283501625, -0.009384683333337307, 0.00726923206821084, -0.008425936102867126, -0.014464001171290874, -0.013069899752736092, -0.013629697263240814, 0.0005193314864300191, -0.011790221557021141, 0.0074813487008214, 0.021228700876235962, 0.004324258770793676, -0.023946555331349373, -0.0006952332914806902, -0.0010615477804094553, -0.01592981442809105, -0.016495604068040848, -0.016190702095627785, 0.0013169493759050965, -0.015325325541198254, -0.00901324488222599, -0.0013451174600049853, -0.027872737497091293, -0.03323185071349144, -0.0017871264135465026, 0.007261724676936865, -0.004549163393676281, -0.012507556937634945, -0.016296960413455963, 0.0016480160411447287, -0.0058716363273561, 0.013910220935940742, -0.005368372425436974, -0.007320409175008535, -0.0025927640963345766, 0.008285179734230042, 0.018566682934761047, 0.005756076890975237, 0.01067410223186016, 0.0026258272118866444, -0.001027804333716631, -0.010046696290373802, -0.007144307717680931, 0.011168425902724266, -0.0007767776260152459, 0.0005905490834265947, 0.014482997357845306, -0.0026653949171304703, 0.003186283865943551, -0.00933444406837225, -0.015461956150829792, 0.004272210877388716, 0.008724236860871315, -0.025053979828953743, -0.4184871315956116, 0.019148988649249077, 0.0023740578908473253, 0.008010498248040676, -0.023692889139056206, -0.00903136283159256, 0.0262152012437582, 0.0022172522731125355, 0.016703657805919647, 0.002965023973956704, -0.021715154871344566, -0.0076860287226736546, -0.13762003183364868, 0.01692449487745762, -0.013658641837537289, 0.0036100265569984913, -0.018806884065270424, -0.05227793753147125, -0.005720393266528845, 0.03744465485215187, -0.010442163795232773, 0.012858090922236443, 0.015201332047581673, 0.009457375854253769, 0.02055436186492443, -0.00978777650743723, 0.001234103343449533, 0.0034019737504422665, 0.01419228408485651, -0.00936818402260542, 0.006349352654069662, -0.007416198495775461, 0.018127385526895523, -0.00326730078086257, -0.010617382824420929, -0.01226399838924408, -0.0006138188182376325, 0.0021203146316111088, 0.011807604692876339, -0.01944396272301674, 0.00032229055068455637, 0.011465719901025295, 0.010290193371474743, 0.02398820035159588, -0.02592272311449051, -0.018722159788012505, -0.04114585369825363, -0.1513424962759018, 0.005396108143031597, 0.0022953874431550503, 0.02305365726351738, 0.005504682660102844, -0.011037830263376236, 0.00172900827601552, -0.010228374041616917, 0.013424606993794441, -0.0003807679167948663, -0.0005206724163144827, 0.02502565085887909, -0.029272159561514854, -0.019290564581751823, -0.003089101053774357, 0.022766489535570145, -0.0049619064666330814, -0.006485877558588982, 0.014409398660063744, -0.02788504771888256, 0.003265391569584608, -0.01105640921741724, -0.020260700955986977, -0.01321484800428152, -0.005194159224629402, -0.008571185171604156, 0.0025250939652323723, 0.018061140552163124, 0.019533058628439903, 0.007799234241247177, 0.004368479363620281, -0.004036791156977415, 0.0008773068548180163, -0.03373540937900543, -0.008410648442804813, 0.03401589021086693, 0.02693028375506401, 0.011160292662680149, -0.005625636782497168, -0.005714815575629473, 0.010749259032309055, -0.005445440765470266, 0.004832047503441572, -0.0034571937285363674, 0.006297213025391102, 0.010857214219868183, -0.012327502481639385, -0.011822947300970554, 0.005743847228586674, -0.010564601048827171, 0.006314700469374657, -0.02037966437637806, -0.010959884151816368, -0.006900063715875149, -0.003810408990830183, 0.0026319234166294336, -0.030233515426516533, -0.009373251348733902, -0.011388685554265976, 0.16543476283550262, -0.0024241742212325335, -0.0064384909346699715, -0.0006798575050197542, -0.006015910767018795, 0.008712519891560078, -0.005445091985166073, 0.013535578735172749, 0.10756222903728485, 0.01191017683595419, -0.01024632342159748, 0.005734906066209078, -0.00998990423977375, -0.001359971472993493, -0.0044629438780248165, 0.02800113521516323, -0.002560678170993924, 0.011641948483884335, -0.009084301069378853, 0.004688093438744545, 0.018291471526026726, -0.013480388559401035, 0.01550555694848299, -0.007163228467106819, -0.0008659834857098758, -0.004038266837596893, -0.02472793497145176, -0.024690348654985428, 0.00921967439353466, -0.02563497982919216, -0.004946927074342966, 0.011164621450006962, -0.001652656588703394, 0.060918308794498444, -0.0069648525677621365, 0.008948637172579765, 0.018534034490585327, -0.006711648777127266, -0.0032838378101587296, -0.029369162395596504, -0.01117821503430605, 0.0035590557381510735, -0.03478163480758667, 0.030193718150258064, -0.010811342857778072, -6.59747383906506e-05, -0.009558985009789467, -0.017509788274765015, 0.0007590341265313327, -0.012171239592134953, -0.004166208673268557, -0.02340179681777954, -0.016823701560497284, 0.00832692813128233, -0.00845895241945982, -0.0007574179326184094, -0.041695524007081985, -0.008701037615537643, 0.01624452881515026, -0.0050758603028953075, -0.003287792904302478, 0.0056740413419902325, -0.007296436466276646, 0.0015486174961552024, -0.001228991663083434, -0.00535390991717577, 0.014322846196591854, -0.016697663813829422, -0.0067323194816708565, -0.012912362813949585, -0.009215925820171833, 0.010282520204782486, -0.15390059351921082, -0.0003103697672486305, 0.007212779484689236, -0.02534385770559311, -0.017733611166477203, -0.015113764442503452, 0.021177394315600395, 0.013452292419970036, 0.0017663441831246018, 0.013526178896427155, -0.011817257851362228, 0.13391755521297455, -0.004168080631643534, -0.00020885617414023727, 0.0014310356928035617, -0.013080905191600323, 0.005788265727460384, 0.01764259673655033, 0.00011756463936762884, -0.02552700601518154, -0.01475684903562069, 0.1112988144159317, -0.0008491009357385337, 0.021861189976334572, 0.012497208081185818, -0.011564680375158787, 0.003256548661738634, 0.00865832157433033, 0.013608335517346859, -0.0018118622247129679, -0.014279314316809177, 0.008667757734656334, 0.007192390505224466]}], "HIGH": [{"text": "Given the task definition and input, reply with output. In this task, you are given inputs k, i, and A, where k and i are integers and A is a list. You need to find every ith element of A starting from the kth element. The process stops when the position of the next ith element exceeds the length of the list.\n\n4, 4, ['A', '6999', 'J', 'i', '9475', 'Z', '665', 'e', 'I', '5073', 'K', 'C', 'g', '4719', '8661', 'u', '3285', '869', 'W', 'z', '939', 'Z', 'u', '61', '9301']", "label": "HIGH", "embedding": [0.03936084359884262, 0.004960429389029741, 0.010884394869208336, -0.009949573315680027, -0.022208506241440773, -0.010986450128257275, 0.00836564227938652, -0.0008468471933156252, 0.0013064802624285221, -0.07963630557060242, -0.018253762274980545, 0.011012755334377289, 0.006102238316088915, -0.0018839676631614566, -0.03264480456709862, 0.01129607018083334, 0.010702578350901604, 0.01578068733215332, -0.01187190692871809, -0.025896240025758743, 0.009839043021202087, -0.0006306213326752186, 0.048844777047634125, -0.016516944393515587, 0.012465544044971466, -0.008741123601794243, 0.014340304769575596, 0.00645514065399766, -0.006283355411142111, 0.029996050521731377, 0.0031849632505327463, 0.013444977812469006, -0.00013970461441203952, -0.0089696254581213, -0.018163641914725304, 0.0010431997943669558, -0.004424113314598799, -0.023243695497512817, -0.009009333327412605, -0.005546825472265482, -0.022319605574011803, 0.0012824500445276499, 0.04671848565340042, -0.010825157165527344, -0.0033297904301434755, -0.021021058782935143, -0.0010873125866055489, 0.0063275061547756195, 0.010459786280989647, 0.011031822301447392, 0.00026336757582612336, -0.003384066978469491, 0.006018682383000851, 0.009718145243823528, 0.001915810164064169, 0.00544630317017436, 0.006740074139088392, 0.003973191604018211, -0.01290052942931652, 0.0049829124473035336, -0.0031889649108052254, -0.001430980977602303, 0.02290426939725876, 0.0039564985781908035, -0.00905847828835249, 0.007639293093234301, 0.009555244818329811, 0.005036998074501753, -0.007120318245142698, -0.001935538835823536, -0.007482168730348349, 0.00017691102402750403, 0.016627054661512375, 0.0644952729344368, 0.022793062031269073, -0.013834972865879536, 0.02885482646524906, 0.005051023326814175, 0.00596728315576911, -0.0014123653527349234, 0.01740814559161663, 0.0070414855144917965, -0.013410323299467564, 0.003622644115239382, -0.0008463640115223825, -0.007275923155248165, 0.01869695633649826, -0.012905208393931389, -0.006496801506727934, 0.02716889977455139, 0.004317826591432095, -0.01482110470533371, -0.0024611696135252714, -0.016043681651353836, -0.009304000996053219, 0.00626668706536293, 0.0019225910073146224, -0.00014837556227575988, 0.4216194450855255, 0.0010289490455761552, -0.015261863358318806, 0.0024914497043937445, -0.0009882462909445167, -0.00846034660935402, 0.026118027046322823, -0.016023825854063034, -0.004006599076092243, -0.02083694376051426, -0.00388469104655087, 0.0185383427888155, 0.03975048288702965, -0.017148137092590332, -0.0005576536059379578, -0.0019667961169034243, 0.002532905200496316, -0.006437913980334997, 0.013753487728536129, -0.0010790725937113166, 0.011133737862110138, -0.0044019753113389015, 0.009872311726212502, -0.006305181887000799, 0.05859629064798355, 0.002797671826556325, -0.0039597549475729465, -0.015599889680743217, 0.0017818030901253223, -0.009977473877370358, 0.014769199304282665, -0.009305709041655064, -0.04560498148202896, -0.006518700625747442, -0.012815791182219982, -0.0006603594520129263, 0.0038934447802603245, 0.018423015251755714, -0.023251745849847794, -0.0005787766422145069, -0.0328017883002758, 0.016289278864860535, 0.008691678754985332, -0.0195437204092741, 0.0003130900440737605, -0.02277565561234951, -0.012196211144328117, 0.19191108644008636, -0.006690266076475382, -0.015555373392999172, -0.0073830257169902325, 0.0030719484202563763, 0.0026709414087235928, -0.030871765688061714, -0.015262119472026825, 0.004777365829795599, -0.015471406280994415, -0.022903654724359512, 0.01358500774949789, -0.008406268432736397, -0.007219068240374327, -0.00018390346667729318, -0.05968745797872543, 0.009569830261170864, -0.0070080687291920185, 0.03740732744336128, -0.0029267980717122555, -0.007676996290683746, 0.005510649178177118, -0.05134394392371178, 0.0038342576008290052, 0.028197821229696274, 0.012990337796509266, -0.006483576726168394, -0.1502733677625656, 0.009386279620230198, 0.018506251275539398, 0.0019196136854588985, 0.007242754101753235, -0.008244849741458893, 0.00370592949911952, -0.013957678340375423, -0.022089466452598572, -0.0007973433239385486, 0.006162136327475309, 0.006295110564678907, -0.01961774192750454, 0.0050263055600225925, 0.012310645543038845, -0.0075538428500294685, 0.0060178437270224094, -0.003034628229215741, 0.007475488819181919, -0.01280107069760561, 0.003613191656768322, 0.012360608205199242, -0.0010607704753056169, -0.0016430641990154982, 0.018322955816984177, -0.0074977814219892025, -0.0099716791883111, -0.014068781398236752, -0.013911568559706211, 0.008109425194561481, -0.007256336975842714, 0.006560615263879299, 0.02645936608314514, 0.0065756868571043015, 6.768906587240053e-06, 0.0003280367818661034, 0.02296789176762104, -0.0028414155822247267, -0.0020477448124438524, -0.0022706224117428064, -0.012973068282008171, 0.02444504015147686, 0.04322231933474541, -0.019154727458953857, -0.0028426251374185085, -0.00444590300321579, -8.165942563209683e-05, -0.017248891294002533, -0.002214872743934393, 0.013453829102218151, 0.0018504854524508119, -0.0037645110860466957, 0.002221295377239585, 0.011606491170823574, 0.02190999872982502, -0.008500213734805584, 0.017444627359509468, -0.019191071391105652, 0.005209184251725674, -0.0007323648314923048, -0.00022824884217698127, -0.006114617455750704, -0.009999125264585018, -0.020216861739754677, -0.007795633282512426, 0.010790912434458733, -0.0021308534778654575, 0.027151063084602356, 0.001254949369467795, -0.0033276129979640245, 0.03376231715083122, -0.007643222343176603, -0.0037364831659942865, 0.011302698403596878, 0.028194185346364975, 0.004704784601926804, -0.015704870223999023, -0.005563069134950638, 0.060813404619693756, 0.016962295398116112, -0.015350498259067535, -0.014332890510559082, -0.018518997356295586, 0.008069961331784725, -0.011909016408026218, -0.08719787001609802, -0.03205578029155731, -0.000498932262416929, 0.009963742457330227, -0.1835893839597702, 0.015399877913296223, 0.004111625254154205, 0.0031678308732807636, -0.018320856615900993, 0.022072410210967064, 0.006164803169667721, -0.009753965772688389, -0.008036544546484947, -0.024500714614987373, 0.003334181848913431, -0.035069920122623444, 0.004809462931007147, -0.0008397891069762409, 0.018482914194464684, -0.0032052891328930855, 0.013375582173466682, -0.00474209152162075, 0.009467887692153454, -0.011388496495783329, -0.005628897808492184, -0.012542810291051865, 0.03169858455657959, -0.02127988263964653, 0.01795436441898346, -0.004106264095753431, 0.0081102279946208, -0.005263268016278744, 0.2550767660140991, 0.013236827217042446, 0.004257601220160723, 0.022803528234362602, -0.011363692581653595, -0.01130964607000351, 0.0013422329211607575, -0.039517633616924286, 0.02019924856722355, -0.0004069059796165675, 0.007504237350076437, -0.004528014454990625, -0.0002523957518860698, -0.03843925520777702, 0.009575575590133667, -0.02500067837536335, 0.010734829120337963, -0.016376914456486702, 0.007093385327607393, -0.05418330430984497, -0.007184312213212252, 0.034361258149147034, 0.002062082989141345, 0.011069443076848984, 0.0029053010512143373, -0.006179839838296175, -0.03408803418278694, -0.018601279705762863, 0.035261351615190506, 0.010920635424554348, -0.06537473946809769, 0.0055931019596755505, -0.005522231105715036, -0.018162859603762627, -0.004038489423692226, 0.0021376516669988632, 0.0031387051567435265, -0.007446600589901209, -0.012206677347421646, -0.019761284813284874, -0.002727699698880315, -0.0025969347916543484, 0.028290029615163803, -0.01933068223297596, -0.0011201385641470551, 0.014738470315933228, 0.007095866836607456, -0.0005412949831224978, 0.01594255492091179, -0.0053835101425647736, 0.005212331190705299, -0.020784936845302582, -0.001427987008355558, -0.013698970898985863, 0.015075813047587872, 0.01367645338177681, 0.005557122174650431, 0.011919993907213211, -0.0071760439313948154, 0.008511144667863846, 0.0021122414618730545, -0.01395352091640234, -0.002948831534013152, -0.008483248762786388, 0.019530050456523895, 0.016903676092624664, -0.008622846566140652, -0.012738088145852089, -0.02555747702717781, 0.007449566852301359, 0.0022003271151334047, 0.002254829043522477, -0.13526232540607452, 0.017318863421678543, 0.009603122249245644, -0.00169244350399822, 0.008263582363724709, -0.0038939886726439, -0.0023266214411705732, 0.0018572447588667274, 0.031452253460884094, 0.002659160876646638, 0.008328299969434738, 0.015047207474708557, 0.0028821821324527264, -0.02056613191962242, -0.0003775141085498035, -0.0007352853426709771, 0.04043716937303543, 0.01672397553920746, -0.008716871961951256, -0.014246668666601181, 0.011298108845949173, 0.013163551688194275, -0.001695158309303224, -0.008019805885851383, 0.004857487976551056, 0.00010543401003815234, -0.022565728053450584, -0.004384297877550125, 0.002398523734882474, 0.01416393555700779, 0.002402252983301878, -0.0020215159747749567, -0.00047832890413701534, -0.006883001886308193, 0.0006763517158105969, -0.009104389697313309, 0.02096431888639927, 0.010071718133985996, -0.014467940665781498, -0.00499754911288619, 0.003826758824288845, -0.0017142187571153045, -0.015627678483724594, -0.010912921279668808, 0.0017236933344975114, 0.016552839428186417, -0.018938913941383362, -0.09457460045814514, -0.006317329593002796, -0.0073526520282030106, -0.004131995607167482, -0.005957917310297489, 0.011062407866120338, 0.019314667209982872, 0.0030208840034902096, 0.013397927395999432, -0.013677677139639854, 0.020579319447278976, 0.001660458859987557, 0.005813354626297951, -0.001309455605223775, 0.018561450764536858, -0.024427568539977074, 0.006025216542184353, 0.014493962749838829, -0.009273543953895569, -0.01625319942831993, 0.002838427433744073, 0.001524181105196476, -0.05777807906270027, -0.011032734997570515, -0.0043924907222390175, -0.011203689500689507, -0.007034963462501764, -0.022326970472931862, 0.007517748046666384, -0.018603380769491196, -0.01760171726346016, 0.3546610474586487, -0.014473244547843933, 0.021307451650500298, -0.005501639097929001, -0.010185354389250278, 0.00289401994086802, 0.014385392889380455, -0.001251264475286007, 0.05813567340373993, -0.0006742792320437729, -0.031697601079940796, -0.02275996282696724, 0.02273329719901085, -0.0006605364033021033, -0.01696147583425045, 0.03293490782380104, -0.0029090233147144318, 0.005709681194275618, 0.0030047360341995955, 0.004667076747864485, -0.011398304253816605, -0.013994035311043262, 0.003006967017427087, -0.006476409733295441, 0.014306717552244663, 0.015366040170192719, 0.007222964894026518, -0.008776516653597355, -0.006987648084759712, -0.02308996021747589, -0.0178068820387125, 0.018976137042045593, 0.007892400957643986, 0.01368053536862135, -0.004157731309533119, -0.009139001369476318, 0.012274029664695263, -0.0022576379124075174, 0.013906385749578476, 0.01928875967860222, 0.002168530598282814, -0.00917446706444025, 0.06667107343673706, -0.0018509157234802842, 0.008392288349568844, 0.015417972579598427, -0.003766206093132496, -0.016462653875350952, 0.011023380793631077, 0.010873465798795223, -0.007668288890272379, 0.016743432730436325, 0.0009340632823295891, -0.01351835299283266, 0.012078463099896908, 0.005784535780549049, -0.00011868354340549558, 0.009350425563752651, 0.005177597980946302, 0.00674378452822566, 0.003692078869789839, -0.016296986490488052, -0.0041174571961164474, -0.01703045703470707, 0.0031046424992382526, -0.010170729830861092, 0.0042769042775034904, -0.015337473712861538, -0.026484332978725433, -0.003390588564798236, -0.009680925868451595, 0.009152295999228954, -0.01677653193473816, 0.013085207901895046, 0.0063329734839499, 0.008921336382627487, -0.0291915163397789, -0.00037468457594513893, -0.005671960301697254, -0.02039402723312378, -0.018743358552455902, -0.023636469617486, -0.00395815446972847, -0.01581772230565548, 0.0006578118773177266, -4.2006620788015425e-05, -0.02801230363547802, -0.020405447110533714, -0.004897511098533869, 0.002794623840600252, -0.00891365297138691, -0.00012599473120644689, -0.00179269106592983, 0.01069055125117302, 0.0009839986450970173, 0.010490295477211475, -0.00394849618896842, 0.004326449707150459, -0.013693070039153099, 0.0038861650973558426, 0.01918630860745907, 0.00431034853681922, -0.00043261743849143386, -0.0043674977496266365, -0.009010013192892075, 0.0011443649418652058, -0.005064596422016621, 0.009549861773848534, -0.010899635963141918, -0.014260319992899895, 0.010492007248103619, 0.007389375474303961, 0.025113821029663086, 0.0009922451572492719, -0.004688404966145754, 0.02125510945916176, -0.004907326772809029, -0.022828159853816032, -0.4106763005256653, 0.0009624669328331947, 0.002162334742024541, 0.005108162760734558, -0.024607786908745766, -0.0027700099162757397, 0.020536184310913086, 0.01457958109676838, 0.015264902263879776, -0.0003150237025693059, -0.019724583253264427, -0.024467933923006058, -0.14908258616924286, 0.017562473192811012, -0.017359424382448196, 0.0016329745994880795, -0.012661985121667385, -0.05746375396847725, 0.0017398666823282838, 0.02470633015036583, -0.0017936484655365348, 0.0146233681589365, 0.010732287541031837, 0.0009638788760639727, 0.007891727611422539, -0.01971980556845665, -0.0031268189195543528, -0.0023471659515053034, 0.02100738324224949, 0.0005486808368004858, 0.0017345957458019257, -0.01768360286951065, 0.027920471504330635, 0.0024211506824940443, 0.000794713560026139, -0.012767139822244644, -0.0016518112970516086, 0.0010233366629108787, 0.010355222038924694, -0.012649948708713055, -0.008298293687403202, -0.004844273440539837, 0.011602380312979221, 0.013450652360916138, -0.04342411458492279, -0.017518233507871628, -0.03382549434900284, -0.16081643104553223, 0.0030609024688601494, -0.008232283405959606, 0.011720278300344944, 0.015184284187853336, -0.016008615493774414, -0.010791744105517864, -0.01388404332101345, 0.005774691700935364, 0.0043488675728440285, 0.016289856284856796, 0.017903797328472137, -0.016114287078380585, -0.020522505044937134, -0.013857745565474033, 0.03514513000845909, 0.001914317486807704, -0.008671927265822887, 0.009891432709991932, -0.019322074949741364, -0.00975746102631092, -0.031102295964956284, -0.017578110098838806, -0.004308516625314951, -0.014928159303963184, -0.003921852447092533, 0.0024582818150520325, 0.016570501029491425, 0.0254583191126585, 0.0005926816374994814, -0.011501758359372616, 0.009450829587876797, 0.014571787789463997, -0.025311460718512535, -0.025732295587658882, 0.016916785389184952, 0.01869768649339676, 0.009637494571506977, -0.011183260008692741, 0.0028476701118052006, -0.0006686230772174895, -0.020090386271476746, -0.0073440177366137505, -0.005273529328405857, 0.00380596867762506, -0.009344622492790222, -0.01311800442636013, -0.004644298460334539, -0.007735313381999731, -0.023896032944321632, -0.0010176245123147964, -0.03570656478404999, -0.0009199180640280247, -0.004871138837188482, 0.006397840101271868, 0.007356174290180206, -0.020236928015947342, 0.0028093094006180763, -0.019400935620069504, 0.17115969955921173, 0.013738063164055347, -0.01126790326088667, -0.0014746224042028189, -0.0027289462741464376, 0.0013449349207803607, 0.0009019997669383883, 0.007593894377350807, 0.11592818796634674, -0.006458608899265528, -0.02373187616467476, 0.004359194543212652, -0.002981423633173108, -0.0037808550987392664, 0.014377713203430176, 0.031569626182317734, 0.006773822475224733, 0.010509909130632877, -0.004873139318078756, 0.007322981022298336, 0.010722823441028595, -0.015119090676307678, 0.00445092236623168, -0.010221099480986595, 0.0008434079354628921, -0.01433646958321333, -0.024143338203430176, -0.010065018199384212, 0.020273098722100258, -0.02127794921398163, 0.007860860787332058, -0.0117475101724267, -0.015656400471925735, 0.06478933990001678, -0.01720525696873665, 0.007457162253558636, 0.02333470620214939, 0.003483541775494814, -0.0003054356493521482, -0.019711121916770935, -0.003619087627157569, 0.002759739523753524, -0.03502533957362175, 0.03854754567146301, 0.0011133529478684068, -0.0019124256214126945, -0.02178957127034664, -0.0006705238483846188, 0.0028162216767668724, -0.005431427154690027, -0.0077752587385475636, -0.020884595811367035, -0.0036162119358778, 0.0005452646291814744, 0.004292683210223913, -0.004010598640888929, -0.048513732850551605, -0.0166843980550766, 0.006491517182439566, -0.008318985812366009, 0.0009974517161026597, -0.003758974839001894, 0.007637607865035534, 0.011934537440538406, 0.0013946620747447014, 0.004232460167258978, -0.0027880610432475805, -0.026783226057887077, 0.008055046200752258, -0.01688033528625965, -0.005945656914263964, 0.01213968638330698, -0.14579230546951294, 0.0016106016701087356, 0.009176061488687992, -0.03389428183436394, -0.009491744451224804, -0.01647457294166088, 0.012214168906211853, 0.011044532991945744, -0.001625920063816011, 0.01542758010327816, 0.0065412092953920364, 0.11922434717416763, -0.006091027054935694, -0.011191274039447308, -0.014461523853242397, 0.0009188756812363863, 0.014454929158091545, 0.006330654490739107, -0.0019365205662325025, -0.022011687979102135, -0.015032125636935234, 0.10641247779130936, 0.00689617358148098, 0.016103768721222878, 0.00841091014444828, -0.010464436374604702, -0.0005004954873584211, 0.006811348255723715, 0.0011270210379734635, -0.0020954441279172897, -0.0090244235470891, 0.002058306708931923, 0.018836405128240585]}, {"text": "There areFor the first 20 items below, please select the response that best indicates how much you agree with each statement. There are no right or wrong answers, so please respond as honestly as possible. If you are an independent contractor, when you see \u201corganization,\u201d bring to mind people in your profession\u2014that is, people who do the same kind of work as you. When you see \u201ccolleague,\u201d bring to mind people who you interact with during work, such as clients, vendors, or peers in your field.\n\nThe final six questions will help our research team see how people\u2019s levels of happiness at work relate to factors like gender, age, and profession.\n\nWhen you're done, you'll get your happiness-at-work score, along with ideas for cultivating more happiness at work.", "label": "HIGH", "embedding": [0.03175613284111023, -0.010096781887114048, 0.0017500001704320312, -0.005493293050676584, -0.019174888730049133, -0.00020082705304957926, 0.015080112963914871, -0.0033308258280158043, 0.006405678577721119, -0.07705485075712204, -0.007743712514638901, -0.010102702304720879, 0.003253225004300475, -0.002850498538464308, -0.0309202428907156, 0.010675614699721336, 0.014795010909438133, 0.002973101567476988, -0.0024277616757899523, -0.01619003899395466, 0.007799918297678232, -0.010625958442687988, 0.03819609805941582, -0.006104083266109228, 0.0061189099214971066, 0.008440854959189892, 0.011225886642932892, 0.011938313022255898, -0.006715971045196056, 0.02276512049138546, -0.005832379683852196, 0.016664771363139153, -0.008868979290127754, -0.007608966901898384, -0.015672871842980385, -0.0042152865789830685, -0.0019163436954841018, -0.028652705252170563, -0.021130874752998352, 0.0005330237909220159, -0.027907254174351692, 0.004144811071455479, 0.038249626755714417, -0.015295922756195068, -0.002437304239720106, -0.02218952775001526, 0.006226943340152502, 0.0016837343573570251, -0.0019155457848683, -0.001981049310415983, -0.0014532549539580941, -0.00380293489433825, -0.0013825582573190331, 0.00523544754832983, 0.009680255316197872, 0.0015915215481072664, 0.002083670115098357, -0.0016448639798909426, -0.01919686235487461, 0.008416595868766308, 0.0012552894186228514, -0.01469564437866211, 0.013257129117846489, 0.006690588779747486, -0.00640873471274972, 0.005447767209261656, 0.007312646601349115, -0.008993410505354404, -0.015881292521953583, -0.01198542769998312, -0.004479646682739258, 0.0018904127646237612, 0.01683744601905346, 0.06405206024646759, 0.013832864351570606, -0.011842906475067139, 0.02793155051767826, 0.010811162181198597, 0.005980649963021278, -0.002922595711424947, 0.00253299786709249, 0.0015150196850299835, -0.006785640027374029, 0.0023338927421718836, -0.006713520735502243, -0.010145807638764381, 0.012822319753468037, -0.01454739086329937, -0.019600708037614822, 0.029209014028310776, 0.0017942370614036918, 0.0030256917234510183, -0.018381163477897644, -0.002502317074686289, 0.004020423628389835, 0.017932767048478127, 0.003559186588972807, -0.0030133661348372698, 0.4127561151981354, 0.011227410286664963, -0.010622294619679451, -0.004701787605881691, 0.0015362495323643088, -0.003543118480592966, 0.020811306312680244, -0.01443496998399496, -0.015502756461501122, -0.027564754709601402, -0.010707668960094452, 0.028358155861496925, 0.03456835076212883, -0.014662140980362892, 0.0016251715132966638, -0.006960534490644932, 0.00569379236549139, -0.01647358201444149, 0.0048467242158949375, 0.004636499099433422, 0.016447994858026505, -0.015288696624338627, -0.0008173772948794067, -0.0037143102381378412, 0.07628293335437775, 0.009621660225093365, -0.00850955955684185, -0.012338005937635899, -0.002958386205136776, -0.005823811981827021, 0.006899700034409761, -0.0169090386480093, -0.03747919574379921, -0.012072362937033176, -0.012414318509399891, -0.003521836595609784, 0.00726955896243453, 0.0006230763392522931, -0.013041507452726364, -0.000566079921554774, -0.048955682665109634, -0.0004839679750148207, 0.0022681287955492735, -0.011053940281271935, 0.004505596589297056, -0.017731521278619766, -0.020202530547976494, 0.1933434158563614, -0.0059208134189248085, -0.008048593066632748, -0.006665385328233242, -0.01164077315479517, -0.003357223467901349, -0.012929231859743595, -0.0040426417253911495, 0.010121887549757957, -0.0006566215306520462, -0.009775863029062748, 0.016707196831703186, 0.0042637549340724945, -0.0016751924995332956, -0.0007042047218419611, -0.05864343419671059, -0.0056501408107578754, -0.008440684527158737, 0.030890921130776405, 0.00893888808786869, -0.011471380479633808, 0.004127564374357462, -0.05068998411297798, 0.014197525568306446, 0.024276675656437874, -0.002237569773569703, -0.008043934591114521, -0.1548350602388382, 0.02191237546503544, 0.0015858853003010154, 0.0066228643991053104, 0.007859948091208935, 0.001549408189021051, 0.008107692934572697, -0.011349791660904884, -0.017751004546880722, 0.010934333316981792, 0.003002288518473506, 0.0067741540260612965, -0.02212698571383953, 0.013824409805238247, 0.011871197260916233, -0.0039008217863738537, -0.0025933152064681053, -0.0009972010739147663, 0.021336035802960396, -0.016924701631069183, 0.001765562454238534, 0.008918397128582, 0.01149795763194561, 0.00557936355471611, 0.014978715218603611, -0.007292915601283312, -0.010963082313537598, 0.0037562325596809387, -0.013012721203267574, -0.000552408630028367, -0.012441208586096764, -0.0011780919739976525, 0.019825411960482597, 0.010741607286036015, -0.001453704433515668, 0.011919092386960983, 0.01728138141334057, 0.0016293330118060112, -0.013923857361078262, 0.011070863343775272, -0.010942618362605572, 0.024809757247567177, 0.03345687687397003, -0.02788887359201908, -0.003010053653270006, 0.001640185946598649, 0.0016310298815369606, 0.0018370329635217786, -0.0005674874992109835, 0.0040881154127418995, -0.023623226210474968, -0.013402455486357212, 0.002830394310876727, 0.0020396269392222166, 0.01479238923639059, -0.006993347313255072, 0.0036509744822978973, -0.004921343643218279, 0.0036948181223124266, 0.006165884900838137, -0.003214494325220585, -0.009394358843564987, 0.0006534582935273647, -0.008156624622642994, -0.016399340704083443, 0.00870139803737402, -0.0024743645917624235, 0.022383637726306915, -0.010104844346642494, 0.0017392259323969483, 0.030486173927783966, -0.006005562376230955, 0.002514333464205265, 0.006431556772440672, 0.024565091356635094, 0.012626429088413715, -0.0036485271994024515, 0.0014840030344203115, 0.05368035286664963, 0.015626979991793633, -0.014163650572299957, -0.012660876847803593, -0.026204772293567657, -0.01176599320024252, -0.00941145047545433, -0.07991975545883179, -0.01784614659845829, -0.0030394233763217926, 0.006935458164662123, -0.19554811716079712, 0.0042005376890301704, 0.01073393039405346, 0.010456192307174206, -0.011275169439613819, 0.022914446890354156, -0.005744928494095802, -0.011446764692664146, -0.0006567758391611278, -0.03527120500802994, 0.0009478094289079309, -0.03308207541704178, 0.012184036895632744, -0.005846772342920303, 0.016874214634299278, -0.0018981137545779347, -0.00025284470757469535, 0.007290124427527189, -0.004144340753555298, -0.014371473342180252, 0.012079768814146519, -0.020380128175020218, 0.03200319781899452, -0.024630043655633926, 0.01699257269501686, 0.0033809440210461617, 0.005876118782907724, 0.01301228441298008, 0.2612913250923157, 0.021281369030475616, -0.001918183290399611, 0.021587559953331947, -0.023320551961660385, -0.0059647997841238976, 0.005844437517225742, -0.04191190376877785, 0.007307039573788643, 0.0010029333643615246, 0.006446092389523983, -0.013644945807754993, -0.00609144801273942, -0.03839421644806862, 0.004361181519925594, -0.017354803159832954, -0.0016276598908007145, 0.007212265394628048, 0.005172074306756258, -0.044080521911382675, -0.00036605060449801385, 0.022531159222126007, 0.0009553453419357538, 0.00189256074372679, 0.0032778966706246138, -0.010648312047123909, -0.02740505523979664, -0.01603224314749241, 0.038200248032808304, 0.001402506255544722, -0.054450977593660355, 0.0038469890132546425, 0.003979122266173363, -0.023676946759223938, -0.008977471850812435, 0.0030832323245704174, -0.00416282843798399, -0.0003849400964099914, -0.021040499210357666, -0.01800149492919445, 0.005434525664895773, -0.009702368639409542, 0.011732838116586208, -0.014050117693841457, -0.008739211596548557, 0.009232616052031517, -0.005522764287889004, -0.009266365319490433, 0.0146708395332098, 0.00778252724558115, 0.010045344941318035, -0.0028503097128123045, -0.005319253541529179, -0.019500743597745895, 0.012848886661231518, 0.017847402021288872, 0.00887271761894226, 0.020576070994138718, -0.0032597389072179794, 0.00833187997341156, 0.00980314239859581, -0.00472960714250803, -0.00399711774662137, -0.005490717012435198, 0.028670459985733032, 0.001828062697313726, -0.011158646084368229, -0.015581071376800537, -0.01712166704237461, 0.0065454463474452496, 0.0023712795227766037, -0.007834885269403458, -0.14139702916145325, 0.01221954170614481, 0.01249172817915678, -0.006186359561979771, 0.004387479275465012, -0.004757468122988939, 0.004864454735070467, 0.0014284198405221105, 0.02886308915913105, 0.011780674569308758, -0.0009572218405082822, 0.006776426453143358, 0.006583092734217644, -0.014467006549239159, 0.010576501488685608, 0.005397708620876074, 0.03384619206190109, 0.007015174254775047, -0.017262596637010574, -0.019752614200115204, 0.0074546257965266705, 0.008221229538321495, -0.00649435818195343, -0.0040894197300076485, -0.0002147338236682117, 0.01064866129308939, -0.022763434797525406, -0.0037328314501792192, 0.002904539927840233, 0.025680750608444214, -0.01811346225440502, -0.007479091640561819, 0.004325652029365301, -0.0006631522555835545, -1.1958464710914996e-05, -0.001991676865145564, 0.02200962044298649, 0.016901573166251183, -0.0267573744058609, -0.007731791120022535, 0.022993624210357666, -0.005148907657712698, 0.01027683075517416, -0.015513092279434204, -0.008334694430232048, 0.0014767857501283288, -0.013520401902496815, -0.10767757147550583, -0.002788047306239605, -0.009425563737750053, -0.01970037631690502, -0.008942075073719025, -0.0009513254626654088, 0.004841488320380449, -0.0006120645557530224, -0.0064019691199064255, -0.004161795601248741, 0.01794915646314621, 0.011194558814167976, 0.011386703699827194, 0.0028729657642543316, 0.010473155416548252, -0.01689998060464859, 0.0013051927089691162, 0.01760558784008026, 0.009802674874663353, -0.017813995480537415, 0.0077735623344779015, 0.015600113198161125, -0.050067685544490814, -0.010536081157624722, 0.006482221186161041, -0.023553963750600815, -0.014052796177566051, -0.015872448682785034, 0.008732163347303867, -0.01579350046813488, -0.0008256075088866055, 0.3419153392314911, -0.009216031059622765, 0.01781020499765873, 0.004780197516083717, -0.013846089132130146, 0.008266815915703773, 0.009671402163803577, -0.006549173966050148, 0.049307677894830704, 0.00465930812060833, -0.01878192089498043, -0.02076612040400505, 0.009980935603380203, 0.010446667671203613, -0.02302504889667034, 0.017067160457372665, -8.595622057328e-05, -0.004594399128109217, -0.007051481865346432, 0.002903391607105732, -0.00933763012290001, -0.011270183138549328, 0.009797892533242702, -0.022632919251918793, 0.012001598253846169, 0.01595653034746647, 0.004811297170817852, -0.0024367196019738913, 0.006162421777844429, -0.0023245345801115036, -0.015223497524857521, 0.009132273495197296, 0.011198252439498901, 0.008882495574653149, -0.004672394134104252, -0.012283503077924252, 0.01315891370177269, 0.0036707331892102957, 0.005931810010224581, 0.013636390678584576, 0.013574876822531223, 0.00820250529795885, 0.058365143835544586, 0.0030212269630283117, 0.009593196213245392, 0.008762367069721222, -0.0055112638510763645, -0.012795868329703808, 0.028824688866734505, 0.01263260655105114, -0.004422924481332302, 0.01921069622039795, 0.014379244297742844, -0.008627948351204395, 0.008018869906663895, 0.005713990889489651, 0.006549368612468243, -0.020208343863487244, 0.003587925573810935, 0.004591146018356085, -0.0036157588474452496, -0.002655389253050089, -0.0067960843443870544, -0.011966057121753693, 0.005676520988345146, -0.008029584772884846, 0.007590858265757561, -0.012621045112609863, -0.01882016472518444, -0.0009518875158391893, -0.009893815033137798, -0.000706878665369004, -0.008729709312319756, 0.01342347264289856, 0.017429091036319733, 0.0021685189567506313, -0.02964012511074543, -0.0016898810863494873, -0.0009790320182219148, -0.01601342484354973, -0.019297273829579353, -0.010603646747767925, -0.004675223957747221, -0.017845625057816505, -0.004687061533331871, -0.0041627963073551655, -0.034114155918359756, -0.022352514788508415, -0.0043302057310938835, 0.0038342499174177647, 0.012345831841230392, -0.009815139696002007, -0.01959293521940708, 0.0017874095356091857, -0.006199148017913103, 0.009930756874382496, -0.0021871936041861773, -0.01210322231054306, 0.009210084564983845, 0.013689500279724598, 0.025574762374162674, 0.0031278440728783607, 0.0031100742053240538, -0.0036533758975565434, -0.00013206596486270428, 0.005421502515673637, -0.014562713913619518, 0.011210386641323566, -0.0029276330024003983, -0.005254935473203659, 0.018514422699809074, 0.00168291584122926, 0.011654636822640896, -0.0016517905751243234, -0.001269910717383027, 0.0161272082477808, -0.003764200024306774, -0.02275998704135418, -0.43710607290267944, 0.011529894545674324, -0.0067541329190135, 0.006973045412451029, -0.021797284483909607, -0.0005575213581323624, 0.02319888398051262, 0.008160348050296307, 0.02199791930615902, 0.006744608283042908, -0.014759686775505543, -0.009615901857614517, -0.1385050117969513, 0.018764778971672058, -0.01722620241343975, 0.001025591162033379, -0.026222338899970055, -0.06284405291080475, 0.004912697244435549, 0.03678944334387779, 0.0029210911598056555, 0.02194151096045971, 0.020292403176426888, 0.007377488072961569, 0.01733691245317459, -0.005288115236908197, -0.008904176764190197, -0.002460370771586895, 0.018763158470392227, -0.006403753068298101, 0.0027747033163905144, -0.008548565208911896, 0.020369693636894226, 0.0053582098335027695, -0.0033115011174231768, -0.013619053177535534, -0.003883497789502144, 0.0033483398146927357, 0.008005666546523571, -0.013208451680839062, 0.003422769717872143, 0.0002618624712340534, 0.008444827981293201, 0.018052686005830765, -0.030881367623806, -0.013070518150925636, -0.029834546148777008, -0.15523561835289001, 0.0051672677509486675, -0.004872259218245745, 0.016929270699620247, -0.0007998169166967273, -0.006909756921231747, -0.0059412079863250256, -0.010823024436831474, 0.011835073120892048, 0.005133113823831081, 0.0004134245391469449, 0.011388720013201237, -0.02790956199169159, -0.018819153308868408, -0.0039221798069775105, 0.01992027275264263, -0.002175629138946533, -0.007312837056815624, 0.012563405558466911, -0.025112150236964226, -0.006629807408899069, -0.012350969016551971, -0.019431136548519135, -0.008902444504201412, -0.004164357203990221, -0.009548013098537922, 0.009151983074843884, 0.006864199880510569, 0.020201755687594414, 0.0008470991160720587, -0.010545935481786728, -0.003626397345215082, -0.0015591197879984975, -0.02373996190726757, -0.01057033333927393, 0.024029040709137917, 0.01726173423230648, 0.009374474175274372, -0.008514577522873878, 0.0029538196977227926, 0.010616444051265717, -0.004190776497125626, 0.014341500587761402, -0.002284505171701312, 0.005089922808110714, 0.007256838493049145, -0.0025276343803852797, -0.006117844022810459, 0.004403323866426945, -0.010777635499835014, 0.00023478233197238296, -0.022016353905200958, -0.006028801202774048, -0.004779117181897163, 0.005305275786668062, 0.009273837320506573, -0.030027003958821297, 0.004405364394187927, -0.018014689907431602, 0.17106041312217712, -0.0017060410464182496, -0.00844417605549097, -0.0018065833719447255, -0.012183873914182186, 0.014778499491512775, 0.011482547037303448, 0.0033188576344400644, 0.1010018140077591, 0.007895300164818764, -0.017003297805786133, 0.010892221704125404, -0.005164097063243389, -0.0030563848558813334, -0.004030361771583557, 0.03558965399861336, 0.0051087005995213985, 0.009430723264813423, -0.010044591501355171, 0.007733828853815794, 0.018170876428484917, -0.01878982037305832, 0.010004465468227863, -0.010283475741744041, 0.006260390859097242, -0.001349389785900712, -0.023831574246287346, -0.01760699972510338, 0.01940978318452835, -0.023553019389510155, 0.003576207673177123, -0.0006366035086102784, -0.009569992311298847, 0.06411442905664444, -0.007499521132558584, 0.01027713157236576, 0.0135386623442173, -0.0006888909847475588, -0.01112333219498396, -0.028094079345464706, -0.013262196443974972, 0.0015999634051695466, -0.04410097748041153, 0.03183741495013237, -0.0017592855729162693, -0.0037528008688241243, -0.006853803992271423, -0.010225174948573112, -0.0018268033163622022, -0.0066506993025541306, -0.009905870072543621, -0.030319632962346077, 0.0001537170319352299, 0.010265651158988476, 0.004953790456056595, -0.00048594095278531313, -0.04260385036468506, -0.009570960886776447, 0.007734989747405052, -0.011329330503940582, 0.0017088351305574179, -0.010869327932596207, -0.0025666875299066305, -0.008473736234009266, 0.000842987559735775, 0.0033941389992833138, 0.00535108195617795, -0.013263625092804432, -0.001992730190977454, -0.01349619310349226, -0.008115450851619244, 0.017578808590769768, -0.152586430311203, -0.006176742725074291, 0.018798496574163437, -0.03187151253223419, -0.01748816668987274, -0.01802665926516056, 0.014273012988269329, 0.012175831943750381, 0.003233163384720683, 0.0122830206528306, -0.021766377612948418, 0.12721876800060272, 0.005080587230622768, -0.0015748782316222787, -0.0008181261364370584, -0.01403118297457695, 0.013247922994196415, 0.009132593870162964, -0.002362856175750494, -0.02228337898850441, -0.015198471024632454, 0.1153784990310669, 0.004176785238087177, 0.024438608437776566, 0.004605927038937807, -0.010780737735331059, 0.004907727241516113, 0.011173807084560394, 0.0012403330765664577, -0.003946632146835327, -0.019072573632001877, 0.0037975001614540815, 0.012021943926811218]}, {"text": "You will be given a definition of a task first, then some input of the task.\nThe input is taken from a negotiation between two participants who take the role of campsite neighbors and negotiate for Food, Water, and Firewood packages, based on their individual preferences and requirements. Given an utterance and recent dialogue context containing past 3 utterances (wherever available), output Yes if the utterance contains the small-talk strategy, otherwise output No. small-talk is a cooperative negotiation strategy. It is used for discussing topics apart from the negotiation, in an attempt to build a rapport with the opponent. For example, discussing how the opponent is doing during the pandemic or sharing excitement for the camping trip.\n\nContext: 'Hello there! Are you excited for your camping trip coming up?! I am excited to see how I can put my skills to the test!' 'Yeah. Nice to get out after being inside for most of the spring. This covid-19 crap! wish it would go away.' 'I agree! I will say it has really sparked my interest in camping and being outdoors even more though! I saw just how connected I was to technology and everything else!'\nUtterance: 'I'm up here to do some cross training and noticed that there is some extra water around. Even though the nights are cold I could use some extra water. Do you exercise much?'\nOutput:", "label": "HIGH", "embedding": [0.02097022347152233, -0.004369220230728388, -0.0017507118172943592, -0.009697705507278442, -0.024211008101701736, -0.006609389558434486, 0.016151877120137215, -0.01088809035718441, -0.00036297328188084066, -0.07029043883085251, -0.015476536937057972, -0.0033059848938137293, 0.006392703391611576, -0.009658143855631351, -0.033278919756412506, 0.015518778003752232, 0.017372507601976395, -0.007968191057443619, -0.010733040049672127, -0.015157519839704037, 0.00857547391206026, -0.009239564649760723, 0.03866586461663246, -0.012023426592350006, 0.017018543556332588, -0.00029703404288738966, 0.009056348353624344, 0.005635657347738743, -0.014201287180185318, 0.03240927681326866, -0.00509974779561162, 0.017522040754556656, -0.004658373538404703, -0.006905778776854277, -0.01421413104981184, -0.0014770480338484049, 0.002741528209298849, -0.022603686898946762, -0.008299492299556732, 0.003070591716095805, -0.028333820402622223, 0.004878114443272352, 0.028470918536186218, -0.020937854424118996, -0.008650178089737892, -0.02775544859468937, -0.004232421517372131, -0.003465931862592697, 0.0038973395712673664, 2.0833804228459485e-05, -0.0019443219061940908, -0.0017648108769208193, 0.002325588371604681, 0.008284538052976131, 0.006964058615267277, 0.005403584334999323, 0.001043770113028586, -0.003632579930126667, -0.011067710816860199, 0.016384443268179893, -0.008297189138829708, -0.008234452456235886, 0.016321225091814995, 0.003044720273464918, -0.014306800439953804, 0.0011168004712089896, 0.004939896985888481, -0.0013245183508843184, -0.01183011569082737, -0.012235276401042938, -0.0018602911150082946, 0.009833380579948425, 0.014510502107441425, 0.06179720535874367, 0.01532120630145073, -0.006786874495446682, 0.02663039229810238, 0.01793121173977852, 0.007124199997633696, -0.0014896588400006294, 0.0044557200744748116, 0.002508907113224268, -0.011794403195381165, 0.005367474164813757, -0.005421864800155163, 0.0034567571710795164, 0.013681412674486637, -0.016846340149641037, -0.014527853578329086, 0.015079200267791748, 0.0042466772720217705, -0.008906044065952301, -0.025371188297867775, -0.005096475128084421, -0.011037678457796574, 0.020789235830307007, 0.003969920799136162, 0.0035049442667514086, 0.4049341380596161, 0.0030757375061511993, -0.004956619814038277, 0.0015297613572329283, 0.0011348376283422112, 0.0005115129752084613, 0.018098488450050354, -0.011098351329565048, -0.009581909514963627, -0.02367938868701458, -0.0033088421914726496, 0.027910947799682617, 0.029258545488119125, 0.0034236221108585596, 0.004616097547113895, 0.0017378840129822493, -0.0012457063421607018, -0.02292543649673462, -0.0015620725462213159, 0.006057647988200188, 0.016756538301706314, -0.00974081177264452, 0.011754103936254978, 0.0010723195737227798, 0.06619712710380554, 0.009683380834758282, -0.009418662637472153, -0.00903476681560278, 0.002935564611107111, -0.013775859959423542, 0.0090322345495224, -0.006801027338951826, -0.033686988055706024, -0.010086692869663239, -0.0044031268917024136, 0.0016607294091954827, 0.011058901436626911, -0.000497513625305146, -0.00904515665024519, 0.0006144575891084969, -0.04782120883464813, 0.00972677767276764, 0.011449198238551617, -0.01712685637176037, 0.010582040064036846, -0.02186397649347782, -0.025253385305404663, 0.18804773688316345, -0.003195506986230612, -0.013454404659569263, -0.0031885376665741205, -0.007693212479352951, -0.001946758828125894, -0.017842311412096024, -0.012705796398222446, 0.010646204464137554, 0.004529325291514397, -0.010754495859146118, 0.020387185737490654, 0.00040853681275621057, -0.0007999177905730903, 0.005527019500732422, -0.057026948779821396, -0.0006852099322713912, -0.011241977103054523, 0.036101195961236954, 0.006345205474644899, -0.009724961593747139, -0.0010234597139060497, -0.05421486496925354, 0.01887579634785652, 0.031604938209056854, 0.0012320965761318803, -0.005332090426236391, -0.15585821866989136, 0.01381760835647583, 0.013382314704358578, 0.004662747494876385, 0.006070825271308422, 0.007456205785274506, -0.005116784945130348, -0.012430679053068161, -0.015543055720627308, 0.010741318576037884, 0.006544355768710375, 0.0003401745925657451, -0.020678412169218063, 0.011324383318424225, 0.01013749185949564, -0.019834188744425774, -0.004335750825703144, 0.00022011609689798206, 0.0054007526487112045, -0.016416320577263832, 0.005657725501805544, 0.006817271001636982, 0.015985554084181786, -0.0017270444659516215, 0.012217782437801361, -0.013873056508600712, -0.009019669145345688, 0.0004460039781406522, -0.011030156165361404, 0.003174826502799988, -0.015452335588634014, -0.001118419342674315, 0.01863214187324047, 0.011117657646536827, 0.0035748067311942577, 0.00609815726056695, 0.017794352024793625, 0.0028574285097420216, -0.01774796098470688, -0.004545307718217373, -0.0036736316978931427, 0.02077566646039486, 0.03575446829199791, -0.016739489510655403, 0.0048050773330032825, 0.005845879204571247, 0.00519740954041481, -0.004354827105998993, -0.0007293774397112429, 0.009709789417684078, -0.013647634536027908, -3.4138818591600284e-05, 0.0014555897796526551, 0.0076501211151480675, 0.005992223974317312, -0.017624827101826668, -0.0025880439206957817, -0.01446490827947855, 0.008778639137744904, 0.014683819375932217, 0.0035648636985570192, -0.007330542895942926, 0.00186175259295851, -0.009409462101757526, -0.006365787237882614, 0.012984918430447578, 0.0019954442977905273, 0.01815822906792164, 0.004223640076816082, -0.0030994650442153215, 0.021254654973745346, -0.010538513772189617, -0.0010713770752772689, 0.011517023667693138, 0.02087157964706421, 0.009095550514757633, -0.005490378476679325, -0.0027090711519122124, 0.05360639840364456, 0.015144318342208862, -0.014207273721694946, -0.013202124275267124, -0.02834099717438221, -0.002231049118563533, -0.011391009204089642, -0.07880819588899612, -0.019392915070056915, -0.0068978201597929, 0.009264707565307617, -0.19192683696746826, 0.01293808314949274, 0.01791219785809517, 8.0978570622392e-05, -0.008437850512564182, 0.018758634105324745, -0.0020859423093497753, -0.013625075109302998, -0.003565669059753418, -0.031298067420721054, 0.004591320641338825, -0.035708505660295486, 0.0037988717667758465, -0.01197122409939766, 0.020125817507505417, -0.0020873534958809614, -0.006226880010217428, 0.0033891478087753057, -0.0025380277074873447, -0.010174546390771866, 0.00858669076114893, -0.009604323655366898, 0.037741292268037796, -0.023043235763907433, 0.012422914616763592, 0.0035245444159954786, 0.007548332214355469, 0.014019536785781384, 0.251288503408432, 0.027280345559120178, 0.0019370527006685734, 0.021075919270515442, -0.025813480839133263, -0.005301231052726507, -0.0014066030271351337, -0.033034853637218475, 0.0033729032147675753, 0.001645048032514751, -0.0008068750612437725, -0.015099477022886276, -0.006337057799100876, -0.03702835738658905, 0.003802512539550662, -0.021590281277894974, -0.005723395850509405, -0.002284484915435314, 0.015298973768949509, -0.041419390588998795, -0.004640225321054459, 0.018159424886107445, 0.0029816688038408756, 0.008025558665394783, 0.006675052922219038, -0.009407740086317062, -0.03340199589729309, -0.022405924275517464, 0.032420556992292404, 0.011965801939368248, -0.04901342839002609, 0.0009932599496096373, 0.012963165529072285, -0.026639074087142944, -0.008498531766235828, 0.005407733842730522, 0.004712013527750969, -0.006985832937061787, -0.018440432846546173, -0.016954120248556137, 0.0023095046635717154, 0.0011763478396460414, 0.01327161118388176, -0.019253069534897804, -0.011411641724407673, 0.0005729945842176676, -0.010652609169483185, -0.004637314006686211, 0.01814819499850273, 0.0030704743694514036, 0.0033292255830019712, -0.020464714616537094, -0.006002663169056177, -0.013803992420434952, 0.008294327184557915, 0.013695771805942059, 0.0027001728303730488, 0.017884742468595505, -0.00675587123259902, 0.014232882298529148, 0.005150592420250177, -0.0011929317843168974, 0.011970275081694126, -0.00020247873908374459, 0.02525816671550274, -0.011494622565805912, -0.011424089781939983, -0.008766510523855686, -0.01209302432835102, 0.00046659319195896387, 0.0064234319142997265, 0.00474164355546236, -0.1345837116241455, 0.021528590470552444, 0.010719888843595982, -0.012849122285842896, 0.006240712944418192, -0.0042886557057499886, -0.0015430357307195663, 0.0004915404133498669, 0.02833476848900318, 0.006856171879917383, -0.009785940870642662, 0.009011870250105858, 0.012760494835674763, -0.014790494926273823, 0.003581338794901967, 0.009828855283558369, 0.04114069044589996, 0.009924870915710926, -0.01632988639175892, -0.018036553636193275, 0.002079735277220607, 0.019526902586221695, -0.0051713502034544945, -0.001154646510258317, -0.005403199698776007, 0.0027993284165859222, -0.032100167125463486, -0.005355239380151033, -0.004296197555959225, 0.026332642883062363, -0.014379935339093208, 0.0006837589317001402, -0.0027139876037836075, -0.0012711816234514117, 0.00166658207308501, 0.0018234610324725509, 0.016310207545757294, 0.011192855425179005, -0.021683987230062485, -0.017013894394040108, 0.008641228079795837, 0.004246944095939398, 0.00785343162715435, -0.016663435846567154, -0.0012816331582143903, 0.004961339291185141, -0.021489381790161133, -0.10696616768836975, -0.01712295040488243, -0.005995514336973429, -0.017720071598887444, -0.011893881484866142, 0.009334523230791092, 0.011790679767727852, -0.00023590178170707077, -0.0015280862571671605, -0.0019485366065055132, 0.017535215243697166, 0.013325396925210953, 0.007206453010439873, -0.00019408081425353885, 0.006093115545809269, -0.021983813494443893, -0.0023606542963534594, 0.02477640099823475, 0.007117456290870905, -0.015776557847857475, 0.008100870065391064, 0.016591424122452736, -0.057483237236738205, -0.0034693842753767967, -0.004508281592279673, -0.027930838987231255, -0.009476313367486, -0.012451335787773132, 0.003925575874745846, -0.0186696108430624, -0.009167877025902271, 0.33741942048072815, -0.0050614215433597565, 0.012546336278319359, 0.0029392975848168135, -0.017881028354167938, 0.005992253310978413, 0.010283490642905235, -0.00985571276396513, 0.054668691009283066, 0.004141271114349365, -0.019514823332428932, -0.02145269140601158, 0.006608589552342892, 0.0014876879286020994, -0.01889641582965851, 0.030736297369003296, 0.003966257907450199, -0.0011384374229237437, -0.011920280754566193, -0.001969211967661977, -0.00467730313539505, -0.00602044677361846, 0.01327419187873602, -0.01640086993575096, 0.008825773373246193, 0.01621912606060505, 0.004083969630300999, -0.0005541503196582198, -0.0014436495257541537, -0.00971678365021944, -0.010479327291250229, 0.021590039134025574, 0.010460038669407368, 0.010274138301610947, 0.005339257884770632, -0.015950458124279976, 0.010855426080524921, 0.0022004542406648397, 0.005522096995264292, 0.00863470695912838, 0.014646399766206741, -0.002894118195399642, 0.053492479026317596, -0.0023761510383337736, 0.008908823132514954, 0.014047897420823574, -0.0069345273077487946, -0.004637870471924543, 0.01948617398738861, 0.014830359257757664, -0.01012710016220808, 0.01677773706614971, 0.009484595619142056, -0.011353382840752602, 0.011768719181418419, 0.014142694883048534, 0.01000466663390398, -0.01430897694081068, 0.0020341970957815647, 0.00886404886841774, 0.0039896490052342415, -0.010463311336934566, -0.012243733741343021, -0.002663240535184741, 0.011661709286272526, -0.010023723356425762, 0.013912378810346127, -0.011314647272229195, -0.01262218039482832, -0.01118854433298111, -0.012637081556022167, -6.770709296688437e-05, -0.007140979170799255, 0.008072788827121258, 0.015689508989453316, 0.0080292709171772, -0.030791884288191795, -0.0018460400169715285, 0.0018756297649815679, -0.009317141026258469, -0.02006065659224987, -0.013302366249263287, -0.0011909196618944407, -0.012443818151950836, -0.006777186878025532, 0.0008461675024591386, -0.035622816532850266, -0.02547316439449787, 0.0010441045742481947, 0.007544719614088535, 0.0024396637454628944, -0.00985526479780674, -0.010602026246488094, 0.007193258497864008, 0.002602117834612727, 0.008900025859475136, -0.00014088516763877124, -0.007901104167103767, -0.00896727666258812, 0.01349995844066143, 0.026993511244654655, -0.00028254982316866517, 0.011294967494904995, -0.008759165182709694, -0.004688120447099209, -0.0007492068689316511, -8.430174784734845e-05, 0.015168462879955769, -0.005230848677456379, 0.004492072854191065, 0.018789663910865784, 0.0009780930122360587, 0.005929945036768913, 0.004375317133963108, -0.004090371076017618, 0.01139445323497057, -0.011413550935685635, -0.024114718660712242, -0.47423967719078064, 0.008717136457562447, -0.004360011778771877, 0.006598907057195902, -0.019274543970823288, -0.0006434047827497125, 0.022911880165338516, 0.011921596713364124, 0.01578226312994957, 0.000812510319519788, -0.019144322723150253, -0.009322457946836948, -0.13641013205051422, 0.018195170909166336, -0.012306428514420986, -0.0024619936011731625, -0.01516583003103733, -0.05794670805335045, 0.0051151905208826065, 0.02992730215191841, -0.011691829189658165, 0.01820489764213562, 0.017589302733540535, 0.012499017640948296, 0.022172806784510612, -0.0026344701182097197, -0.009194989688694477, -0.0008573894156143069, 0.008696956560015678, -0.002476254478096962, -0.0023840905632823706, -0.00855986401438713, 0.02228427305817604, 0.002604201901704073, -0.003257936565205455, -0.012980549596250057, -0.004181983880698681, -0.0026827019173651934, 0.012630967423319817, -0.020802801474928856, -0.005520148668438196, 0.004386918619275093, 0.010412688367068768, 0.007900970987975597, -0.03703220933675766, -0.012953515164554119, -0.029500558972358704, -0.1511998474597931, 0.00114737288095057, -0.0018270158907398582, 0.011964279226958752, -0.0006102338084019721, -0.0024675526656210423, -0.0028347200714051723, -0.01380885113030672, 0.013493899255990982, 0.008722757920622826, -0.001750538358464837, 0.015931541100144386, -0.018738603219389915, -0.01631133444607258, -0.007697701919823885, 0.029586108401417732, -0.0009039948345161974, 0.0005460886750370264, 0.0071763708256185055, -0.01904180459678173, -0.00026316841831430793, -0.00804136972874403, -0.02469320222735405, -0.003908350598067045, 0.0009877149714156985, -0.004626819398254156, 0.0073111276142299175, 0.01716233417391777, 0.0195477157831192, 0.0007486658287234604, -0.0023714383132755756, 0.009246451780200005, -0.003834368893876672, -0.0230763740837574, -0.003334503388032317, 0.021221259608864784, 0.02663431130349636, 0.01795773394405842, -0.013633943162858486, -0.0012397594982758164, 0.018533267080783844, -0.012497101910412312, 0.002713587833568454, -0.0015688012354075909, 0.0035198479890823364, 0.006946417968720198, -0.008940530009567738, -0.002667473629117012, 0.006246425211429596, -0.015519184991717339, 0.00028973602456972003, -0.023582642897963524, -0.01542600616812706, -0.00485308887436986, 0.008225361816585064, 0.009229864925146103, -0.026558056473731995, -0.006083701271563768, -0.02138127200305462, 0.17396242916584015, 0.003946243319660425, -0.004289144650101662, -0.0052990783005952835, -0.003975321538746357, 0.01541078370064497, -0.004461473319679499, 0.005518772639334202, 0.09371162950992584, 0.00583690544590354, -0.011096891015768051, 0.01309591718018055, -0.002853096928447485, 0.001146074035204947, -0.0009543578489683568, 0.025493871420621872, 0.0027819767128676176, 0.005981987342238426, -0.006608216557651758, 0.00558258593082428, 0.012502925470471382, -0.012086748145520687, 0.013713348656892776, -0.007963196374475956, 0.0024229881819337606, 0.0034177417401224375, -0.02662755735218525, -0.011048832908272743, 0.018717458471655846, -0.020952133461833, 0.0032562881242483854, 0.004586957860738039, -0.014999909326434135, 0.06270445138216019, -0.017583608627319336, 0.0050298674032092094, 0.01635526306927204, -0.0068793706595897675, -0.0023930007591843605, -0.024358632043004036, -0.010301775299012661, -0.004185901954770088, -0.032251983880996704, 0.03414241597056389, -0.0018554824637249112, -0.008953011594712734, -0.017522431910037994, -0.013254757970571518, 0.0020176635589450598, -0.008206196129322052, -0.015221131034195423, -0.009796847589313984, -0.008111291565001011, 0.0053167156875133514, -0.0031628136057406664, -0.0027838817331939936, -0.03410114720463753, -0.004961937200278044, 0.017257291823625565, -0.005970523227006197, 0.001730006537400186, 0.0004900294006802142, 0.00016542086086701602, -0.002856866456568241, 0.004783200100064278, -0.004749262239784002, -0.004532736260443926, -0.023805364966392517, 0.00139817432500422, -0.011689232662320137, -0.009818688966333866, 0.014156000688672066, -0.14711301028728485, -0.004627136047929525, 0.005320393480360508, -0.03693709149956703, -0.022617321461439133, -0.017308421432971954, 0.022724172100424767, 0.014989112503826618, -0.002818354871124029, 0.01318431831896305, -0.0032003424130380154, 0.11439359933137894, -0.01119931973516941, -0.003706853836774826, -0.0028451455291360617, -0.0151633620262146, 0.016564032062888145, 0.01333682518452406, -0.006984272040426731, -0.017313336953520775, -0.001134516904130578, 0.11178696900606155, 0.004079824313521385, 0.0286286398768425, 0.011108658276498318, -0.008215953595936298, -0.0010762395104393363, 0.004289655480533838, 0.012484685517847538, -0.005259339697659016, -0.010291028767824173, 0.007088017649948597, 0.011573768220841885]}, {"text": "Whats the significance of 42 ?", "label": "HIGH", "embedding": [0.0302217248827219, 0.0034414727706462145, -0.0071603828109800816, -0.011159433983266354, -0.01548239029943943, -0.0038161203265190125, 0.020291313529014587, -0.005428831558674574, 0.011269758455455303, -0.07888820767402649, -0.02044907584786415, 0.007811347953975201, -0.00926171150058508, -0.004001171328127384, -0.03684096410870552, 0.011654668487608433, 0.017757583409547806, 0.00217657838948071, -0.005836795084178448, -0.014704770408570766, 0.008268648758530617, -0.006432552821934223, 0.04385457932949066, -0.016333311796188354, 0.014209904707968235, 0.0022362868767231703, 0.011946997605264187, 0.005864973179996014, -0.007249544374644756, 0.030237190425395966, 0.020809557288885117, 0.009104197844862938, -0.003875497728586197, 0.003864946076646447, -0.013984155841171741, 0.00506649911403656, -0.001222560415044427, -0.016148008406162262, -0.007086448837071657, -0.009134029038250446, -0.03228382393717766, 0.011349373497068882, 0.04013978689908981, -0.012566235847771168, 0.006566056981682777, -0.025507301092147827, 0.006696630734950304, 0.003587722312659025, -0.015878356993198395, 0.014251144602894783, 6.50492511340417e-05, 0.012025902047753334, -0.007010640110820532, 0.007076314650475979, 0.009130878373980522, 0.005328930448740721, -0.0053462861105799675, 0.012329681776463985, -0.03246286138892174, 0.021327506750822067, 0.000895767065230757, 0.001053881598636508, 0.022715598344802856, 8.994472591439262e-05, -0.0036609629169106483, 0.010507777333259583, 0.0032142901327461004, 0.006853005383163691, -0.015622608363628387, -0.01047916617244482, -0.003006153739988804, 0.011301827616989613, 0.0076979100704193115, 0.06586091220378876, 0.022977784276008606, -0.011430243030190468, 0.03388150408864021, 0.0070971339009702206, 0.005376449786126614, 0.001026767771691084, 0.009339535608887672, -0.005950176157057285, -0.015216249041259289, -0.011446944437921047, 0.00439589936286211, -0.009133157320320606, 0.016710398718714714, 0.007800823077559471, -0.016651270911097527, 0.012876186519861221, 0.014668739400804043, -0.006823178380727768, -0.01642647013068199, 0.0002659418969415128, -0.007196583319455385, 0.017689786851406097, -0.001308953738771379, 0.006149854976683855, 0.3966776728630066, -0.014946255832910538, -0.012304568663239479, 0.003629714250564575, 0.005080569069832563, -0.006255623884499073, 0.02624949999153614, -0.02680942788720131, -0.002886081812903285, -0.02228190191090107, 0.00336664030328393, 0.02671590819954872, 0.03102125972509384, -0.00479055754840374, 0.0072212121449410915, -0.01241318043321371, -0.006390667986124754, -0.02262749709188938, 0.0055066621862351894, -0.00038672404480166733, 0.013743003830313683, -0.01210204791277647, 0.005259741563349962, -0.008210321888327599, 0.05693593993782997, 0.012050513178110123, -0.006699400953948498, -0.010378033854067326, -0.004831444472074509, -0.006687374785542488, 0.01149022951722145, -0.014752514660358429, -0.044230613857507706, -0.015368003398180008, -0.010285676456987858, 0.0033734901808202267, 0.0065061356872320175, 0.005514517426490784, -0.013698422349989414, 0.01082686148583889, -0.06178320571780205, 0.014453727751970291, 0.014363582246005535, -0.01478196308016777, 0.01156462263315916, -0.012598181143403053, -0.02364644594490528, 0.1859147548675537, -0.009069298394024372, -0.0029880423098802567, -0.003597727743908763, 0.012097313068807125, 0.016396114602684975, -0.02110876329243183, -0.011545957066118717, 0.008876313455402851, -0.009913936257362366, -0.005728767719119787, 0.013205118477344513, 0.005195429548621178, 0.0026118431705981493, -0.00031266253790818155, -0.05713281407952309, 0.010173439979553223, -0.006988647393882275, 0.03223571926355362, 0.00736293476074934, -0.007934045977890491, 0.010386546142399311, -0.03656021133065224, 0.012791052460670471, 0.020930984988808632, -0.002491164952516556, -0.0012443129671737552, -0.15265296399593353, 0.021455062553286552, 0.007996848784387112, 0.003923814278095961, -0.0016159637598320842, 0.006878371816128492, -0.014046205207705498, -0.011975793167948723, -0.008430573157966137, 0.019104648381471634, 0.007210428360849619, 0.005625123158097267, -0.02956322766840458, 0.004485010635107756, 0.003850144101306796, -0.010292490012943745, -0.003108469070866704, 0.0023539746180176735, -0.0026108832098543644, -0.026646731421351433, 0.002743627643212676, 0.010995722375810146, 0.006142852362245321, 0.004892861936241388, 0.001584101002663374, -0.007596555165946484, -0.005564607214182615, 0.004301134962588549, 0.0025165542028844357, -0.009829019196331501, -0.021500738337635994, 0.01756848767399788, 0.026169192045927048, -0.002009146148338914, -0.0035091566387563944, 0.009017235599458218, 0.02204865589737892, -0.0024964450858533382, -0.009044313803315163, 0.007827585563063622, 0.008552014827728271, 0.03222888335585594, 0.03165600448846817, -0.022654635831713676, -0.0031096895691007376, 0.018804805353283882, 0.009038802236318588, 0.0005889812018722296, -0.013846689835190773, 0.004369418602436781, -0.0191494133323431, -0.009944016113877296, 0.006153517868369818, -0.0036498240660876036, 0.008532540872693062, 0.0024942054878920317, 0.0018296560738235712, -0.022389080375432968, 0.010079137980937958, 0.009829110465943813, -0.0025293983053416014, -0.010974711738526821, -0.002719435840845108, 0.004303358495235443, -0.01828281208872795, 0.02632179483771324, 0.014313139952719212, 0.016818538308143616, 0.0009476859122514725, -0.009318648837506771, 0.024925461038947105, 0.004905737470835447, -1.799613528419286e-05, 0.007873941212892532, 0.020648419857025146, 0.008158988319337368, 0.0030602372717112303, 0.004182367119938135, 0.04758056625723839, 0.012590009719133377, -0.02022368647158146, -0.015559948980808258, -0.0406346470117569, -0.010897687636315823, -0.00026224047178402543, -0.08482993394136429, -0.030406026169657707, -0.004127780441194773, 0.010211258195340633, -0.18582287430763245, 0.006165335886180401, 0.001972866477444768, 0.0020463771652430296, -0.000922794861253351, 0.02000328339636326, 0.0073820482939481735, -0.015915416181087494, 0.009385155513882637, -0.027080968022346497, -0.0038839830085635185, -0.04249027371406555, 0.007556877098977566, -0.01257857121527195, 0.016502393409609795, -0.007732601836323738, -0.0024685643147677183, -0.01697658747434616, -0.01332033984363079, -0.018391631543636322, 0.015659639611840248, -0.016726316884160042, 0.012843656353652477, -0.019711283966898918, 0.031494978815317154, 0.021560363471508026, 0.010558177717030048, -0.0005209947121329606, 0.2546648681163788, 0.021868523210287094, -0.012879593297839165, 0.01455112174153328, -0.009819833561778069, -0.006895221769809723, -0.0036889463663101196, -0.03580045700073242, 0.004025651142001152, 0.006155109498649836, 0.013748534955084324, -0.0054117413237690926, 0.0067819454707205296, -0.03575818985700607, 0.0012114751152694225, -0.022124579176306725, 0.007905438542366028, 0.0019146788399666548, 0.004072922747582197, -0.038063373416662216, -0.003426693379878998, 0.006375713273882866, 0.0050319284200668335, 0.0016773493262007833, 0.00400337902829051, -0.014340005815029144, -0.030132649466395378, -0.01660824567079544, 0.032918334007263184, -0.002049932489171624, -0.057566892355680466, 0.013203027658164501, 0.007803825195878744, -0.018122924491763115, -0.004898636601865292, 0.003065175376832485, -0.003161407308652997, -0.004025210626423359, -0.01815224066376686, -0.015431389212608337, -0.0009584694053046405, -0.008325589820742607, 0.007107958197593689, -0.031959064304828644, -0.020926261320710182, -0.00629466911777854, -0.0046275295317173, -0.006283757276833057, 0.008065298199653625, -0.0014698310988023877, -0.004394639749079943, -0.02766214869916439, -0.00691306684166193, -0.02103784680366516, 0.01449886616319418, 0.026148151606321335, 0.0031772456131875515, 0.01993025839328766, -0.002694149734452367, 0.007102711126208305, 0.006716717500239611, 0.010534880682826042, 0.004131698049604893, -0.0009790624026209116, 0.023198528215289116, -0.004595187027007341, -0.009338279254734516, -0.012529383413493633, -0.0299606341868639, 0.0009714844054542482, 0.008493621833622456, 0.001460116938687861, -0.1321924328804016, 0.008764130063354969, 0.01030928734689951, -0.008237751200795174, 0.005352329928427935, -0.009602285921573639, 0.006615393795073032, 0.0008088654140010476, 0.028110941872000694, 0.01598994992673397, 0.007799434009939432, 0.02666471153497696, 0.004588741343468428, -0.031135614961385727, 0.00028937324532307684, 0.0027884123846888542, 0.03631841763854027, 0.010514691472053528, -0.026116911321878433, -0.01840818114578724, 0.008435847237706184, 0.01661578379571438, -0.005761934444308281, 0.006349827162921429, 0.007714474573731422, 0.006262167822569609, -0.019884420558810234, -0.0013504853704944253, 0.0008760240743868053, 0.010612824000418186, -0.015344676561653614, -0.005609934683889151, -0.01091052033007145, -0.016318561509251595, 0.005688834004104137, -0.01694847084581852, 0.01668127439916134, 0.016747089102864265, -0.02834649756550789, -0.012220034375786781, 0.007694129832088947, -0.007698636502027512, 0.007271212060004473, -0.020469168201088905, -0.008749286644160748, 9.81524062808603e-05, -0.012876828201115131, -0.10883098095655441, -0.000977401970885694, -0.0020177869591861963, -0.01395803689956665, -0.008576001971960068, 0.02236102893948555, 0.005561702884733677, -0.0018783455016091466, 0.013676625676453114, 0.002500757575035095, 0.019129924476146698, 0.01209675706923008, 0.01568353734910488, -0.009796901606023312, 0.002290891483426094, -0.023121515288949013, -0.010173534974455833, 0.02377091906964779, -0.0008142684237100184, -0.015227137133479118, 0.008286702446639538, 0.00163277518004179, -0.04063376039266586, -0.013481185771524906, -0.005142058711498976, -0.016673900187015533, -0.013444862328469753, -0.011684255674481392, 0.0013830073876306415, -0.021670900285243988, -0.007984142750501633, 0.33416876196861267, -0.02525620348751545, 0.006119773257523775, -0.012490041553974152, -0.004652185365557671, -0.007337100803852081, 0.0198791716247797, -0.007237940561026335, 0.05746074020862579, -0.0028023081831634045, -0.0119901392608881, -0.025357849895954132, 0.008475934155285358, 0.007625544909387827, -0.037667226046323776, 0.01704198494553566, 0.007868117652833462, -0.0034773359075188637, -0.007853726856410503, -0.0149417445063591, -0.0024587230291217566, -0.005863334517925978, 0.013700678944587708, -0.012693984434008598, 0.0108892060816288, 0.01752074435353279, 0.00467616505920887, -0.012433722615242004, -0.0018585370853543282, -0.016240743920207024, -0.017074815928936005, 0.006378226913511753, 0.0013577841455116868, 0.020446516573429108, -0.0033468722831457853, -0.011825070716440678, 0.010864363051950932, 0.008347486145794392, 0.016735749319195747, 0.006414888892322779, 0.014517967589199543, 0.01433494407683611, 0.06226368993520737, -0.006459095049649477, 0.014456086792051792, 0.017357228323817253, -0.005355320405215025, -0.0061012255027890205, 0.02201804704964161, 0.008980336599051952, -0.010670348070561886, 0.017078295350074768, 0.017889192327857018, -0.016685795038938522, 0.0019211972830817103, 0.01250832062214613, 0.006634383928030729, -0.007912801578640938, -0.010777430608868599, -0.0008334440644830465, -0.015670835971832275, 0.002741033909842372, -0.012928190641105175, -0.015209043398499489, 0.006497716996818781, -0.02045467123389244, 0.008693002164363861, -0.015048360452055931, -0.02051592990756035, -0.012801828794181347, -0.03234605863690376, 0.0061966488137841225, -0.029697967693209648, 0.012873672880232334, 0.008042475208640099, 0.00249235681258142, -0.02681327797472477, -0.006218505557626486, 0.0023838875349611044, -0.013910944573581219, -0.011361274868249893, -0.006870596203953028, -0.013827506452798843, -0.019675496965646744, -0.008731377311050892, 0.0008519721450284123, -0.03159056976437569, -0.03021271526813507, -0.0012702919775620103, 0.01665169931948185, -0.008257153443992138, -0.012554322369396687, -0.0037748245522379875, -0.002242165617644787, -0.00987654272466898, 0.012540414929389954, -0.012028399854898453, -0.011181075125932693, -0.008702428080141544, 0.006331276148557663, 0.03287690877914429, 0.004710100591182709, 0.008778441697359085, 0.012768435291945934, -0.006662546191364527, -0.009911867789924145, -0.0029526851139962673, 0.0027662827633321285, -0.0074380310252308846, 0.0006906468188390136, 0.012737120501697063, 0.007024922873824835, 0.019508998841047287, -0.012021713890135288, -0.004836375825107098, 0.011839748360216618, 0.003161002416163683, -0.025066887959837914, -0.4614383280277252, 0.009823022410273552, 0.011508352123200893, 0.01961623877286911, -0.012913177721202374, -0.018389694392681122, 0.023879775777459145, 0.003382778028026223, 0.012954880483448505, 0.013056186027824879, -0.022757550701498985, -0.014389822259545326, -0.13360708951950073, 0.008020726963877678, -0.009561609476804733, 0.01249047089368105, -0.007314068730920553, -0.039817024022340775, -0.0033672431018203497, 0.03553428128361702, -0.010088102892041206, 0.007501672487705946, 0.013694900088012218, 0.01931573636829853, 0.01855592243373394, -0.009732229635119438, -0.005809564609080553, 0.011585433967411518, 0.01429519709199667, -0.006690122652798891, -0.0028659363742917776, -0.014654804952442646, 0.03632967546582222, -0.002242704387754202, -0.0054860590025782585, -0.02620028890669346, 0.006486218422651291, -0.001437075436115265, 0.01105553936213255, -0.021929092705249786, -0.00304270233027637, 0.007361790630966425, 0.01875358447432518, 0.016175897791981697, -0.03593708202242851, -0.007932188920676708, -0.02390826866030693, -0.1487477719783783, 0.007915066555142403, -0.008540965616703033, 0.020697196945548058, 0.010380338877439499, 0.005801997613161802, -0.007066246122121811, -0.01573854312300682, 0.0128252524882555, 0.0014352374710142612, 0.001978643937036395, 0.02433273196220398, -0.0028300343547016382, -0.020342422649264336, 0.0023019730579108, 0.0331723690032959, -0.0036248338874429464, -0.006594216916710138, 0.005808786954730749, -0.027304746210575104, -0.00448306230828166, -0.017432676628232002, -0.012258266098797321, -0.012353125959634781, 0.0008780425414443016, -0.020168092101812363, 0.01258349884301424, 0.012787376530468464, 0.022174689918756485, 0.007356088608503342, 0.0069221691228449345, -0.004574590362608433, 0.0054345387034118176, -0.02267969772219658, -0.013072079978883266, 0.03136757016181946, 0.02345333620905876, 0.004966772627085447, -0.006612925324589014, 0.0002566543116699904, 0.01614174246788025, -0.008638533763587475, -0.001567648141644895, 0.004388528876006603, 0.00034626221167854965, 0.0017592168878763914, -0.010418987832963467, -0.01391632854938507, 0.01290780957788229, -0.018717197701334953, 0.011687132529914379, -0.026618298143148422, -0.011934402398765087, -0.00843440555036068, 0.004824040457606316, 0.005430598743259907, -0.02847527526319027, -0.009919301606714725, -0.00917668268084526, 0.16299913823604584, -2.460641917423345e-05, -0.0008107532630674541, -0.007604830898344517, 0.00011412457388360053, 0.01219266839325428, -0.004266969859600067, 0.018414951860904694, 0.10997643321752548, 0.0003617087786551565, -0.00509458826854825, 0.011461232788860798, -0.008977331221103668, -0.005780109204351902, 0.007392391096800566, 0.02790789306163788, 0.00017634137475397438, 0.02216184511780739, -0.009241590276360512, 0.0109700383618474, 0.02226083166897297, -0.0160116758197546, 0.014771702699363232, -0.0019638314843177795, -0.011017861776053905, -0.004823542665690184, -0.01934359036386013, -0.024317627772688866, 0.021850235760211945, -0.021941708400845528, 0.0038709985092282295, 0.009520255960524082, -0.01045465748757124, 0.06117790937423706, -0.01899079792201519, -0.004709441214799881, 0.02597178891301155, 0.0026871042791754007, -0.0062805539928376675, -0.020114943385124207, -0.012094389647245407, -0.003307234263047576, -0.02400745078921318, 0.030807292088866234, -0.006871921010315418, 0.0021707569248974323, -0.019648533314466476, -0.010498392395675182, 0.0008480817778035998, -0.001588552026078105, -0.0022532478906214237, -0.024846123531460762, -0.013142578303813934, 0.009454789571464062, -0.018843019381165504, 0.008074194192886353, -0.04257078841328621, -0.00928663183003664, 0.015498642809689045, -0.008282048627734184, 0.007357089314609766, 0.009052460081875324, 0.0009136053849942982, 0.007408265024423599, -0.002800087444484234, 0.0007521833176724613, 0.014569565653800964, -0.014834681525826454, -0.0027385372668504715, -0.009614583104848862, -0.007187677547335625, 0.014914066530764103, -0.14526565372943878, 0.0006100988830439746, 3.2905195439525414e-06, -0.014082544483244419, -0.01601274125277996, -0.01736455038189888, 0.025464298203587532, 0.011672786436975002, -0.0019599103834480047, 0.0077719115652143955, 0.0018924195319414139, 0.11992714554071426, -0.014883646741509438, -0.008328277617692947, -0.010149556212127209, -0.005451436620205641, 0.006013126112520695, 0.013588346540927887, -0.007267612963914871, -0.02922242134809494, -0.021982882171869278, 0.11952237784862518, 0.004305231850594282, 0.016645777970552444, 0.017144009470939636, -0.0043621971271932125, -0.0027030049823224545, 0.0029774249996989965, 0.014306304045021534, -0.009607779793441296, -0.001422452973201871, 0.014896297827363014, -0.0037921862676739693]}, {"text": "You are a professional Python developer and data analyst.\n\nWrite code that:\n- reads all files with txt extension from the current working directory (these are books);\n- for each book, display a word cloud in a neat grid;\n- find the two most similar books as accurately as possible and display the result;\n- implement word preprocessing to solve these problems as correctly as possible.\n\nThe code should be as efficient and concise as possible. \n\nThen check your code for errors and correct them if necessary.", "label": "HIGH", "embedding": [0.032732270658016205, -0.001392580452375114, -0.0008776120957918465, -0.019797036424279213, -0.02071225270628929, -0.01397427637130022, 0.016620652750134468, -0.0073995888233184814, 0.006394912954419851, -0.08429377526044846, -0.02382824942469597, 0.0012837708927690983, 0.006723617669194937, -0.0007806746289134026, -0.04158041626214981, 0.00734165171161294, 0.014670937322080135, -0.0009752815240062773, -0.005979623179882765, -0.017601635307073593, 0.015230469405651093, -0.009917262010276318, 0.049660101532936096, -0.017470156773924828, 0.008302592672407627, -0.001624629134312272, 0.025489583611488342, 0.01129896566271782, -0.011973467655479908, 0.028172148391604424, 0.00044453819282352924, 0.021169133484363556, -0.0029544627759605646, -0.006823827978223562, -0.021010078489780426, 0.01016394142061472, -0.007514026947319508, -0.018792469054460526, -0.014685545116662979, -0.011440543457865715, -0.03301452472805977, 0.007305459585040808, 0.046548549085855484, -0.015697838738560677, -0.0056541007943451405, -0.02686305344104767, -0.004668164066970348, 0.01676640287041664, 0.009419670328497887, 0.01047518104314804, -0.0004332006792537868, -0.004346809815615416, 0.005667649209499359, 0.0151443462818861, 0.006213145796209574, -0.0014872885076329112, -0.004151480738073587, 0.007512174546718597, -0.01886146515607834, 0.012535901740193367, -0.004818020388484001, -0.010539875365793705, 0.020257793366909027, 0.0026189631316810846, -0.021288692951202393, 0.0029254888650029898, 0.0046872105449438095, 0.0032249963842332363, -0.01969590224325657, -0.004999887198209763, -0.0020287891384214163, 0.001139319036155939, 0.01914866827428341, 0.06009266898036003, 0.02137751318514347, -0.014255558140575886, 0.03628271073102951, 0.0023973460774868727, 0.0019151890883222222, 0.00062469148542732, 0.011787671595811844, 0.010810469277203083, -0.011567851528525352, 0.0007999634835869074, -0.005430176854133606, -0.011918492615222931, 0.017552336677908897, -0.009654697962105274, -0.004444767255336046, 0.02215336449444294, 0.007978745736181736, -0.00627146428450942, -0.007229777984321117, -0.006578929256647825, -0.005155934486538172, 0.005478505045175552, -0.002069938462227583, -0.003599452320486307, 0.41182124614715576, 0.0006781216361559927, -0.008221198804676533, 0.007121092639863491, 0.008906831964850426, -0.012283188290894032, 0.02722267061471939, -0.0191746074706316, -0.008422034792602062, -0.026830943301320076, 0.011681247502565384, 0.025530477985739708, 0.031038980931043625, -0.008647152222692966, 0.011639261618256569, -0.002818220527842641, 0.0069000315852463245, -0.006281233858317137, 0.010972997173666954, 0.010288025252521038, 0.010261046700179577, -0.002712931949645281, 0.007119164802134037, -0.002537449821829796, 0.06443534046411514, 0.006125344429165125, 0.0047857630997896194, -0.013773267157375813, 0.004947385750710964, -0.014807240106165409, 0.021301446482539177, -0.008626936003565788, -0.045600730925798416, -0.003955150488764048, -0.009614968672394753, -0.005304102320224047, 0.008328555151820183, 0.008415954187512398, -0.010419306345283985, 0.0026603105943650007, -0.039900362491607666, 0.009295756928622723, 0.0008917689556255937, -0.0247377697378397, -0.00542929582297802, -0.02128087915480137, -0.01378373522311449, 0.18733613193035126, -0.008422079496085644, -0.00724388612434268, -0.00897043664008379, -0.0019424958154559135, -0.009772676043212414, -0.02111320197582245, -0.019190533086657524, 0.004934749566018581, -0.01256105862557888, -0.01323375478386879, 0.010647039860486984, -0.005927267484366894, -0.016797546297311783, -0.007992868311703205, -0.06525037437677383, 0.0009375126101076603, -0.007862456142902374, 0.025849422439932823, -0.007039099931716919, -0.005645800847560167, 0.0014218161813914776, -0.04151077941060066, 0.0011594484094530344, 0.03055650182068348, 0.013227966614067554, -0.005709455348551273, -0.15677037835121155, 0.015814505517482758, 0.015191003680229187, 0.006003809627145529, 0.004187790676951408, -0.00391772948205471, 0.0008394576725549996, -0.007543763145804405, -0.022815130650997162, 0.007159125991165638, 0.006396957207471132, 0.004680739715695381, -0.02420666068792343, 0.005246797110885382, 0.007663372438400984, -0.0076994458213448524, 0.015476305969059467, 0.008856697008013725, 0.011213775724172592, -0.008245720528066158, 0.003160090884193778, 0.003724931739270687, 0.007198771461844444, -0.0009447194170206785, 0.01920301839709282, -0.010841818526387215, -0.0030987276695668697, -0.002097106771543622, -0.01245529018342495, -7.303972961381078e-05, -0.0018870312487706542, 0.00289120408706367, 0.020406564697623253, 0.005309820640832186, -0.004867970943450928, 0.01690206490457058, 0.020743291825056076, -0.0006492685060948133, -0.0081311771646142, -0.0016567722195759416, -0.005096208304166794, 0.02339642494916916, 0.03647664189338684, -0.012904783710837364, -0.006811431143432856, 0.005024233367294073, 0.00639649061486125, -0.0028374476823955774, -0.006247507408261299, 0.006187991704791784, -0.01488504372537136, -0.0019690131302922964, -0.00014788072439841926, 0.003415425308048725, 0.01422678492963314, -0.006348082330077887, 0.004294952843338251, -0.0137094771489501, 0.001034875400364399, 0.013869809918105602, 0.002926202956587076, -0.008757790550589561, -0.0009722207323648036, -0.0190729983150959, 0.0024535595439374447, 0.0012940536253154278, -0.0021669212728738785, 0.02083180658519268, -0.006941305007785559, -0.0009379085968248546, 0.028794042766094208, -0.010265383869409561, -0.0005420715315267444, 0.01795695535838604, 0.017364099621772766, 0.007664191070944071, -0.008249497972428799, -0.003707421710714698, 0.061074674129486084, 0.01711598038673401, -0.01687528006732464, -0.012870549224317074, -0.02399769425392151, 0.0006761978729628026, -0.00030671461718156934, -0.09646031260490417, -0.023278802633285522, -0.0047495500184595585, 0.011790989898145199, -0.18856768310070038, 0.004124174360185862, 0.010158592835068703, 0.00942336767911911, -0.016543826088309288, 0.0221080482006073, 0.003320016199722886, -0.015495057217776775, -0.0023014976177364588, -0.03539203479886055, 0.005722504574805498, -0.032027412205934525, -0.00014177327102515846, 0.003800577949732542, 0.01883377507328987, -0.007105870172381401, 0.010673913173377514, -0.0061312210746109486, 0.0008664530469104648, -0.025442838668823242, 0.004842058755457401, -0.02601492404937744, 0.03274323046207428, -0.015122488141059875, 0.01888875477015972, 0.009127217344939709, 0.01536165364086628, 0.006691036745905876, 0.25407853722572327, 0.009428336285054684, -0.002579713938757777, 0.022352850064635277, -0.024481330066919327, -0.004322129301726818, 0.006914897821843624, -0.03490087762475014, 0.016378499567508698, -0.008082750253379345, 0.005274488124996424, -0.008843131363391876, -0.004372385796159506, -0.02657277137041092, 0.004904853645712137, -0.030849507078528404, 0.0037701050750911236, -0.0024298657663166523, 0.0053367349319159985, -0.045713283121585846, 0.00016333254461642355, 0.031721215695142746, 0.004041912499815226, 0.00959846656769514, 0.0033744070678949356, -0.009088192135095596, -0.03410037234425545, -0.02126564271748066, 0.03198036924004555, 0.005501669831573963, -0.05952372029423714, 0.006202475633472204, -0.0031345104798674583, -0.0273741502314806, -0.0038737875875085592, 0.007444665767252445, 0.003512820228934288, -0.002196655375882983, -0.005579507909715176, -0.020858628675341606, -0.006384318694472313, -0.004771856125444174, 0.018028102815151215, -0.02085813134908676, 0.0024252901785075665, 0.004374645184725523, -0.0009755136561580002, -0.000516252126544714, 0.013341065496206284, 0.0024231208954006433, 0.006182785611599684, -0.009785993956029415, 0.005206659901887178, -0.020291635766625404, 0.01548200286924839, 0.014980022795498371, 0.008872032165527344, 0.011932175606489182, -0.005690422840416431, 0.01775158941745758, 0.0008468261221423745, -0.018216196447610855, -0.0024980229791253805, -0.007920539937913418, 0.0182171743363142, 0.02187010832130909, -0.0072097876109182835, -0.010041650384664536, -0.016395995393395424, 0.011125228367745876, 0.006889717653393745, 0.006138180382549763, -0.14322003722190857, 0.01462645549327135, 0.011836049146950245, -0.004808417055755854, 0.007291429676115513, -0.008980964310467243, 0.008080030791461468, -0.008504046127200127, 0.029654281213879585, 0.018520962446928024, 0.0029513754416257143, 0.014630693010985851, 0.013446887023746967, -0.021800413727760315, 0.004141977056860924, 0.00257085170596838, 0.0406280942261219, 0.014863087795674801, -0.022315191105008125, -0.021671531721949577, 0.010726152919232845, 0.011167292483150959, -0.010693368501961231, -0.006800828035920858, -0.0020422234665602446, -0.0022193712648004293, -0.02113325148820877, -0.0005036143120378256, 0.002921775449067354, 0.02823103964328766, -0.0007286420441232622, -0.011020482517778873, -0.011077110655605793, -0.007092249114066362, -0.012252910062670708, -0.01063743606209755, 0.016128284856677055, 0.018194258213043213, -0.014682246372103691, -0.010718107223510742, -0.0009213841403834522, -0.008084372617304325, -0.004665510728955269, -0.015584036707878113, 0.00421485910192132, 0.018309878185391426, -0.015611466951668262, -0.08869112282991409, -0.004873471334576607, -0.014783318154513836, -0.010722058825194836, -0.00570627162232995, 0.013618594966828823, 0.021373162046074867, -0.004459973890334368, 0.001267715124413371, -0.015258613042533398, 0.017688678577542305, 0.004090003669261932, 0.011045313440263271, 0.003848026040941477, 0.011027054861187935, -0.027501937001943588, 0.0006369936745613813, 0.015816407278180122, -0.004868984688073397, -0.019200898706912994, 0.00559211103245616, -0.0021571102552115917, -0.05196228250861168, -0.015921365469694138, 0.01219891756772995, -0.019693523645401, -0.020848898217082024, -0.021029075607657433, 0.005670684855431318, -0.007328446488827467, 0.004175282083451748, 0.34338831901550293, -0.015268171206116676, 0.01901041902601719, -0.0027801457326859236, -0.014403686858713627, 0.0033961646258831024, 0.014419538900256157, 0.00042031900375150144, 0.05522884055972099, 0.007992373779416084, -0.023764077574014664, -0.02164766564965248, 0.007887511514127254, 0.0008350171847268939, -0.017290007323026657, 0.02684425376355648, 0.0021827942691743374, -0.01034014392644167, 0.0016845106147229671, 0.00850075762718916, 0.006136463489383459, -0.011942736804485321, 0.011660267598927021, -0.019217483699321747, 0.017220746725797653, 0.012838589958846569, 0.005767268594354391, 0.00036328835994936526, -0.006326344329863787, -0.018631145358085632, -0.02094995602965355, 0.012211127206683159, 0.025819504633545876, 0.01049082912504673, 0.00025143363745883107, -0.00823915097862482, 0.00934404507279396, 0.0010407945374026895, 0.008519642055034637, 0.02151450142264366, 0.018504444509744644, -0.0030676773749291897, 0.05844160169363022, 0.0013531704898923635, 0.020518802106380463, 0.017287395894527435, 0.0005133055965416133, -0.012470039539039135, 0.02124164253473282, 0.00272346381098032, -0.0016545846592634916, 0.023522164672613144, 0.0075057679787278175, -0.012690523639321327, 0.008895245380699635, 0.002820811700075865, 0.002316024387255311, -0.0020950480829924345, 2.3939308448461816e-05, 0.010338949970901012, 0.003580415854230523, -0.01314419973641634, -0.007272879593074322, -0.015251991339027882, 0.008559169247746468, -0.0034654918126761913, 0.001627915888093412, -0.018863258883357048, -0.023351963609457016, -0.006876007653772831, -0.006745068822056055, 0.012902967631816864, -0.024199403822422028, 0.01315145380795002, 0.01704074628651142, 0.008813329041004181, -0.02857677824795246, 0.0017795704770833254, -0.0046593546867370605, -0.020205382257699966, -0.007364429999142885, -0.022506574168801308, -0.004266205243766308, -0.014368502423167229, 0.006077993661165237, -0.007778868544846773, -0.0338706448674202, -0.020801424980163574, -0.0032948791049420834, 0.0038966627325862646, 0.0013354028342291713, -0.004057361278682947, -0.01116037555038929, 0.0004979542572982609, -0.0014924905262887478, 0.02228676900267601, 0.00030853820499032736, 0.0032361503690481186, -0.007311032619327307, 0.0005657610017806292, 0.01570758782327175, 0.0065332623198628426, -0.004962607752531767, -0.00043092138366773725, 0.004675432574003935, -0.0050036306492984295, -0.01355694979429245, 0.012455311603844166, -0.010332843288779259, -0.006332680117338896, 0.01629943959414959, 0.00022889582032803446, 0.019646810367703438, -0.00645048450678587, 0.00018199885380454361, 0.021935712546110153, -0.000669611559715122, -0.024545805528759956, -0.43374237418174744, 0.01338146347552538, -0.00577061390504241, 0.010791081003844738, -0.030768774449825287, -0.000248340074904263, 0.028682900592684746, 0.016294529661536217, 0.02170475758612156, 0.004017048515379429, -0.019540751352906227, -0.006580394227057695, -0.13620400428771973, 0.01912953518331051, -0.022863149642944336, -0.0014566757017746568, -0.018966596573591232, -0.06582166254520416, -0.010411419905722141, 0.03066517785191536, -0.005918117240071297, 0.025835353881120682, 0.01267832238227129, 0.009197103790938854, 0.018832538276910782, 0.005453452467918396, 0.005494879558682442, -0.003756850492209196, 0.006762545555830002, -0.0054822345264256, 0.0004532787424977869, -0.005451512522995472, 0.026262624189257622, 0.006062537431716919, 5.2028499339940026e-05, -0.0081722941249609, -0.0064022960141301155, -0.0028097990434616804, 0.008897855877876282, -0.01511690765619278, -0.009734614752233028, -0.0009863842278718948, 0.017546869814395905, 0.0075432173907756805, -0.043936505913734436, -0.009013839066028595, -0.032600127160549164, -0.14329268038272858, 0.00488486560061574, -0.013245992362499237, 0.012149952352046967, 0.007384683471173048, -0.011745926924049854, -0.016930073499679565, -0.020095206797122955, -0.0014918777160346508, 0.0059597003273665905, 0.0061284457333385944, 0.01760634034872055, -0.019487448036670685, -0.01537919882684946, -0.01407971978187561, 0.030047781765460968, 0.005343243945389986, -0.019699424505233765, 0.003926097881048918, -0.028690986335277557, -0.009443176910281181, -0.021185321733355522, -0.016413919627666473, -0.003658981528133154, -0.013958023861050606, -0.00257857795804739, 0.004975579679012299, 0.021740838885307312, 0.02140122465789318, 0.012838726863265038, -0.011973136104643345, 0.004580433014780283, -0.002931915456429124, -0.03209826350212097, -0.016737569123506546, 0.023438503965735435, 0.013478938490152359, 0.01442253403365612, -0.0067999050952494144, 0.003133262973278761, 0.0004907906986773014, -0.017385315150022507, -0.0002390776644460857, -0.0013594918418675661, 0.010029762051999569, -6.668922651442699e-06, -0.01538124494254589, -0.000784639734774828, 0.002918561454862356, -0.018107565119862556, -0.002243755152449012, -0.02875298261642456, -0.002286843489855528, -0.007222936954349279, 0.017300134524703026, 0.006860439665615559, -0.031133407726883888, -0.005813203752040863, -0.017268402501940727, 0.17043600976467133, 0.010559913702309132, -0.0057410565204918385, 0.0013125024270266294, -0.012719777412712574, -0.0012119015445932746, 0.00634392537176609, 0.0026553927455097437, 0.11043155193328857, 0.0013203280977904797, -0.019190805032849312, 0.00838405080139637, -0.00188065052498132, 0.002253645798191428, -0.000574819277971983, 0.03280145674943924, -0.002098045777529478, 0.011521426029503345, -0.0121723972260952, 0.009705677628517151, 0.015630444511771202, -0.007735606282949448, 0.00569238280877471, -0.0037131605204194784, 0.007841743528842926, -5.696633525076322e-05, -0.01864611543715, -0.003599918447434902, 0.01736084744334221, -0.023667721077799797, 0.005330719519406557, -0.010251360014081001, -0.01295823510736227, 0.06270076334476471, -0.009333936497569084, 0.011737006716430187, 0.013586423359811306, -0.0033054028172045946, -0.0005970350466668606, -0.029645636677742004, -0.012136433273553848, -0.0026931853499263525, -0.03559974208474159, 0.032441310584545135, -0.0003459737345110625, 0.00295159756205976, -0.004855248611420393, -0.009727121330797672, 0.0031688339076936245, -0.007183929439634085, -0.012171517126262188, -0.027038224041461945, -0.009694920852780342, 0.00845372211188078, 0.0038547960575670004, -0.01620481163263321, -0.0452495738863945, -0.018863722681999207, 0.010479321703314781, -0.013060204684734344, 0.004159659147262573, -0.00302547006867826, -0.0029191672801971436, -0.0022642319090664387, -0.007219057064503431, 0.0019106592517346144, 0.008937021717429161, -0.025046739727258682, -0.005377864930778742, -0.015469738282263279, 0.003018368501216173, 0.010102585889399052, -0.14894463121891022, 0.0014075853396207094, 0.00414086552336812, -0.02763873152434826, -0.013417866080999374, -0.019957050681114197, 0.012801596894860268, 0.00011234940029680729, 0.008322950452566147, 0.012228051200509071, -0.0014867965364828706, 0.12239447981119156, -0.0001346774079138413, -0.010775008238852024, -0.016029132530093193, -0.005007494240999222, 0.014944982714951038, -5.520668128156103e-05, 0.002162717515602708, -0.02308199182152748, -0.011082983575761318, 0.1109391376376152, 0.0034826139453798532, 0.02676858939230442, 0.0037683569826185703, -0.013597832061350346, 4.714470196631737e-05, 0.009496408514678478, -0.0024041917640715837, 0.007655952125787735, -0.012766871601343155, 0.0074783796444535255, 0.01734108291566372]}]}} \ No newline at end of file diff --git a/scripts/adaptive_router/tensors.safetensors b/scripts/adaptive_router/tensors.safetensors deleted file mode 100644 index 8308c1e..0000000 Binary files a/scripts/adaptive_router/tensors.safetensors and /dev/null differ diff --git a/scripts/docker_smoke.py b/scripts/docker_smoke.py new file mode 100644 index 0000000..763f23e --- /dev/null +++ b/scripts/docker_smoke.py @@ -0,0 +1,101 @@ +"""Smoke-test the serving image: save a classifier, mount it, query the container. + + docker build -f docker/Dockerfile -t adaptive-classifier . + python scripts/docker_smoke.py adaptive-classifier + +Uses a tiny randomly initialised encoder, so it needs no Hugging Face access, and +starts the container from the saved directory alone (ONNX encoder + tokenizer). +""" + +import json +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request +from pathlib import Path + +WORDS = "great terrible okay product service love hate awful fine good bad slow fast cheap broke works the a is it very".split() +PORT = 18765 + + +def build_model(directory: Path) -> Path: + import torch + from transformers import BertConfig, BertModel, BertTokenizerFast + + from adaptive_classifier import AdaptiveClassifier + + base = directory / "base" + base.mkdir() + vocab = ["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"] + WORDS + (base / "vocab.txt").write_text("\n".join(vocab), encoding="utf-8") + torch.manual_seed(0) + BertModel(BertConfig(vocab_size=len(vocab), hidden_size=32, num_hidden_layers=2, + num_attention_heads=2, intermediate_size=64)).save_pretrained(base) + BertTokenizerFast(str(base / "vocab.txt"), do_lower_case=True).save_pretrained(base) + + clf = AdaptiveClassifier(str(base), device="cpu") + clf.add_examples( + ["great love good", "love fine good", "works great fast", "good great love", + "terrible awful bad", "hate broke bad", "awful slow hate", "bad terrible broke"], + ["positive"] * 4 + ["negative"] * 4, + ) + model = directory / "model" + clf.save(str(model)) + return model + + +def call(path: str, body: dict | None = None): + request = urllib.request.Request( + f"http://127.0.0.1:{PORT}{path}", + data=None if body is None else json.dumps(body).encode(), + headers={"content-type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read()) + + +def main(image: str) -> int: + with tempfile.TemporaryDirectory() as tmp: + model = build_model(Path(tmp)) + # The image runs as uid 1000; make the read-only mount readable by it. + subprocess.run(["chmod", "-R", "a+rX", str(model)], check=True) + container = subprocess.run( + ["docker", "run", "-d", "--rm", "-p", f"127.0.0.1:{PORT}:8000", + "-v", f"{model}:/model:ro", image], + check=True, capture_output=True, text=True, + ).stdout.strip() + try: + deadline = time.time() + 180 + while True: + try: + status, health = call("/health") + break + except (urllib.error.URLError, ConnectionError, OSError): + if time.time() > deadline: + raise SystemExit("server did not become healthy:\n" + subprocess.run( + ["docker", "logs", container], capture_output=True, text=True).stdout) + time.sleep(2) + assert status == 200, health + status, result = call("/predict", {"text": "great love"}) + assert status == 200 and result, result + print("health:", health) + print("predict:", result) + + # Updates must be refused unless the server was started with --allow-updates. + try: + call("/examples", {"texts": ["x"], "labels": ["y"]}) + raise SystemExit("update was accepted on a read-only server") + except urllib.error.HTTPError as error: + assert error.code == 403, error.code + print("OK") + return 0 + finally: + logs = subprocess.run(["docker", "logs", container], capture_output=True, text=True) + print(logs.stdout[-2000:], logs.stderr[-2000:]) + subprocess.run(["docker", "rm", "-f", container], capture_output=True) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1] if len(sys.argv) > 1 else "adaptive-classifier")) diff --git a/src/adaptive_classifier/__init__.py b/src/adaptive_classifier/__init__.py index cd2df02..04e76d5 100644 --- a/src/adaptive_classifier/__init__.py +++ b/src/adaptive_classifier/__init__.py @@ -5,7 +5,7 @@ from .sklearn_api import SklearnAdaptiveClassifier from huggingface_hub import ModelHubMixin -__version__ = "0.3.0" +__version__ = "0.3.1" __all__ = [ "AdaptiveClassifier", diff --git a/src/adaptive_classifier/classifier.py b/src/adaptive_classifier/classifier.py index 24a2d91..81b8f7a 100644 --- a/src/adaptive_classifier/classifier.py +++ b/src/adaptive_classifier/classifier.py @@ -3,7 +3,7 @@ import torch.nn.functional as F import numpy as np from transformers import AutoModel, AutoTokenizer -from typing import List, Dict, Optional, Tuple, Any, Set, Union +from typing import Sequence, List, Dict, Optional, Tuple, Any, Set, Union import logging import asyncio import copy @@ -77,6 +77,12 @@ def _lock(self) -> "threading.RLock": lock = self.__dict__.setdefault('_instance_lock', threading.RLock()) return lock + def __getstate__(self): + # A lock cannot be copied or pickled; a copy gets its own, created on first use. + state = self.__dict__.copy() + state.pop('_instance_lock', None) + return state + def __init__( self, model_name: str, @@ -198,6 +204,25 @@ def _validate_training_inputs(texts: List[str], labels: List[str]): f"labels must be strings; got {type(bad_label).__name__}. " "Convert with str(label)." ) + # A blank text has no content to learn from but still counts as an example of its + # class (and moves the prototype), and a blank label is a class with no name. + blank = next((i for i, t in enumerate(texts) if not t.strip()), None) + if blank is not None: + raise ValueError(f"texts must not be empty or blank (text at index {blank} is)") + blank = next((i for i, l in enumerate(labels) if not l.strip()), None) + if blank is not None: + raise ValueError(f"labels must not be empty or blank (label at index {blank} is)") + + def _base_model_name(self) -> str: + """The encoder this classifier was built on (not the folder an ONNX copy was loaded from).""" + return getattr(self, '_model_name', None) or self.model.config._name_or_path + + def _extra_state(self) -> Dict[str, Any]: + """Subclass settings that `save` stores in config.json.""" + return {} + + def _restore_extra_state(self, state: Dict[str, Any]): + """Counterpart of `_extra_state`, called by `load`.""" @staticmethod def _validate_k(k: int): @@ -213,6 +238,60 @@ def add_examples(self, texts: List[str], labels: List[str]): raise ValueError("Mismatched text and label lists") self._validate_training_inputs(texts, labels) + # Encode first: the usual failures (tokenizer, encoder, memory) then happen before any + # state has changed. If anything fails later, every structure is put back as it was. + embeddings = self._get_embeddings(texts) + snapshot = self._snapshot() + try: + self._add_examples_unchecked(texts, labels, embeddings) + except BaseException: + self._restore(snapshot) + raise + + def _snapshot(self) -> Dict[str, Any]: + memory = self.memory + return { + 'label_to_id': dict(self.label_to_id), + 'id_to_label': dict(self.id_to_label), + 'training_history': dict(self.training_history), + 'train_steps': self.train_steps, + 'calibration': self.calibration, + 'head': copy.deepcopy(self.adaptive_head), + 'examples': {label: list(items) for label, items in memory.examples.items()}, + 'prototypes': {label: proto.clone() for label, proto in memory.prototypes.items()}, + 'saved_radii': dict(memory.saved_radii), + 'prototype_base': dict(memory.prototype_base), + 'post_sum': {label: v.clone() for label, v in memory.post_sum.items()}, + 'post_n': dict(memory.post_n), + 'updates_since_rebuild': memory.updates_since_rebuild, + 'just_rebuilt': getattr(memory, 'just_rebuilt', False), + } + + def _restore(self, snapshot: Dict[str, Any]): + memory = self.memory + self.label_to_id = snapshot['label_to_id'] + self.id_to_label = snapshot['id_to_label'] + self.training_history = snapshot['training_history'] + self.train_steps = snapshot['train_steps'] + self.calibration = snapshot['calibration'] + self.adaptive_head = snapshot['head'] + memory.examples.clear() + memory.examples.update(snapshot['examples']) + memory.prototypes.clear() + memory.prototypes.update(snapshot['prototypes']) + memory.saved_radii.clear() + memory.saved_radii.update(snapshot['saved_radii']) + memory.prototype_base.clear() + memory.prototype_base.update(snapshot['prototype_base']) + memory.post_sum.clear() + memory.post_sum.update(snapshot['post_sum']) + memory.post_n.clear() + memory.post_n.update(snapshot['post_n']) + memory.updates_since_rebuild = snapshot['updates_since_rebuild'] + memory.just_rebuilt = snapshot['just_rebuilt'] + memory._rebuild_index() + + def _add_examples_unchecked(self, texts: List[str], labels: List[str], embeddings: List[torch.Tensor]): # Check if classifier has any existing classes (before updating mappings) has_existing_classes = len(self.label_to_id) > 0 @@ -230,9 +309,6 @@ def add_examples(self, texts: List[str], labels: List[str]): self.label_to_id[label] = idx self.id_to_label[idx] = label - # Get embeddings for all texts - embeddings = self._get_embeddings(texts) - # Add examples to memory and update training history for text, embedding, label in zip(texts, embeddings, labels): example = Example(text, label, embedding) @@ -419,7 +495,7 @@ def _train_new_classes(self, old_head: Optional[nn.Module], new_classes: Set[str # Add EWC loss if applicable if ewc is not None: - ewc_loss = ewc.ewc_loss(batch_size=len(batch_embeddings)) + ewc_loss = ewc.ewc_loss(batch_size=len(batch_embeddings), model=self.adaptive_head) loss = task_loss + ewc_loss else: loss = task_loss @@ -504,14 +580,16 @@ def predict( # If strategic mode is not enabled, use regular prediction # Calibration rescales the whole distribution, so it needs every class. - internal_k = len(self.id_to_label) if self.calibration else k + # The strategic blend renormalises over the classes it was given, so it needs all of them too. + internal_k = len(self.id_to_label) if (self.calibration or self.strategic_mode) else k if not self.strategic_mode: predictions = self._predict_regular(text, internal_k) else: # Dual prediction system: blend strategic and regular predictions predictions = self._predict_dual(text, internal_k) if self.calibration: - predictions = self._calibrate(predictions)[:k] + predictions = self._calibrate(predictions) + predictions = predictions[:k] if abstain_below is not None and (not predictions or predictions[0][1] < abstain_below): return [] @@ -856,6 +934,8 @@ def remove_examples( are present, which shifts it more than removing from a live classifier would. """ + if isinstance(texts, str): + raise ValueError("texts must be a list of strings, not a single string") if not texts: raise ValueError("Empty input list") if label is not None and label not in self.label_to_id: @@ -1039,7 +1119,7 @@ def _save_pretrained( # Save configuration and metadata config_dict = { - 'model_name': self.model.config._name_or_path, + 'model_name': self._base_model_name(), 'embedding_dim': self.embedding_dim, 'label_to_id': self.label_to_id, 'id_to_label': {str(k): v for k, v in self.id_to_label.items()}, @@ -1048,6 +1128,7 @@ def _save_pretrained( 'ood_radii': {label: self.memory.class_radius(label) for label in self.memory.prototypes}, 'calibration': self.calibration, 'config': saved_settings, + **self._extra_state(), 'library_name': 'adaptive-classifier' # Tell HuggingFace Hub this requires the adaptive-classifier library } @@ -1290,6 +1371,14 @@ def _from_pretrained( with open(model_path / "config.json", "r", encoding="utf-8") as f: config_dict = json.load(f) + # A saved multi-label classifier loaded through the base class would come back as a + # single-label one: its sigmoid head read as softmax logits, and no thresholds. + if cls is AdaptiveClassifier and 'multilabel' in config_dict: + from .multilabel import MultiLabelAdaptiveClassifier + return MultiLabelAdaptiveClassifier._from_pretrained( + str(model_path), use_onnx=use_onnx, prefer_quantized=prefer_quantized, + trust_remote_code=trust_remote_code, **kwargs) + # Classifiers saved before 0.2.0 were embedded with CLS pooling, and # their stored config has no 'pooling' key. Reloading them under the new # 'auto' default would re-embed every prototype differently and silently @@ -1381,6 +1470,7 @@ def _from_pretrained( classifier.tokenizer = AutoTokenizer.from_pretrained(tokenizer_source, trust_remote_code=trust_remote_code) # Initialize memory and other components + classifier._model_name = config_dict['model_name'] classifier.embedding_dim = classifier.model.config.hidden_size classifier.memory = PrototypeMemory( classifier.embedding_dim, @@ -1434,6 +1524,7 @@ def _from_pretrained( # Restore training history with backward compatibility classifier.training_history = config_dict.get('training_history', {}) classifier.calibration = config_dict.get('calibration') + classifier._restore_extra_state(config_dict) # Load tensors tensors = load_file(model_path / "model.safetensors") @@ -1466,6 +1557,15 @@ def _from_pretrained( classifier._initialize_adaptive_head() classifier.adaptive_head.load_state_dict(adaptive_head_params) + # Prototypes were averaged over every example a class was trained on, but only a few + # are kept. Remember the real counts so later additions move a prototype by their + # proper share (only when the count was saved; the estimate below is too rough). + if classifier.training_history: + cap = classifier.config.max_examples_per_class + for label in classifier.memory.prototypes: + classifier.memory.set_prototype_base( + label, min(classifier.training_history.get(label, 0), cap)) + # Backward compatibility: estimate training history if not present if not classifier.training_history: for label, examples in saved_examples.items(): @@ -1506,7 +1606,7 @@ def _generate_model_card(self) -> str: ## Model Details -- Base Model: {self.model.config._name_or_path} +- Base Model: {self._base_model_name()} - Number of Classes: {stats['num_classes']} - Total Examples: {stats['total_examples']} - Embedding Dimension: {self.embedding_dim} @@ -1629,7 +1729,7 @@ def export_onnx( return save_directory # Get the base model name - model_name = self.model.config._name_or_path + model_name = self._base_model_name() logger.info(f"Exporting {model_name} to ONNX format...") @@ -1774,6 +1874,7 @@ def load(cls, save_dir: str, device: Optional[str] = None, use_onnx: Optional[Un kwargs['device'] = device return cls._from_pretrained(save_dir, use_onnx=use_onnx, prefer_quantized=prefer_quantized, trust_remote_code=trust_remote_code, **kwargs) + @_synchronized def to(self, device: str) -> 'AdaptiveClassifier': """Move the model to specified device. @@ -1890,34 +1991,39 @@ def _get_embeddings(self, texts: List[str]) -> List[torch.Tensor]: was_training = self.model.training self.model.eval() - # Get embeddings + # Encode in chunks of `batch_size`: one forward pass over a long list needs memory + # proportional to the whole list. + chunk = max(1, int(getattr(self.config, 'batch_size', 32) or 32)) + result = [] with torch.no_grad(): - inputs = self.tokenizer( - texts, - max_length=self.config.max_length, - truncation=True, - padding=True, - return_tensors="pt" - ) + for start in range(0, len(texts), chunk): + inputs = self.tokenizer( + texts[start:start + chunk], + max_length=self.config.max_length, + truncation=True, + padding=True, + return_tensors="pt" + ) - # For ONNX models, inputs don't need to be moved to device - if not self.use_onnx: - inputs = inputs.to(self.device) + # For ONNX models, inputs don't need to be moved to device + if not self.use_onnx: + inputs = inputs.to(self.device) - outputs = self.model(**inputs) - embeddings = self._pool( - outputs.last_hidden_state, inputs.get('attention_mask') - ) + outputs = self.model(**inputs) + embeddings = self._pool( + outputs.last_hidden_state, inputs.get('attention_mask') + ) - # Normalize embeddings - embeddings = F.normalize(embeddings, p=2, dim=1) + # Normalize embeddings + embeddings = F.normalize(embeddings, p=2, dim=1) + result.extend(emb.cpu() for emb in embeddings) # Restore original training mode (only for PyTorch models) if was_training and hasattr(self.model, 'train'): self.model.train() # Return embeddings as list - return [emb.cpu() for emb in embeddings] + return result @_synchronized def get_example_statistics(self) -> Dict[str, Any]: @@ -1956,6 +2062,11 @@ def predict_batch( raise ValueError("Empty input batch") self._validate_k(k) + if self.strategic_mode: + # The strategic blend is worked out per text; batching the embeddings + # would skip it and return different answers from `predict`. + return [self.predict(text, k) for text in texts] + all_predictions = [] # Process in batches @@ -2036,39 +2147,76 @@ def clear_memory(self, labels: Optional[List[str]] = None): classes are ignored. """ if labels is None: + # Everything goes, not just the memory: a head left behind would keep predicting. + if self.calibration: + self.calibration = None self.memory.clear() + self.label_to_id = {} + self.id_to_label = {} + self.training_history = {} + self.adaptive_head = None return known = [label for label in labels if label in self.label_to_id] if known: self.forget(known) - @_synchronized def merge_classifiers(self, other: 'AdaptiveClassifier') -> 'AdaptiveClassifier': - """Merge another classifier into this one.""" - # Verify compatibility + """Merge another classifier into this one and return this one. + + The other classifier's examples (copied, so it is left untouched) and + training counts are added, and the head is retrained on the combined + memory. Both must use the same encoder embedding size. A classifier + loaded from disk holds only a few representative examples per class, + so merging from one contributes only those. + """ + if other is self: + return self if self.embedding_dim != other.embedding_dim: raise ValueError("Classifiers have different embedding dimensions") - - # Merge label mappings - next_idx = max(self.id_to_label.keys()) + 1 - for label in other.label_to_id: + + # Read `other` under its own lock and release it before taking ours, so two + # threads merging a into b and b into a cannot wait on each other forever. + with other._lock: + incoming = {label: [copy.copy(ex) for ex in examples] + for label, examples in other.memory.examples.items()} + incoming_history = dict(other.training_history) + incoming_labels = list(other.label_to_id) + incoming_radii = {label: other.memory.class_radius(label) + for label in other.memory.prototypes} + + with self._lock: + self._merge_snapshot(incoming, incoming_history, incoming_labels, incoming_radii) + return self + + def _merge_snapshot(self, incoming, incoming_history, incoming_labels, incoming_radii): + if self.calibration: + logger.warning("Discarding confidence calibration: the set of classes changed") + self.calibration = None + + next_idx = max(self.id_to_label, default=-1) + 1 + for label in incoming_labels: if label not in self.label_to_id: self.label_to_id[label] = next_idx self.id_to_label[next_idx] = label next_idx += 1 - - # Merge examples and update prototypes - for label, examples in other.memory.examples.items(): + + for label, examples in incoming.items(): for example in examples: self.memory.add_example(example, label) - - # Retrain adaptive head - if self.adaptive_head is not None: + for label, count in incoming_history.items(): + self.training_history[label] = self.training_history.get(label, 0) + count + # The other classifier's recorded spread (from examples we no longer see) still applies. + for label, radius in incoming_radii.items(): + self.memory.saved_radii[label] = max(self.memory.saved_radii.get(label, 0.0), radius) + + # add_example only rebuilds the search index every few updates; searches + # would otherwise miss the new classes. + self.memory._rebuild_index() + + if self.memory.examples and self.label_to_id: self._initialize_adaptive_head() self._train_adaptive_head() - - return self - + def _train_adaptive_head(self, epochs: int = 10): """Train the adaptive head on every stored example. @@ -2146,6 +2294,8 @@ def select_representative_examples(self, examples: List[Example], k: int = 5) -> Returns: List of selected examples """ + if k <= 0: + return [] if len(examples) <= k: return examples @@ -2177,26 +2327,44 @@ def select_representative_examples(self, examples: List[Example], k: int = 5) -> return [examples[idx] for idx in selected_indices] def _initialize_strategic_components(self): - """Initialize strategic classification components.""" + """Initialize strategic classification components. + + `cost_coefficients` must hold one cost per embedding dimension. When it + does not (or the cost function type is unknown) strategic mode is + switched off and the reason is logged, so a mistake never silently + leaves a classifier unprotected without saying so. + """ try: - # Create cost function from config - if self.config.cost_coefficients: - self.strategic_cost_function = CostFunctionFactory.create_cost_function( - cost_type=self.config.cost_function_type, - cost_coefficients=self.config.cost_coefficients + coefficients = self.config.cost_coefficients + if isinstance(coefficients, dict): + raise ValueError( + "cost_coefficients must be a list with one cost per embedding dimension " + f"({self.embedding_dim} for this encoder); a dict of named features is not supported" ) - - # Initialize strategic optimizer and evaluator - self.strategic_optimizer = StrategicOptimizer(self.strategic_cost_function) - self.strategic_evaluator = StrategicEvaluator(self.strategic_cost_function) - - logger.info(f"Initialized strategic mode with {self.config.cost_function_type} cost function") - else: - logger.warning("Strategic mode enabled but no cost coefficients provided") + if coefficients is None or len(coefficients) == 0: + raise ValueError( + "Strategic mode needs cost_coefficients: a list with one cost per " + f"embedding dimension ({self.embedding_dim} for this encoder)" + ) + if len(coefficients) != self.embedding_dim: + raise ValueError( + f"cost_coefficients has {len(coefficients)} entries but the encoder's " + f"embeddings have {self.embedding_dim} dimensions" + ) + self.strategic_cost_function = CostFunctionFactory.create_cost_function( + cost_type=self.config.cost_function_type, + cost_coefficients=coefficients + ) + self.strategic_optimizer = StrategicOptimizer(self.strategic_cost_function) + self.strategic_evaluator = StrategicEvaluator(self.strategic_cost_function) + logger.info(f"Initialized strategic mode with {self.config.cost_function_type} cost function") except Exception as e: - logger.error(f"Failed to initialize strategic components: {e}") + logger.error(f"Strategic mode disabled: {e}") + self.strategic_cost_function = None + self.strategic_optimizer = None + self.strategic_evaluator = None self.config.enable_strategic_mode = False - + @property def strategic_mode(self) -> bool: """Check if strategic mode is enabled and properly initialized.""" @@ -2266,6 +2434,9 @@ def predict_strategic(self, text: str, k: int = 5) -> List[Tuple[str, float]]: Returns: List of (label, confidence) tuples for strategic predictions """ + if not text: + raise ValueError("Empty input text") + self._validate_k(k) if not self.strategic_mode: return self._predict_regular(text, k) @@ -2314,6 +2485,9 @@ def predict_robust(self, text: str, k: int = 5) -> List[Tuple[str, float]]: Returns: List of (label, confidence) tuples for robust predictions """ + if not text: + raise ValueError("Empty input text") + self._validate_k(k) if not self.strategic_mode: return self._predict_regular(text, k) @@ -2346,26 +2520,21 @@ def _predict_from_embedding( Returns: List of (label, confidence) tuples """ + # Score every class and trim at the end, so a score does not depend on k. + num_classes = len(self.id_to_label) with torch.no_grad(): - # Get prototype predictions - proto_preds = self.memory.get_nearest_prototypes(embedding, k=k) - - # Get neural predictions if available + proto_preds = self.memory.get_nearest_prototypes(embedding, k=max(num_classes, 1)) + if self.adaptive_head is not None: self.adaptive_head.eval() input_embedding = embedding.unsqueeze(0).to(self.device) - logits = self.adaptive_head(input_embedding) - logits = logits.squeeze(0) + logits = self.adaptive_head(input_embedding).squeeze(0) probs = F.softmax(logits, dim=0) - - values, indices = torch.topk(probs, min(k, len(self.id_to_label))) - head_preds = [ - (self.id_to_label[idx.item()], val.item()) - for val, idx in zip(values, indices) - ] + head_preds = [(self.id_to_label[i], probs[i].item()) for i in range(len(probs)) + if i in self.id_to_label] else: head_preds = [] - + # Combine predictions with strategic adjustments combined_scores = {} @@ -2404,16 +2573,34 @@ def _predict_from_embedding( return predictions[:k] + @_synchronized def evaluate_strategic_robustness( self, test_texts: List[str], test_labels: List[str], - gaming_levels: List[float] = [0.0, 0.5, 1.0] + gaming_levels: Sequence[float] = (0.0, 0.5, 1.0) ) -> Dict[str, float]: - """Evaluate strategic robustness of the classifier.""" + """Evaluate strategic robustness of the classifier. + + `gaming_levels` are the shares (0 to 1) of inputs that game the classifier. + The 0.0 and 1.0 baselines are always included, since the robustness + scores compare them. + """ if not self.strategic_mode: raise ValueError("Strategic mode not enabled") - + if not gaming_levels or any( + isinstance(level, bool) or not isinstance(level, numbers.Real) or not 0.0 <= level <= 1.0 + for level in gaming_levels + ): + raise ValueError(f"gaming_levels must be numbers between 0 and 1; got {list(gaming_levels)!r}") + if not test_texts or len(test_texts) != len(test_labels): + raise ValueError("test_texts and test_labels must be non-empty and the same length") + unknown = sorted(set(test_labels) - set(self.label_to_id)) + if unknown: + raise ValueError(f"Unknown labels: {unknown}") + if self.adaptive_head is None: + raise ValueError("The classifier has no trained head to evaluate") + # Get test embeddings test_embeddings = torch.stack(self._get_embeddings(test_texts)) diff --git a/src/adaptive_classifier/ewc.py b/src/adaptive_classifier/ewc.py index 2fb9774..b0998c4 100644 --- a/src/adaptive_classifier/ewc.py +++ b/src/adaptive_classifier/ewc.py @@ -93,24 +93,37 @@ def _compute_fisher( return fisher - def ewc_loss(self, batch_size: Optional[int] = None) -> torch.Tensor: - """Compute EWC loss. - + def ewc_loss( + self, + batch_size: Optional[int] = None, + model: Optional[nn.Module] = None + ) -> torch.Tensor: + """Compute the EWC penalty. + Args: batch_size: Batch size for normalization - + model: The model being trained. The penalty measures how far *its* + parameters have moved from the ones stored when this object was + built; with no model it uses the one given at construction, which + is that same model and so always gives zero. When classes have + been added the live model's output layer is larger; only the rows + that existed before are compared. + Returns: EWC loss tensor """ + live = self.model if model is None else model loss = 0 - for n, p in self.model.named_parameters(): - if p.requires_grad: - # Compute squared distance - _loss = (self.fisher_info[n] * (p - self.old_params[n]) ** 2).sum() + for n, p in live.named_parameters(): + if p.requires_grad and n in self.old_params: + old = self.old_params[n].to(p.device) + fisher = self.fisher_info[n].to(p.device) + overlap = tuple(slice(0, size) for size in old.shape) + _loss = (fisher * (p[overlap] - old) ** 2).sum() loss += _loss # Normalize by batch size if provided if batch_size is not None: loss = loss / batch_size - return self.ewc_lambda * loss \ No newline at end of file + return self.ewc_lambda * loss diff --git a/src/adaptive_classifier/memory.py b/src/adaptive_classifier/memory.py index 6c69d55..696af72 100644 --- a/src/adaptive_classifier/memory.py +++ b/src/adaptive_classifier/memory.py @@ -1,4 +1,5 @@ import torch +import torch.nn.functional as F import numpy as np from typing import List, Dict, Tuple, Optional, Any from collections import defaultdict @@ -32,6 +33,14 @@ def __init__( # Class spread recorded when a classifier was saved. A reloaded memory # keeps only a few representative examples, which understate the spread. self.saved_radii = {} # label -> float + # A reloaded memory holds only a few examples per class, but its prototypes were + # averaged over many more. For such classes the prototype keeps that average + # (prototype_base: label -> (prototype, number of examples behind it)) and folds in + # examples added since (post_sum / post_n) instead of re-averaging the few that + # were kept, which would jerk the prototype away from where it was saved. + self.prototype_base = {} # label -> (tensor, int) + self.post_sum = {} # label -> tensor + self.post_n = {} # label -> int # Initialize FAISS index for fast similarity search self.index = faiss.IndexFlatL2(embedding_dim) @@ -62,6 +71,9 @@ def add_example(self, example: Example, label: str): # Add new example self.examples[label].append(example) + if label in self.prototype_base: + self.post_sum[label] = self.post_sum[label] + example.embedding + self.post_n[label] += 1 # Check if we need to prune examples after adding if len(self.examples[label]) > self.config.max_examples_per_class: @@ -149,9 +161,13 @@ def _update_prototype(self, label: str): if not examples: return - # Compute mean of embeddings - embeddings = torch.stack([ex.embedding for ex in examples]) - prototype = torch.mean(embeddings, dim=0) + if label in self.prototype_base: + base, count = self.prototype_base[label] + prototype = (base * count + self.post_sum[label]) / (count + self.post_n[label]) + else: + # Compute mean of embeddings + embeddings = torch.stack([ex.embedding for ex in examples]) + prototype = torch.mean(embeddings, dim=0) # Update prototype self.prototypes[label] = prototype @@ -239,8 +255,24 @@ def get_stats(self) -> Dict[str, Any]: 'updates_since_rebuild': self.updates_since_rebuild } + def set_prototype_base(self, label: str, count: int): + """Treat the current prototype of `label` as the mean of `count` examples. + + Used after loading, when fewer examples than that are held in memory. + """ + if label in self.prototypes and count > len(self.examples.get(label, [])): + self.prototype_base[label] = (self.prototypes[label].clone(), int(count)) + self.post_sum[label] = torch.zeros_like(self.prototypes[label]) + self.post_n[label] = 0 + + def _forget_base(self, label: str): + self.prototype_base.pop(label, None) + self.post_sum.pop(label, None) + self.post_n.pop(label, None) + def remove_class(self, label: str): """Drop a class and everything stored for it.""" + self._forget_base(label) self.examples.pop(label, None) self.prototypes.pop(label, None) self.strategic_prototypes.pop(label, None) @@ -263,6 +295,7 @@ def remove_examples(self, label: str, texts) -> int: self.remove_class(label) return removed self.examples[label] = kept + self._forget_base(label) # the prototype is recomputed from what remains self.prototypes[label] = torch.mean(torch.stack([ex.embedding for ex in kept]), dim=0) self.strategic_prototypes.pop(label, None) # The saved spread described the old membership. @@ -298,7 +331,11 @@ def clear(self): """Clear all memory.""" self.examples.clear() self.prototypes.clear() + self.strategic_prototypes.clear() self.saved_radii.clear() + self.prototype_base.clear() + self.post_sum.clear() + self.post_n.clear() self.index = faiss.IndexFlatL2(self.embedding_dim) self.label_to_index.clear() self.index_to_label.clear() diff --git a/src/adaptive_classifier/multilabel.py b/src/adaptive_classifier/multilabel.py index d740879..37f20b0 100644 --- a/src/adaptive_classifier/multilabel.py +++ b/src/adaptive_classifier/multilabel.py @@ -1,3 +1,4 @@ +import math import torch import torch.nn as nn import torch.nn.functional as F @@ -6,7 +7,7 @@ import logging from collections import defaultdict -from .classifier import AdaptiveClassifier +from .classifier import AdaptiveClassifier, _synchronized from .models import AdaptiveHead logger = logging.getLogger(__name__) @@ -67,6 +68,19 @@ def update_num_classes(self, new_num_classes: int): self.model[-1] = new_final_layer self.num_classes = new_num_classes + def remove_classes(self, keep_indices: list): + """Keep only the given output rows, in the given order.""" + if not keep_indices: + raise ValueError("Cannot remove every class from the head") + layer = self.model[-1] + new_layer = nn.Linear(layer.in_features, len(keep_indices)) + with torch.no_grad(): + index = torch.tensor(keep_indices, dtype=torch.long, device=layer.weight.device) + new_layer.weight.copy_(layer.weight[index]) + new_layer.bias.copy_(layer.bias[index]) + self.model[-1] = new_layer.to(layer.weight.device) + self.num_classes = len(keep_indices) + class MultiLabelAdaptiveClassifier(AdaptiveClassifier): """ @@ -86,9 +100,21 @@ def __init__( seed: int = 42, default_threshold: float = 0.5, min_predictions: int = 1, - max_predictions: Optional[int] = None + max_predictions: Optional[int] = None, + use_onnx: Optional[Union[bool, str]] = "auto", + trust_remote_code: bool = False ): - super().__init__(model_name, device, config, seed) + if not 0.0 <= default_threshold <= 1.0: + raise ValueError(f"default_threshold must be between 0 and 1; got {default_threshold!r}") + if min_predictions < 0: + raise ValueError(f"min_predictions must be >= 0; got {min_predictions!r}") + if max_predictions is not None and max_predictions < 1: + raise ValueError(f"max_predictions must be >= 1 or None; got {max_predictions!r}") + if (config or {}).get('enable_strategic_mode'): + raise ValueError( + "Strategic mode is not supported by MultiLabelAdaptiveClassifier; " + "use AdaptiveClassifier for it") + super().__init__(model_name, device, config, seed, use_onnx, trust_remote_code) # Multi-label specific configuration self.default_threshold = default_threshold @@ -110,6 +136,29 @@ def _initialize_adaptive_head(self): hidden_dims=hidden_dims ).to(self.device) + def _extra_state(self) -> Dict[str, Any]: + """Settings that `save` stores beside the base classifier's.""" + return { + 'multilabel': { + 'default_threshold': self.default_threshold, + 'min_predictions': self.min_predictions, + 'max_predictions': self.max_predictions, + 'label_thresholds': dict(self.label_thresholds), + } + } + + def _restore_extra_state(self, state: Dict[str, Any]): + saved = (state or {}).get('multilabel') or {} + self.default_threshold = saved.get('default_threshold', self.default_threshold) + self.min_predictions = saved.get('min_predictions', self.min_predictions) + self.max_predictions = saved.get('max_predictions', self.max_predictions) + self.label_thresholds = dict(saved.get('label_thresholds', {})) + + def _drop_classes(self, drop): + super()._drop_classes(drop) + for label in drop: + self.label_thresholds.pop(label, None) + def _get_adaptive_threshold(self, num_labels: int) -> float: """ Calculate adaptive threshold based on number of labels. @@ -129,6 +178,7 @@ def _get_adaptive_threshold(self, num_labels: int) -> float: # For many labels (20+), use very low threshold return self.default_threshold * 0.2 + @_synchronized def predict_multilabel( self, text: str, @@ -148,16 +198,23 @@ def predict_multilabel( """ if not text: raise ValueError("Empty input text") + if max_labels is not None and (isinstance(max_labels, bool) or not isinstance(max_labels, int) or max_labels < 0): + raise ValueError(f"max_labels must be a non-negative integer or None; got {max_labels!r}") + if threshold is not None and not 0.0 <= threshold <= 1.0: + raise ValueError(f"threshold must be between 0 and 1; got {threshold!r}") num_labels = len(self.label_to_id) if num_labels == 0: return [] - # Use adaptive threshold if not specified + # An explicit threshold applies to every label; otherwise each label uses the + # threshold learned for it (falling back to the adaptive one). + explicit_threshold = threshold is not None if threshold is None: threshold = self._get_adaptive_threshold(num_labels) - max_labels = max_labels or self.max_predictions + if max_labels is None: + max_labels = self.max_predictions with torch.no_grad(): # Get embedding @@ -175,7 +232,7 @@ def predict_multilabel( if i < len(self.id_to_label): label = self.id_to_label[i] # Use label-specific threshold if available - label_threshold = self.label_thresholds.get(label, threshold) + label_threshold = threshold if explicit_threshold else self.label_thresholds.get(label, threshold) if prob.item() >= label_threshold: predictions.append((label, prob.item())) @@ -183,14 +240,14 @@ def predict_multilabel( predictions.sort(key=lambda x: x[1], reverse=True) # Apply max_labels limit - if max_labels and len(predictions) > max_labels: + if max_labels is not None and len(predictions) > max_labels: predictions = predictions[:max_labels] else: # Fallback to prototype-based prediction proto_predictions = self.memory.get_nearest_prototypes( embedding, - k=min(num_labels, max_labels) if max_labels else num_labels + k=num_labels if max_labels is None else min(num_labels, max_labels) ) # Filter by threshold @@ -226,57 +283,89 @@ def predict_multilabel( predictions.extend(additional_predictions[:self.min_predictions - len(predictions)]) predictions.sort(key=lambda x: x[1], reverse=True) + # max_labels is a hard cap: it also bounds the min_predictions top-up. + if max_labels is not None: + predictions = predictions[:max_labels] return predictions - def predict(self, text: str, k: int = 5) -> List[Tuple[str, float]]: - """ - Override base predict to use multi-label prediction. - Falls back to single-label prediction if needed. + @_synchronized + def predict( + self, + text: str, + k: int = 5, + abstain_below: Optional[float] = None, + abstain_ood: bool = False, + ) -> List[Tuple[str, float]]: + """The labels `predict_multilabel` picks, at most `k` of them. + + Falls back to the single-label prediction when no label clears its + threshold and `min_predictions` is 0. An empty list means the classifier + abstained: the best score is below `abstain_below`, or the text is out of + distribution (`abstain_ood`). """ - # Use multi-label prediction but limit to k results + self._validate_k(k) + if abstain_ood and self.is_ood(text): + return [] multilabel_preds = self.predict_multilabel(text, max_labels=k) if multilabel_preds: - return multilabel_preds[:k] + predictions = multilabel_preds[:k] else: # Fallback to base prediction if no multi-label predictions - return super().predict(text, k) + predictions = super().predict(text, k) + if abstain_below is not None and (not predictions or predictions[0][1] < abstain_below): + return [] + return predictions + + @_synchronized + def predict_batch(self, texts: List[str], k: int = 5, batch_size: int = 32) -> List[List[Tuple[str, float]]]: + """`predict` for each text. (The single-label batch path softmaxes the scores, which is wrong for sigmoid outputs.)""" + if not texts: + raise ValueError("Empty input batch") + self._validate_k(k) + return [self.predict(text, k) for text in texts] + @_synchronized def add_examples(self, texts: List[str], labels: List[List[str]]): """ Add multi-label training examples. Args: texts: List of input texts - labels: List of label lists (each text can have multiple labels) + labels: One list of labels per text (each text can have several). + A text with an empty label list is skipped. """ if not texts or not labels: raise ValueError("Empty input lists") if len(texts) != len(labels): raise ValueError("Mismatched text and label lists") + for text_labels in labels: + # A bare string would otherwise be split into one class per character. + if not isinstance(text_labels, (list, tuple, set, frozenset)): + raise ValueError( + "labels must be a list of label lists, one per text; got " + f"{type(text_labels).__name__} {text_labels!r} (wrap a single label as ['label'])" + ) - # Flatten labels for single-label training approach - # We'll train one example per text-label pair + # One training example per (text, label) pair; a label repeated within a + # text counts once. flattened_texts = [] flattened_labels = [] - for text, text_labels in zip(texts, labels): - if not text_labels: # Skip texts with no labels - continue - - # For multi-label, we create multiple training examples - # Each example represents the text with one of its labels - for label in text_labels: + for label in dict.fromkeys(text_labels): flattened_texts.append(text) flattened_labels.append(label) if flattened_texts: - # Use parent class method with flattened examples + # The parent validates that texts and labels are non-empty strings. super().add_examples(flattened_texts, flattened_labels) + elif not all(isinstance(t, str) for t in texts): + raise ValueError("texts must all be strings") # Update label-specific thresholds based on training data self._update_label_thresholds() + @_synchronized def _update_label_thresholds(self): """Update per-label thresholds based on training data distribution.""" if not self.memory.examples: @@ -306,112 +395,102 @@ def _update_label_thresholds(self): logger.debug(f"Updated label thresholds: {self.label_thresholds}") + def _train_new_classes(self, old_head, new_classes): + """Retrain the head on every stored example when classes are added. + + The single-label routine behind this hook uses a softmax loss, which + does not apply to independent per-label outputs. + """ + self._train_adaptive_head() + def _train_adaptive_head(self, epochs: int = 10): - """Train multi-label adaptive head with BCE loss.""" + """Train the multi-label head with binary cross-entropy. + + Like the single-label head, it runs at least `head_steps` optimiser + steps with a cosine-decayed learning rate and no early stopping; a few + epochs over a small memory is too little for the head to learn anything. + """ if not self.memory.examples: return - # Prepare multi-label training data - all_embeddings = [] - all_labels = [] - - # Create label matrix for multi-label training - num_classes = len(self.label_to_id) - - # Collect unique texts and their labels + # A text's targets are every label it was stored under. text_to_labels = defaultdict(set) + text_to_embedding = {} for label, examples in self.memory.examples.items(): for example in examples: text_to_labels[example.text].add(label) + text_to_embedding.setdefault(example.text, example.embedding) - # Create training data with proper multi-label targets - for text, labels in text_to_labels.items(): - # Get embedding for this text (take first occurrence) - embedding = None - for label in labels: - for example in self.memory.examples[label]: - if example.text == text: - embedding = example.embedding - break - if embedding is not None: - break - - if embedding is not None: - all_embeddings.append(embedding) - - # Create multi-hot encoded label vector - label_vector = torch.zeros(num_classes) - for label in labels: - if label in self.label_to_id: - label_vector[self.label_to_id[label]] = 1.0 - - all_labels.append(label_vector) - - if not all_embeddings: - return - - all_embeddings = torch.stack(all_embeddings) - all_labels = torch.stack(all_labels) - - # Normalize embeddings - all_embeddings = F.normalize(all_embeddings, p=2, dim=1) - - # Create data loader - dataset = torch.utils.data.TensorDataset(all_embeddings, all_labels) - loader = torch.utils.data.DataLoader( - dataset, - batch_size=min(32, len(all_embeddings)), - shuffle=True, - generator=torch.Generator().manual_seed(42) - ) + num_classes = len(self.label_to_id) + texts = sorted(text_to_labels) + targets = torch.zeros(len(texts), num_classes) + for row, text in enumerate(texts): + for label in text_to_labels[text]: + if label in self.label_to_id: + targets[row, self.label_to_id[label]] = 1.0 + embeddings = F.normalize(torch.stack([text_to_embedding[t] for t in texts]), p=2, dim=1) + embeddings, targets = embeddings.to(self.device), targets.to(self.device) + + batch_size = min(32, len(embeddings)) + batches_per_epoch = math.ceil(len(embeddings) / batch_size) + total_steps = max(epochs * batches_per_epoch, + int(getattr(self.config, 'head_steps', 0) or 0)) - # Training setup self.adaptive_head.train() - criterion = nn.BCELoss() # Binary Cross Entropy for multi-label + criterion = nn.BCELoss() optimizer = torch.optim.AdamW( self.adaptive_head.parameters(), - lr=0.001, - weight_decay=0.01 + lr=getattr(self.config, 'head_learning_rate', 0.001), + weight_decay=0.01, ) - - best_loss = float('inf') - patience_counter = 0 - patience = 3 - - for epoch in range(epochs): - total_loss = 0 - for batch_embeddings, batch_labels in loader: - batch_embeddings = batch_embeddings.to(self.device) - batch_labels = batch_labels.to(self.device) - + scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=total_steps) + generator = torch.Generator().manual_seed(42) + + step = 0 + while step < total_steps: + order = torch.randperm(len(embeddings), generator=generator) + for start in range(0, len(order), batch_size): + index = order[start:start + batch_size] optimizer.zero_grad() - outputs = self.adaptive_head(batch_embeddings) - - loss = criterion(outputs, batch_labels) + loss = criterion(self.adaptive_head(embeddings[index]), targets[index]) loss.backward() - - torch.nn.utils.clip_grad_norm_( - self.adaptive_head.parameters(), - max_norm=1.0 - ) + torch.nn.utils.clip_grad_norm_(self.adaptive_head.parameters(), max_norm=1.0) optimizer.step() + scheduler.step() + step += 1 + if step >= total_steps: + break + + self.adaptive_head.eval() + self.train_steps += 1 - total_loss += loss.item() + def merge_classifiers(self, other): + # Not synchronized: the base class never holds two classifiers' locks at once. + merged = super().merge_classifiers(other) + self._update_label_thresholds() + return merged - avg_loss = total_loss / len(loader) + @_synchronized + def remove_examples(self, texts, label=None, retrain=True): + removed = super().remove_examples(texts, label=label, retrain=retrain) + self._update_label_thresholds() + return removed - # Early stopping - if avg_loss < best_loss: - best_loss = avg_loss - patience_counter = 0 - else: - patience_counter += 1 - if patience_counter >= patience: - logger.debug(f"Early stopping at epoch {epoch + 1}") - break + @_synchronized + def clear_memory(self, labels=None): + super().clear_memory(labels) + if labels is None: + self.label_thresholds = {} - self.train_steps += 1 + def calibrate(self, *args, **kwargs): + raise NotImplementedError( + "Confidence calibration assumes one label per text and is not available " + "for MultiLabelAdaptiveClassifier") + + calibration_report = calibrate + predict_set = calibrate + @_synchronized def get_label_statistics(self) -> Dict[str, Any]: """Get statistics about label distribution and thresholds.""" stats = super().get_example_statistics() diff --git a/src/adaptive_classifier/serving.py b/src/adaptive_classifier/serving.py index db2bf1b..1a9ddee 100644 --- a/src/adaptive_classifier/serving.py +++ b/src/adaptive_classifier/serving.py @@ -14,10 +14,11 @@ ``POST /remove_examples``. The classifier takes a lock around every operation, so one process is safe to -share across requests; scale throughput with ``--workers`` (separate -processes, each with its own copy and its own updates) rather than threads. -Updates therefore only reach the worker that handled them. Run a single -worker if you enable them, and pass ``--save-dir`` to persist what was learned. +share across requests; scale throughput with several processes (for example +containers behind a load balancer, each with its own copy and its own updates) +rather than threads. Updates therefore only reach the process that handled +them. Run a single process if you enable them, and pass ``--save-dir`` to +persist what was learned. """ import argparse @@ -51,6 +52,11 @@ def create_app( max_text_chars: Longest accepted text, in characters. save_dir: If set, the classifier is saved there after every update. """ + from .multilabel import MultiLabelAdaptiveClassifier + if isinstance(classifier, MultiLabelAdaptiveClassifier): + raise ValueError( + "The server does not support MultiLabelAdaptiveClassifier: its predictions and " + "updates use lists of labels per text, which these endpoints do not express") try: from fastapi import Depends, FastAPI, HTTPException, Request from fastapi.responses import JSONResponse @@ -124,9 +130,19 @@ def require_updates(request: Request): raise HTTPException(status_code=401, detail="Missing or invalid API key", headers={"WWW-Authenticate": "Bearer"}) - def persist(): + async def persist(): + # Saving (with ONNX export) takes seconds; keep it off the event loop. if save_dir: - classifier.save(save_dir) + await asyncio.to_thread(classifier.save, save_dir) + + def classes(): + # Read under the classifier's lock: an update running in another thread rewrites the label map. + with classifier._lock: + return sorted(classifier.label_to_id) + + def snapshot(): + with classifier._lock: + return sorted(classifier.label_to_id), classifier.get_memory_stats(), bool(classifier.calibration) @app.get("/health") async def health(): @@ -134,13 +150,13 @@ async def health(): @app.get("/info") async def info(): - stats = classifier.get_memory_stats() + class_names, stats, calibrated = await asyncio.to_thread(snapshot) return { "version": _version(), "model": getattr(classifier, "_model_name", None), - "classes": sorted(classifier.label_to_id), + "classes": class_names, "examples_per_class": stats.get("examples_per_class", {}), - "calibrated": bool(classifier.calibration), + "calibrated": calibrated, "updates_allowed": allow_updates, } @@ -177,20 +193,20 @@ async def add_examples(body: ExamplesRequest): if any(not t or len(t) > max_text_chars for t in body.texts): raise HTTPException(status_code=422, detail=f"Each text must be 1-{max_text_chars} characters") await classifier.aadd_examples(body.texts, body.labels) - persist() - return {"added": len(body.texts), "classes": sorted(classifier.label_to_id)} + await persist() + return {"added": len(body.texts), "classes": await asyncio.to_thread(classes)} @app.post("/forget", dependencies=[Depends(require_updates)]) async def forget(body: ForgetRequest): await asyncio.to_thread(classifier.forget, body.labels) - persist() - return {"classes": sorted(classifier.label_to_id)} + await persist() + return {"classes": await asyncio.to_thread(classes)} @app.post("/remove_examples", dependencies=[Depends(require_updates)]) async def remove_examples(body: RemoveRequest): removed = await asyncio.to_thread(classifier.remove_examples, body.texts, body.label) - persist() - return {"removed": removed, "classes": sorted(classifier.label_to_id)} + await persist() + return {"removed": removed, "classes": await asyncio.to_thread(classes)} return app diff --git a/src/adaptive_classifier/sklearn_api.py b/src/adaptive_classifier/sklearn_api.py index 7d1c166..49b7753 100644 --- a/src/adaptive_classifier/sklearn_api.py +++ b/src/adaptive_classifier/sklearn_api.py @@ -44,8 +44,8 @@ class SklearnAdaptiveClassifier(ClassifierMixin, BaseEstimator): ----- `fit` starts from scratch each time (it reloads the encoder, so cross-validation pays that cost per fold). `partial_fit` adds to what is - already learned. A fitted estimator holds a FAISS index and cannot be - pickled; persist `classifier_` with `classifier_.save(...)` instead. + already learned. A fitted estimator can be pickled, but that copies the + whole encoder; persist `classifier_` with `classifier_.save(...)` instead. """ def __init__( @@ -117,21 +117,40 @@ def _new_classifier(self) -> AdaptiveClassifier: def _refresh_classes(self): # The wrapped classifier keys everything by str(label); keep the # caller's original label objects so predict returns what was passed in. - originals = list(self._original_labels.values()) + # Classes declared through partial_fit(classes=...) are listed even + # before an example of them has been seen (scikit-learn's contract). + originals = list({**self._declared_labels, **self._original_labels}.values()) try: ordered = sorted(originals) except TypeError: # mixed label types ordered = sorted(originals, key=str) self.classes_ = np.array(ordered) + def _register_labels(self, labels: List[Any], known: Dict[str, Any]): + """Check labels can be told apart once turned into strings, and record them in `known`.""" + for label in labels: + if label is None or (isinstance(label, float) and label != label): + raise ValueError("y must not contain None or NaN labels") + key = str(label) + previous = known.get(key, label) + if (type(previous), previous) != (type(label), label): + raise ValueError( + f"labels {previous!r} and {label!r} are different but both map to the class " + f"{key!r}; the classifier identifies classes by their string form" + ) + known[key] = label + # --- fitting ------------------------------------------------------------------- def fit(self, X, y): """Fit from scratch on texts `X` and labels `y`.""" texts = self._as_texts(X) labels = self._as_labels(y, len(texts)) + known: Dict[str, Any] = {} + self._register_labels(labels, known) self.classifier_ = self._new_classifier() self._original_labels = {} + self._declared_labels = {} self._add(texts, labels) return self @@ -143,20 +162,27 @@ def partial_fit(self, X, y, classes=None): """ texts = self._as_texts(X) labels = self._as_labels(y, len(texts)) + declared: Dict[str, Any] = {} if classes is not None: - unexpected = set(labels) - set(np.asarray(classes, dtype=object).tolist()) + declared_list = np.asarray(classes, dtype=object).tolist() + self._register_labels(declared_list, declared) + unexpected = set(labels) - set(declared_list) if unexpected: raise ValueError(f"y contains labels not in classes: {sorted(map(str, unexpected))}") if not hasattr(self, "classifier_"): self.classifier_ = self._new_classifier() self._original_labels = {} + self._declared_labels = {} + self._declared_labels.update(declared) self._add(texts, labels) return self def _add(self, texts: List[str], labels: List[Any]): - for label in labels: - self._original_labels.setdefault(str(label), label) + # Validate against everything known so far before changing anything. + known = {**self._declared_labels, **self._original_labels} + self._register_labels(labels, known) self.classifier_.add_examples(texts, [str(label) for label in labels]) + self._original_labels.update({str(label): label for label in labels}) self._refresh_classes() # --- prediction ---------------------------------------------------------------- diff --git a/src/adaptive_classifier/strategic.py b/src/adaptive_classifier/strategic.py index b1deb98..e386d1f 100644 --- a/src/adaptive_classifier/strategic.py +++ b/src/adaptive_classifier/strategic.py @@ -2,7 +2,7 @@ import torch.nn as nn import torch.nn.functional as F import numpy as np -from typing import Dict, List, Optional, Tuple, Any, Union +from typing import Dict, List, Optional, Sequence, Tuple, Any, Union from abc import ABC, abstractmethod import logging @@ -102,24 +102,37 @@ def compute_best_response(self, x: torch.Tensor, f: callable) -> torch.Tensor: return best_response def _generate_candidates(self, x: torch.Tensor, num_candidates: int = 50) -> List[torch.Tensor]: - """Generate candidate points for optimization.""" - candidates = [x] # Always include the original point - - # Generate candidates by varying each feature - for i in range(len(x)): - for delta in torch.linspace(-2.0, 2.0, 10): # Reasonable range for feature changes - if delta == 0: - continue + """Candidate responses: the original point, single-feature moves, one random jitter. + + Single-feature moves are tried on the cheapest features to change (those + are where an agent gets the most for its money) and on a random sample of + the rest, so every dimension of a long embedding can be reached. The + sampling uses a fixed seed: the same input always gets the same response. + """ + generator = torch.Generator().manual_seed(0) + n = len(x) + deltas = (-2.0, -1.0, -0.5, 0.5, 1.0, 2.0) + n_dims = max(1, (num_candidates - 2) // len(deltas)) + + price = self.c2.detach().abs().cpu().to(torch.float32) + if price.shape[0] != n: + price = torch.zeros(n) + cheapest_first = torch.argsort(price + 1e-6 * torch.rand(n, generator=generator)) + n_cheap = min(n, (n_dims + 1) // 2) + chosen = cheapest_first[:n_cheap].tolist() + rest = cheapest_first[n_cheap:] + if len(rest) and n_dims > n_cheap: + pick = torch.randperm(len(rest), generator=generator)[:n_dims - n_cheap] + chosen += rest[pick].tolist() + + candidates = [x] + for i in chosen: + for delta in deltas: candidate = x.clone() candidate[i] += delta candidates.append(candidate) - - # Generate some random candidates - for _ in range(num_candidates - len(candidates)): - noise = torch.randn_like(x) * 0.5 # Small random perturbations - candidate = x + noise - candidates.append(candidate) - + jitter = torch.randn(n, generator=generator) * 0.5 + candidates.append(x + jitter.to(device=x.device, dtype=x.dtype)) return candidates[:num_candidates] @@ -286,7 +299,7 @@ def evaluate_robustness( classifier, test_embeddings: torch.Tensor, test_labels: torch.Tensor, - gaming_levels: List[float] = [0.0, 0.5, 1.0] + gaming_levels: Sequence[float] = (0.0, 0.5, 1.0) ) -> Dict[str, float]: """Evaluate classifier robustness under strategic behavior. @@ -301,7 +314,8 @@ def evaluate_robustness( """ results = {} - for level in gaming_levels: + # The ungamed and fully gamed baselines are what the robustness scores compare. + for level in sorted(set(gaming_levels) | {0.0, 1.0}): # Simulate strategic behavior at this level strategic_embeddings = self._simulate_strategic_behavior( test_embeddings, classifier, level @@ -317,7 +331,9 @@ def evaluate_robustness( # Compute robustness metrics results['robustness_score'] = results['accuracy_gaming_0.0'] - results['accuracy_gaming_1.0'] - results['relative_robustness'] = results['accuracy_gaming_1.0'] / results['accuracy_gaming_0.0'] + # Undefined (nan) when the classifier gets everything wrong without gaming. + base = results['accuracy_gaming_0.0'] + results['relative_robustness'] = results['accuracy_gaming_1.0'] / base if base > 0 else float('nan') return results @@ -338,13 +354,14 @@ def _simulate_strategic_behavior( Modified embeddings after strategic behavior """ strategic_embeddings = [] + generator = torch.Generator().manual_seed(0) # which inputs game is repeatable def classifier_func(x): with torch.no_grad(): return torch.softmax(classifier(x), dim=-1) for embedding in embeddings: - if torch.rand(1).item() < gaming_level: + if torch.rand(1, generator=generator).item() < gaming_level: # Apply strategic behavior strategic_embedding = self.cost_function.compute_best_response( embedding, classifier_func diff --git a/tests/test_concurrency.py b/tests/test_concurrency.py index efb9483..34f24df 100644 --- a/tests/test_concurrency.py +++ b/tests/test_concurrency.py @@ -177,3 +177,27 @@ async def go(): return await asyncio.gather(*[clf.apredict(f"tr{i % 3}_{i % 5}") for i in range(40)]) results = asyncio.run(go()) assert len(results) == 40 and all(r for r in results) + + +# --- the lock must not stop a classifier being copied ----------------------------------------- + +def test_a_classifier_can_be_deep_copied_and_the_copy_is_independent(clf, data): + import copy + + twin = copy.deepcopy(clf) + assert twin._lock is not clf._lock + text = next(iter(data.store)) + assert twin.predict(text, k=3) == clf.predict(text, k=3) + + new_x, _ = data.classes(8, n_classes=1, seed=7, prefix="n") + twin.add_examples(new_x, ["extra"] * len(new_x)) + assert "extra" in twin.label_to_id and "extra" not in clf.label_to_id + + +def test_a_classifier_survives_a_pickle_round_trip(clf, data): + import pickle + + restored = pickle.loads(pickle.dumps(clf)) + text = next(iter(data.store)) + assert restored.predict(text, k=3) == clf.predict(text, k=3) + restored.add_examples([text], ["C0"]) # and still works (has its own lock) diff --git a/tests/test_failure_modes.py b/tests/test_failure_modes.py index a870907..6e7942f 100644 --- a/tests/test_failure_modes.py +++ b/tests/test_failure_modes.py @@ -142,10 +142,11 @@ def test_integer_labels_cannot_corrupt_a_saved_model(new_classifier, tmp_path): assert {l for l, _ in loaded.predict("great", k=10)} == {"1", "2"} -def test_empty_string_text_is_allowed_and_embeds(new_classifier): +def test_empty_string_text_is_rejected_rather_than_learned_as_a_class(new_classifier): + """Blank rows used to be accepted, turning every empty text into an example of its class.""" clf = new_classifier() - clf.add_examples(["", "good"], ["blank", "ok"]) - assert {l for l, _ in clf.predict("good", k=5)} == {"blank", "ok"} + with pytest.raises(ValueError, match="blank"): + clf.add_examples(["", "good"], ["blank", "ok"]) def test_identical_text_with_conflicting_labels_stays_consistent(new_classifier): @@ -368,3 +369,32 @@ def test_state_after_random_operations_survives_save_and_reload(new_classifier, assert set(loaded.label_to_id) == set(clf.label_to_id) for text in ("great love", "terrible hate", "okay fine"): assert [l for l, _ in loaded.predict(text, k=9)] == [l for l, _ in clf.predict(text, k=9)] + + +# --- blank training data ---------------------------------------------------------------------- + +@pytest.mark.parametrize("texts, labels, message", [ + ([""], ["p"], "text at index 0"), + (["great", " "], ["p", "p"], "text at index 1"), + (["great", "\n\t"], ["p", "n"], "text at index 1"), + (["great"], [""], "label at index 0"), + (["great", "bad"], ["p", " "], "label at index 1"), +]) +def test_blank_training_text_or_label_is_rejected_and_nothing_is_learned(new_classifier, texts, labels, message): + clf = new_classifier() + with pytest.raises(ValueError, match=message): + clf.add_examples(texts, labels) + assert clf.label_to_id == {} and clf.memory.examples == {} + + +def test_a_blank_row_does_not_change_an_existing_classifier(trained): + before = dict(trained.get_memory_stats()["examples_per_class"]) + with pytest.raises(ValueError): + trained.add_examples(["great love", ""], ["p", "p"]) + assert trained.get_memory_stats()["examples_per_class"] == before + + +def test_selecting_zero_representative_examples_returns_nothing(trained): + examples = trained.memory.examples["p"] + assert trained.select_representative_examples(examples, k=0) == [] + assert trained.select_representative_examples([], k=0) == [] diff --git a/tests/test_js_example.py b/tests/test_js_example.py new file mode 100644 index 0000000..9e78c35 --- /dev/null +++ b/tests/test_js_example.py @@ -0,0 +1,73 @@ +"""The Node.js port in examples/javascript must agree with `AdaptiveClassifier.predict`. + +Skipped unless Node and the example's npm dependencies are installed +(`cd examples/javascript && npm install`; CI does this). +""" + +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +from adaptive_classifier import AdaptiveClassifier + +JS_DIR = Path(__file__).resolve().parent.parent / "examples" / "javascript" +TEXTS = ["great product love it", "works very good fast", "terrible awful service", + "hate it broke bad", "okay fine the product", "it is okay fine"] * 3 +LABELS = ["pos", "pos", "neg", "neg", "neu", "neu"] * 3 +QUERIES = ["great fast product", "awful slow broke", "okay it is fine", "love the service", + "great " * 40] # longer than max_length below: exercises truncation + +SCRIPT = """ +import { PortableClassifier } from "./portable_inference.mjs"; +const [dir, queries] = [process.argv[1], JSON.parse(process.argv[2])]; +const classifier = await PortableClassifier.load(dir, { preferQuantized: false }); +const out = []; +for (const q of queries) out.push(await classifier.predict(q, 3)); +console.log(JSON.stringify(out)); +""" + +pytestmark = pytest.mark.skipif( + shutil.which("node") is None or not (JS_DIR / "node_modules").exists(), + reason="Node.js or examples/javascript/node_modules not available", +) + + +def _onnx_available(): + try: + import optimum.onnxruntime # noqa: F401 + return True + except ImportError: + return False + + +@pytest.mark.skipif(not _onnx_available(), reason="optimum[onnxruntime] not installed") +@pytest.mark.parametrize("pooling", ["mean", "cls"]) +@pytest.mark.parametrize("config", [ + {}, + {"prototype_temperature": None}, + {"new_class_prototype_weight": 0.3, "new_class_neural_weight": 0.7}, + {"new_class_example_threshold": 4}, +], ids=["default", "legacy-scoring", "fixed-new-class", "mixed-established"]) +def test_node_port_matches_predict(base_model, tmp_path, pooling, config): + clf = AdaptiveClassifier(base_model, config={"pooling": pooling, "max_length": 32, **config}, + use_onnx=False, device="cpu") + clf.add_examples(TEXTS, LABELS) + clf.add_examples(["great product love it"] * 6, ["pos"] * 6) + clf.save(str(tmp_path), include_onnx=True, quantize_onnx=False) + + result = subprocess.run( + ["node", "--input-type=module", "-e", SCRIPT, str(tmp_path), json.dumps(QUERIES)], + cwd=JS_DIR, capture_output=True, text=True, timeout=120, + ) + assert result.returncode == 0, result.stderr + got = json.loads(result.stdout.strip().splitlines()[-1]) + + for query, ranked in zip(QUERIES, got): + expected = dict(clf.predict(query, k=3)) + actual = {label: score for label, score in ranked} + assert actual.keys() == expected.keys() + for label in expected: + assert actual[label] == pytest.approx(expected[label], abs=1e-4) diff --git a/tests/test_memory.py b/tests/test_memory.py index 54b56db..192d270 100644 --- a/tests/test_memory.py +++ b/tests/test_memory.py @@ -256,4 +256,29 @@ def add_examples(label): assert len(memory.examples[label]) == 100 if __name__ == "__main__": - pytest.main([__file__]) \ No newline at end of file + pytest.main([__file__]) + +def test_strategic_prototypes_can_be_computed_and_searched(): + """get_strategic_prototypes used to raise NameError (F was never imported).""" + from adaptive_classifier.strategic import LinearCostFunction + + memory = PrototypeMemory(embedding_dim=4) + for label, axis in (("a", 0), ("b", 1)): + for i in range(3): + vector = torch.zeros(4) + vector[axis] = 1.0 + memory.add_example(Example(f"{label}{i}", label, vector + 0.01 * i), label) + + memory._rebuild_index() # the classifier does this after adding examples + + # Before any are computed it falls back to the ordinary prototypes. + assert memory.get_strategic_prototypes(torch.tensor([1.0, 0, 0, 0]), k=2) + + memory.compute_strategic_prototypes( + LinearCostFunction(torch.full((4,), 1000.0)), # too costly to move: the response is the input + lambda x: torch.softmax(x[:, :2], dim=-1), + ) + assert set(memory.strategic_prototypes) == {"a", "b"} + ranked = memory.get_strategic_prototypes(torch.tensor([1.0, 0, 0, 0]), k=2) + assert [label for label, _ in ranked] == ["a", "b"] + assert ranked[0][1] > ranked[1][1] diff --git a/tests/test_merge.py b/tests/test_merge.py new file mode 100644 index 0000000..88d9653 --- /dev/null +++ b/tests/test_merge.py @@ -0,0 +1,88 @@ +"""merge_classifiers: the merged classifier should know both sets of classes.""" + +import pytest + +from adaptive_classifier import AdaptiveClassifier + +from .synthetic import Dataset + + +@pytest.fixture +def data(monkeypatch): + return Dataset().patch(monkeypatch) + + +def _trained(new_classifier, data, axes, seed, prefix, n_per=12): + clf = new_classifier() + texts, labels = [], [] + for axis in axes: + for i in range(n_per): + name = data.add(f"{prefix}{axis}_{i}", axis, 0.1, seed * 1000 + axis * 50 + i) + texts.append(name) + labels.append(f"C{axis}") + clf.add_examples(texts, labels) + return clf, texts, labels + + +def _accuracy(clf, texts, labels): + return sum(clf.predict(t)[0][0] == l for t, l in zip(texts, labels)) / len(texts) + + +def test_merged_classifier_recognises_both_sets_of_classes(new_classifier, data): + a, a_x, a_y = _trained(new_classifier, data, [0, 1], 1, "a") + b, b_x, b_y = _trained(new_classifier, data, [2, 3], 2, "b") + + a.merge_classifiers(b) + + assert set(a.label_to_id) == {"C0", "C1", "C2", "C3"} + assert _accuracy(a, a_x, a_y) >= 0.9 + assert _accuracy(a, b_x, b_y) >= 0.9 + + +def test_merging_into_an_empty_classifier_gives_a_working_classifier(new_classifier, data): + empty = new_classifier() + b, b_x, b_y = _trained(new_classifier, data, [2, 3], 2, "b") + empty.merge_classifiers(b) + assert _accuracy(empty, b_x, b_y) >= 0.9 + + +def test_merging_classifiers_that_share_a_class_adds_up_their_training_counts(new_classifier, data): + a, _, _ = _trained(new_classifier, data, [0, 1], 1, "a") + b, _, _ = _trained(new_classifier, data, [1, 2], 2, "b") + before = a.training_history["C1"] + a.merge_classifiers(b) + assert a.training_history["C1"] == before + b.training_history["C1"] + assert a.training_history["C2"] == b.training_history["C2"] # so the head gets its full weight + + +def test_merge_leaves_the_other_classifier_alone(new_classifier, data): + a, _, _ = _trained(new_classifier, data, [0, 1], 1, "a") + b, b_x, b_y = _trained(new_classifier, data, [2, 3], 2, "b") + a.merge_classifiers(b) + a.forget("C2") + assert "C2" in b.label_to_id + assert _accuracy(b, b_x, b_y) >= 0.9 + + +def test_merge_discards_calibration_because_the_classes_changed(new_classifier, data): + a, a_x, a_y = _trained(new_classifier, data, [0, 1], 1, "a") + a.calibrate(a_x, a_y) + assert a.calibration + b, _, _ = _trained(new_classifier, data, [2, 3], 2, "b") + a.merge_classifiers(b) + assert not a.calibration + + +def test_merge_rejects_different_embedding_sizes(new_classifier, data): + a, _, _ = _trained(new_classifier, data, [0, 1], 1, "a") + b, _, _ = _trained(new_classifier, data, [2, 3], 2, "b") + b.embedding_dim = 64 + with pytest.raises(ValueError, match="embedding"): + a.merge_classifiers(b) + + +def test_merging_a_classifier_with_itself_changes_nothing_harmful(new_classifier, data): + a, a_x, a_y = _trained(new_classifier, data, [0, 1], 1, "a") + a.merge_classifiers(a) + assert set(a.label_to_id) == {"C0", "C1"} + assert _accuracy(a, a_x, a_y) >= 0.9 diff --git a/tests/test_multilabel_robustness.py b/tests/test_multilabel_robustness.py new file mode 100644 index 0000000..3b92d9c --- /dev/null +++ b/tests/test_multilabel_robustness.py @@ -0,0 +1,346 @@ +"""MultiLabelAdaptiveClassifier: learning quality, lifecycle operations and bad input. + +The accuracy tests use hand-made embeddings (see synthetic.py) in which a text with +several labels sits between their axes, so a correct classifier should be near perfect. +""" + +import threading + +import pytest +import torch +import torch.nn.functional as F + +from adaptive_classifier import MultiLabelAdaptiveClassifier +from .synthetic import DIM, Dataset, point + +AXES = {"a": 0, "b": 1, "c": 2, "d": 3} + + +def _embed(labels, noise, seed): + v = sum(point(AXES[label], 0.0, 0) for label in labels) + g = torch.Generator().manual_seed(seed) + return F.normalize(F.normalize(v, dim=0) + noise * torch.randn(DIM, generator=g), dim=0) + + +class MultiLabelData(Dataset): + def sample(self, label_sets, n_per, noise=0.05, seed=0, prefix="t"): + texts, labels = [], [] + for s, label_set in enumerate(label_sets): + for i in range(n_per): + name = f"{prefix}{'+'.join(label_set)}_{i}_{seed}" + self.store[name] = _embed(label_set, noise, seed * 7919 + s * 131 + i) + texts.append(name) + labels.append(list(label_set)) + return texts, labels + + +@pytest.fixture +def data(monkeypatch, base_model): + ds = MultiLabelData() + ds.patch(monkeypatch) + return ds + + +@pytest.fixture +def make(base_model): + def build(**kwargs): + return MultiLabelAdaptiveClassifier(base_model, device="cpu", use_onnx=False, **kwargs) + return build + + +def _predicted(clf, text, **kwargs): + return {label for label, _ in clf.predict_multilabel(text, **kwargs)} + + +def _exact_match(clf, texts, labels): + return sum(_predicted(clf, t) == set(l) for t, l in zip(texts, labels)) / len(texts) + + +# --- learning --------------------------------------------------------------- + +def test_learns_separable_multilabel_data(data, make): + sets = [("a",), ("b",), ("c",), ("a", "b"), ("b", "c")] + train_x, train_y = data.sample(sets, 12, seed=1) + test_x, test_y = data.sample(sets, 6, seed=2, prefix="v") + clf = make() + clf.add_examples(train_x, train_y) + assert _exact_match(clf, test_x, test_y) >= 0.9 + + +def test_a_label_added_later_is_learned_without_forgetting_the_rest(data, make): + sets = [("a",), ("b",), ("a", "b")] + train_x, train_y = data.sample(sets, 12, seed=1) + clf = make() + clf.add_examples(train_x, train_y) + + new_x, new_y = data.sample([("c",), ("a", "c")], 12, seed=3, prefix="n") + clf.add_examples(new_x, new_y) + + test_x, test_y = data.sample(sets + [("c",), ("a", "c")], 6, seed=4, prefix="v") + assert _exact_match(clf, test_x, test_y) >= 0.85 + + +def test_a_text_repeated_with_different_labels_is_one_training_text(data, make): + texts, labels = data.sample([("a", "b")], 10, seed=1) + clf = make() + clf.add_examples(texts, labels) + assert _predicted(clf, texts[0]) == {"a", "b"} + + +# --- lifecycle operations --------------------------------------------------- + +def test_forget_removes_a_label_everywhere(data, make): + sets = [("a",), ("b",), ("c",), ("a", "b")] + train_x, train_y = data.sample(sets, 10, seed=1) + clf = make() + clf.add_examples(train_x, train_y) + + clf.forget("b") + + assert "b" not in clf.label_to_id and "b" not in clf.label_thresholds + assert clf.adaptive_head.model[-1].out_features == len(clf.label_to_id) == 2 + assert all("b" not in _predicted(clf, t) for t in train_x) + assert _predicted(clf, train_x[0]) == {"a"} # a-only text + assert _predicted(clf, train_x[20]) == {"c"} + + +def test_remove_examples_retrains_and_can_drop_a_label(data, make): + train_x, train_y = data.sample([("a",), ("b",), ("c",)], 10, seed=1) + clf = make() + clf.add_examples(train_x, train_y) + + only_c = [t for t, l in zip(train_x, train_y) if l == ["c"]] + assert clf.remove_examples(only_c) == len(only_c) + assert "c" not in clf.label_to_id + assert clf.predict_multilabel(train_x[0]) + + +def test_save_and_load_keep_settings_thresholds_and_predictions(data, make, tmp_path): + sets = [("a",), ("b",), ("a", "b"), ("c",)] + train_x, train_y = data.sample(sets, 10, seed=1) + clf = make(default_threshold=0.45, min_predictions=2, max_predictions=3) + clf.add_examples(train_x, train_y) + clf.save(str(tmp_path), include_onnx=False) + + loaded = MultiLabelAdaptiveClassifier.load(str(tmp_path), device="cpu", use_onnx=False) + + assert isinstance(loaded, MultiLabelAdaptiveClassifier) + assert (loaded.default_threshold, loaded.min_predictions, loaded.max_predictions) == (0.45, 2, 3) + assert loaded.label_thresholds == clf.label_thresholds + for text in train_x[::7]: + a, b = dict(clf.predict_multilabel(text)), dict(loaded.predict_multilabel(text)) + assert a.keys() == b.keys() + assert all(a[l] == pytest.approx(b[l], abs=1e-4) for l in a) + + +def test_a_plain_classifier_save_has_no_multilabel_block(data, base_model, tmp_path): + from adaptive_classifier import AdaptiveClassifier + clf = AdaptiveClassifier(base_model, device="cpu", use_onnx=False) + texts, labels = data.sample([("a",), ("b",)], 5, seed=1) + clf.add_examples(texts, [l[0] for l in labels]) + clf.save(str(tmp_path), include_onnx=False) + assert "multilabel" not in (tmp_path / "config.json").read_text() + + +# --- limits and thresholds ---------------------------------------------------- + +def test_max_labels_is_a_hard_cap_even_with_min_predictions(data, make): + train_x, train_y = data.sample([("a", "b", "c")], 10, seed=1) + clf = make(min_predictions=3) + clf.add_examples(train_x, train_y) + assert len(clf.predict_multilabel(train_x[0], max_labels=1)) == 1 + assert clf.predict_multilabel(train_x[0], max_labels=0) == [] + + +def test_min_predictions_returns_something_when_nothing_clears_the_threshold(data, make): + train_x, train_y = data.sample([("a",), ("b",)], 10, seed=1) + clf = make(min_predictions=1) + clf.add_examples(train_x, train_y) + assert len(clf.predict_multilabel(train_x[0], threshold=1.0)) >= 1 + + +# --- bad input ---------------------------------------------------------------- + +@pytest.mark.parametrize("labels, message", [ + (["pos"], "list of label lists"), # a bare string used to become classes 'p', 'o', 's' + ([b"pos"], "list of label lists"), + ([5], "list of label lists"), + ([[1, 2]], "labels must be strings"), + ([[None]], "labels must be strings"), +]) +def test_malformed_labels_are_rejected_before_anything_changes(make, labels, message): + clf = make() + with pytest.raises(ValueError, match=message): + clf.add_examples(["great love"], labels) + assert clf.label_to_id == {} + + +def test_non_string_texts_are_rejected(make): + clf = make() + with pytest.raises(ValueError, match="strings"): + clf.add_examples([None], [["a"]]) + with pytest.raises(ValueError, match="strings"): + clf.add_examples([7], [[]]) # also when no label would be stored + + +def test_empty_and_mismatched_inputs_are_rejected(make): + clf = make() + with pytest.raises(ValueError): + clf.add_examples([], []) + with pytest.raises(ValueError): + clf.add_examples(["a b"], [["x"], ["y"]]) + + +@pytest.mark.parametrize("kwargs", [ + {"threshold": -0.1}, {"threshold": 1.5}, {"max_labels": -1}, {"max_labels": 1.5}, {"max_labels": True}, +]) +def test_bad_prediction_arguments_are_rejected(data, make, kwargs): + train_x, train_y = data.sample([("a",), ("b",)], 5, seed=1) + clf = make() + clf.add_examples(train_x, train_y) + with pytest.raises(ValueError): + clf.predict_multilabel(train_x[0], **kwargs) + + +def test_predict_validates_k_and_empty_text(data, make): + train_x, train_y = data.sample([("a",), ("b",)], 5, seed=1) + clf = make() + clf.add_examples(train_x, train_y) + with pytest.raises(ValueError): + clf.predict(train_x[0], k=-1) + with pytest.raises(ValueError): + clf.predict_multilabel("") + assert clf.predict(train_x[0], k=0) == [] + + +@pytest.mark.parametrize("kwargs", [ + {"default_threshold": 1.5}, {"default_threshold": -0.2}, {"min_predictions": -1}, {"max_predictions": 0}, +]) +def test_bad_constructor_settings_are_rejected(make, kwargs): + with pytest.raises(ValueError): + make(**kwargs) + + +def test_an_untrained_classifier_predicts_nothing(make): + assert make().predict_multilabel("anything") == [] + + +# --- concurrency -------------------------------------------------------------- + +def test_predicting_while_labels_are_added_and_forgotten_does_not_crash(data, make): + train_x, train_y = data.sample([("a",), ("b",), ("c",)], 8, seed=1) + clf = make() + clf.add_examples(train_x, train_y) + extra_x, extra_y = data.sample([("d",)], 8, seed=2, prefix="x") + stop, errors = threading.Event(), [] + + def reader(): + while not stop.is_set(): + try: + clf.predict_multilabel(train_x[0]) + clf.predict(train_x[9], k=3) + except Exception as error: # noqa: BLE001 - any failure is the bug + errors.append(error) + return + + threads = [threading.Thread(target=reader) for _ in range(3)] + for t in threads: + t.start() + try: + for _ in range(3): + clf.add_examples(extra_x, extra_y) + clf.forget("d") + finally: + stop.set() + for t in threads: + t.join() + assert not errors, errors[0] + + +def test_predict_batch_agrees_with_predict(data, make): + train_x, train_y = data.sample([("a",), ("b",), ("a", "b")], 8, seed=1) + clf = make() + clf.add_examples(train_x, train_y) + batch = clf.predict_batch(train_x[::5], k=3) + assert batch == [clf.predict(text, k=3) for text in train_x[::5]] + with pytest.raises(ValueError): + clf.predict_batch([]) + + +# --- thresholds, loading, unsupported combinations --------------------------------------------------- + +def test_an_explicit_threshold_applies_to_every_label(data, make): + train_x, train_y = data.sample([("a",), ("b",), ("c",)], 8, seed=1) + clf = make(min_predictions=0) + clf.add_examples(train_x, train_y) + assert _predicted(clf, train_x[0], threshold=0.0) == {"a", "b", "c"} # nothing is filtered out + assert _predicted(clf, train_x[0], threshold=1.0) == set() # sigmoid never reaches 1.0 + low = _predicted(clf, train_x[0], threshold=0.001) + high = _predicted(clf, train_x[0], threshold=0.9) + assert high <= low + + +def test_without_a_threshold_the_learned_per_label_ones_are_used(data, make): + train_x, train_y = data.sample([("a",), ("b",), ("c",)], 8, seed=1) + clf = make() + clf.add_examples(train_x, train_y) + clf.label_thresholds["b"] = 0.0 + assert "b" in _predicted(clf, train_x[0]) + + +def test_the_base_class_loader_returns_a_multilabel_classifier(data, make, tmp_path): + from adaptive_classifier import AdaptiveClassifier + + train_x, train_y = data.sample([("a",), ("b",), ("a", "b")], 8, seed=1) + clf = make(default_threshold=0.45) + clf.add_examples(train_x, train_y) + clf.save(str(tmp_path), include_onnx=False) + + loaded = AdaptiveClassifier.load(str(tmp_path), device="cpu", use_onnx=False) + + assert isinstance(loaded, MultiLabelAdaptiveClassifier) + assert loaded.default_threshold == 0.45 + for text in train_x[::5]: + assert dict(loaded.predict_multilabel(text)).keys() == dict(clf.predict_multilabel(text)).keys() + + +def test_predict_accepts_the_abstain_options_of_the_base_class(data, make): + train_x, train_y = data.sample([("a",), ("b",)], 8, seed=1) + clf = make() + clf.add_examples(train_x, train_y) + assert clf.predict(train_x[0], abstain_below=0.0) + assert clf.predict(train_x[0], abstain_below=1.1) == [] + far = "far_away" + data.store[far] = -torch.ones(32) / 32 ** 0.5 + assert clf.predict(far, abstain_ood=True) == [] + + +@pytest.mark.parametrize("method", ["calibrate", "calibration_report", "predict_set"]) +def test_confidence_calibration_is_refused_rather_than_silently_wrong(data, make, method): + clf = make() + with pytest.raises(NotImplementedError, match="one label per text"): + getattr(clf, method)(["x"], ["a"]) + + +def test_strategic_mode_is_refused_for_multilabel(make): + with pytest.raises(ValueError, match="not supported"): + make(config={"enable_strategic_mode": True, "cost_coefficients": [0.1] * 32}) + + +def test_labels_given_as_a_dict_are_rejected(make): + with pytest.raises(ValueError, match="list of label lists"): + make().add_examples(["great love"], [{"a": 1}]) + + +def test_thresholds_follow_merges_and_removals(data, make): + a_x, a_y = data.sample([("a",)], 8, seed=1) + b_x, b_y = data.sample([("b",), ("c",)], 8, seed=2, prefix="o") + one, other = make(), make() + one.add_examples(a_x, a_y) + other.add_examples(b_x, b_y) + one.merge_classifiers(other) + assert set(one.label_thresholds) == {"a", "b", "c"} + + c_texts = [t for t, l in zip(b_x, b_y) if l == ["c"]] + one.remove_examples(c_texts) + assert set(one.label_thresholds) == {"a", "b"} diff --git a/tests/test_serving.py b/tests/test_serving.py index 3de5209..31d6a7f 100644 --- a/tests/test_serving.py +++ b/tests/test_serving.py @@ -283,3 +283,43 @@ def test_cli_reads_the_key_from_the_environment(saved, launched, monkeypatch): def test_cli_fails_clearly_for_a_missing_model(launched, tmp_path): with pytest.raises(FileNotFoundError, match="config.json"): serving.main([str(tmp_path), "--no-onnx"]) + + +# --- unsupported classifiers and a busy event loop ------------------------------------------------------- + +def test_a_multilabel_classifier_is_refused_with_a_clear_message(base_model): + from adaptive_classifier import MultiLabelAdaptiveClassifier + + multi = MultiLabelAdaptiveClassifier(base_model, device="cpu", use_onnx=False) + with pytest.raises(ValueError, match="MultiLabelAdaptiveClassifier"): + create_app(multi) + + +def test_saving_after_an_update_does_not_block_other_requests(clf, tmp_path, monkeypatch): + """A slow save (ONNX export takes seconds) used to freeze /health and /info while it ran.""" + import time + + real_save = clf.save + + def slow_save(path, *args, **kwargs): + time.sleep(1.5) + return real_save(path, *args, **kwargs) + + monkeypatch.setattr(clf, "save", slow_save) + client = updatable(clf, save_dir=str(tmp_path / "saved")) + done = {} + + def update(): + done["response"] = client.post("/examples", json={"texts": ["tr0_0"], "labels": ["C0"]}) + + worker = threading.Thread(target=update) + worker.start() + time.sleep(0.3) # the update is now inside the slow save + started = time.perf_counter() + assert client.get("/health").status_code == 200 + assert client.get("/info").status_code == 200 + elapsed = time.perf_counter() - started + worker.join() + + assert done["response"].status_code == 200 + assert elapsed < 1.0, f"/health and /info waited {elapsed:.2f}s behind a save" diff --git a/tests/test_sklearn_api.py b/tests/test_sklearn_api.py index 75bdf0e..e685d15 100644 --- a/tests/test_sklearn_api.py +++ b/tests/test_sklearn_api.py @@ -220,3 +220,51 @@ def test_underlying_classifier_features_stay_reachable(base_model, separable, tm def test_string_input_tag_is_declared(base_model): tags = est(base_model).__sklearn_tags__() assert tags.input_tags.string + + +# --- classes declared up front and label identity --------------------------------------------- + +def test_partial_fit_lists_declared_classes_even_before_they_are_seen(base_model, separable): + e = est(base_model) + e.partial_fit(["a1", "a2", "b1", "b2"], ["A", "A", "B", "B"], classes=["A", "B", "Z"]) + assert list(e.classes_) == ["A", "B", "Z"] + proba = e.predict_proba(["a1", "b1"]) + assert proba.shape == (2, 3) + assert proba[:, 2].tolist() == [0.0, 0.0] # nothing known about Z yet + assert proba.sum(axis=1).tolist() == pytest.approx([1.0, 1.0]) + assert set(e.predict(["a1", "b1"])) <= {"A", "B"} + + e.partial_fit(["c1", "c2"], ["Z", "Z"]) # when Z arrives it is learned normally + assert list(e.classes_) == ["A", "B", "Z"] + assert e.predict(["c1"])[0] == "Z" + + +@pytest.mark.parametrize("labels", [[1, "1"], [1, "1", 2], [True, "True"]]) +def test_labels_that_would_merge_into_one_class_are_rejected(base_model, separable, labels): + e = est(base_model) + texts = [f"a{i}" for i in range(len(labels))] + with pytest.raises(ValueError, match="both map to the class"): + e.fit(texts, labels) + + +def test_a_clashing_label_in_a_later_partial_fit_leaves_the_estimator_unchanged(base_model, separable): + e = est(base_model) + e.partial_fit(["a1", "a2"], [1, 1]) + with pytest.raises(ValueError, match="both map to the class"): + e.partial_fit(["b1"], ["1"]) + assert list(e.classes_) == [1] + assert e.classifier_.memory.examples["1"].__len__() == 2 + + +@pytest.mark.parametrize("bad", [None, float("nan")]) +def test_none_and_nan_labels_are_rejected(base_model, separable, bad): + with pytest.raises(ValueError, match="None or NaN"): + est(base_model).fit(["a1", "b1"], ["A", bad]) + + +def test_a_fitted_estimator_can_be_pickled(base_model, separable): + import pickle + X, y = data(n=6) + e = est(base_model).fit(X, y) + again = pickle.loads(pickle.dumps(e)) + assert list(again.predict(X[:3])) == list(e.predict(X[:3])) diff --git a/tests/test_state_safety.py b/tests/test_state_safety.py new file mode 100644 index 0000000..cca0b9b --- /dev/null +++ b/tests/test_state_safety.py @@ -0,0 +1,286 @@ +"""State must stay consistent when an operation fails, and operations must do what their names say.""" + +import threading + +import pytest +import torch + +from adaptive_classifier import AdaptiveClassifier +from adaptive_classifier.ewc import EWC + +from .synthetic import Dataset + + +@pytest.fixture +def data(monkeypatch): + return Dataset().patch(monkeypatch) + + +@pytest.fixture +def clf(new_classifier, data): + X, y = data.classes(10, n_classes=3, noise=0.15, seed=1) + c = new_classifier(head_steps=30) + c.add_examples(X, y) + return c + + +def state_of(c): + return { + "labels": dict(c.label_to_id), + "ids": dict(c.id_to_label), + "history": dict(c.training_history), + "counts": {k: len(v) for k, v in c.memory.examples.items()}, + "head_out": c.adaptive_head.model[-1].out_features, + "prototypes": {k: v.clone() for k, v in c.memory.prototypes.items()}, + "steps": c.train_steps, + } + + +def assert_unchanged(c, before): + after = state_of(c) + assert {k: v for k, v in after.items() if k != "prototypes"} == {k: v for k, v in before.items() if k != "prototypes"} + assert after["prototypes"].keys() == before["prototypes"].keys() + assert all(torch.equal(after["prototypes"][k], before["prototypes"][k]) for k in before["prototypes"]) + + +# --- a failed add_examples changes nothing ----------------------------------------------------- + +@pytest.mark.parametrize("failure", [RuntimeError("boom"), KeyboardInterrupt()], ids=["error", "interrupt"]) +@pytest.mark.parametrize("where", ["_train_adaptive_head", "_train_new_classes"]) +def test_a_failure_while_training_rolls_everything_back(clf, data, monkeypatch, failure, where): + before = state_of(clf) + reference = clf.predict("tr1_3", k=3) + + def explode(*args, **kwargs): + raise failure + + monkeypatch.setattr(clf, where, explode) + if where == "_train_adaptive_head": + texts, labels = data.classes(4, 3, seed=5, prefix="x") # existing classes only + else: + texts, labels = ["nx0", "nx1"], ["NEW", "NEW"] # a new class + data.add("nx0", 7, 0.1, 1), data.add("nx1", 7, 0.1, 2) + with pytest.raises(type(failure)): + clf.add_examples(texts, labels) + + assert_unchanged(clf, before) + assert clf.predict("tr1_3", k=3) == reference + assert clf.predict("tr0_0", k=1)[0][0] == "C0" # the search index is intact too + + monkeypatch.undo() # and the classifier still works + data.patch(monkeypatch) + clf.add_examples(texts, labels) + assert set(labels) <= set(clf.label_to_id) + + +def test_a_failure_while_encoding_changes_nothing(clf, monkeypatch): + before = state_of(clf) + + def explode(texts): + raise RuntimeError("encoder down") + + monkeypatch.setattr(clf, "_get_embeddings", explode) + with pytest.raises(RuntimeError): + clf.add_examples(["a new text"], ["BRAND_NEW"]) + assert_unchanged(clf, before) + + +def test_a_failure_discards_nothing_from_calibration(clf, data, monkeypatch): + X, y = data.classes(6, 3, seed=8, prefix="cal") + clf.calibrate(X, y) + monkeypatch.setattr(clf, "_train_new_classes", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("x"))) + data.add("q0", 9, 0.1, 1) + with pytest.raises(RuntimeError): + clf.add_examples(["q0"], ["Q"]) + assert clf.calibration + + +# --- clear_memory without labels really clears -------------------------------------------------- + +def test_clear_memory_with_no_labels_empties_the_classifier(clf, data): + X, y = data.classes(6, 3, seed=8, prefix="cal") + clf.calibrate(X, y) + + clf.clear_memory() + + assert clf.label_to_id == {} and clf.id_to_label == {} and clf.training_history == {} + assert clf.adaptive_head is None and clf.calibration is None + assert clf.predict("tr0_0") == [] + assert clf.ood_score("tr0_0") == float("inf") + assert clf.get_memory_stats()["num_classes"] == 0 + + texts, labels = data.classes(8, 2, seed=3, prefix="again") # and it can start over + clf.add_examples(texts, labels) + assert clf.predict("again0_1", k=1)[0][0] == "C0" + + +# --- EWC --------------------------------------------------------------------------------------------- + +def _small_head(): + from adaptive_classifier.models import AdaptiveHead + torch.manual_seed(0) + return AdaptiveHead(8, 3, hidden_dims=[8, 4]) + + +def _dataset(n=24): + g = torch.Generator().manual_seed(0) + return torch.utils.data.TensorDataset(torch.randn(n, 8, generator=g), torch.randint(0, 3, (n,), generator=g)) + + +def test_ewc_penalty_is_zero_at_the_stored_weights_and_grows_as_the_model_moves(): + head = _small_head() + ewc = EWC(head, _dataset(), ewc_lambda=10.0) + assert ewc.ewc_loss(model=head).item() == 0.0 + with torch.no_grad(): + for p in head.parameters(): + p.add_(0.1) + small = ewc.ewc_loss(model=head).item() + with torch.no_grad(): + for p in head.parameters(): + p.add_(0.1) + assert 0 < small < ewc.ewc_loss(model=head).item() + + +def test_ewc_compares_only_the_rows_that_existed_before_new_classes_were_added(): + import copy + old = _small_head() + ewc = EWC(copy.deepcopy(old), _dataset()) + grown = copy.deepcopy(old) + grown.update_num_classes(5) # two new output rows + assert ewc.ewc_loss(model=grown).item() == pytest.approx(0.0) # unchanged old rows cost nothing + with torch.no_grad(): + grown.model[-1].weight[:3] += 0.5 + assert ewc.ewc_loss(model=grown).item() > 0 + grown2 = copy.deepcopy(old) + grown2.update_num_classes(5) + with torch.no_grad(): + grown2.model[-1].weight[3:] += 5.0 # changing only the new rows is free + assert ewc.ewc_loss(model=grown2).item() == pytest.approx(0.0) + + +def test_adding_a_class_applies_a_real_ewc_penalty(clf, data, monkeypatch): + import adaptive_classifier.classifier as module + penalties = [] + real = module.EWC.ewc_loss + + def spy(self, batch_size=None, model=None): + value = real(self, batch_size=batch_size, model=model) + penalties.append(float(value)) + return value + + monkeypatch.setattr(module.EWC, "ewc_loss", spy) + texts = [data.add(f"n{i}", 6, 0.1, i) for i in range(10)] + clf.add_examples(texts, ["NEW"] * 10) + assert penalties and max(penalties) > 0 + + +# --- merging ------------------------------------------------------------------------------------------ + +def test_merging_two_classifiers_into_each_other_at_once_cannot_deadlock(new_classifier, data): + a, b = new_classifier(head_steps=20), new_classifier(head_steps=20) + x1, y1 = data.classes(6, 2, seed=1, prefix="a") + x2, y2 = [data.add(f"b{i}", 4 + i % 2, 0.1, i) for i in range(12)], ["D0", "D1"] * 6 + a.add_examples(x1, y1) + b.add_examples(x2, y2) + threads = [threading.Thread(target=a.merge_classifiers, args=(b,), daemon=True), + threading.Thread(target=b.merge_classifiers, args=(a,), daemon=True)] + for t in threads: + t.start() + for t in threads: + t.join(120) + assert not any(t.is_alive() for t in threads), "the two merges are waiting on each other" + + +def test_merge_keeps_the_larger_recorded_class_spread(new_classifier, data): + a, b = new_classifier(head_steps=20), new_classifier(head_steps=20) + x, y = data.classes(8, 2, seed=1, prefix="a") + a.add_examples(x, y) + b.add_examples(x, y) + b.memory.saved_radii["C0"] = 5.0 # as if recorded from many more examples + a.merge_classifiers(b) + assert a.memory.class_radius("C0") >= 5.0 + + +# --- a reloaded classifier keeps its prototypes where they were ---------------------------------------------- + +def test_adding_one_example_to_a_reloaded_class_barely_moves_its_prototype(new_classifier, data, tmp_path): + texts, labels = data.classes(30, n_classes=3, noise=0.3, seed=1) + live = new_classifier(head_steps=20) + live.add_examples(texts, labels) + live.save(str(tmp_path), include_onnx=False) + loaded = AdaptiveClassifier.load(str(tmp_path), device="cpu", use_onnx=False) + assert len(loaded.memory.examples["C0"]) < 30 # only representative examples came back + saved = loaded.memory.prototypes["C0"].clone() + + extra = data.add("extra", 0, 0.3, 999) + live.add_examples([extra], ["C0"]) + loaded.add_examples([extra], ["C0"]) + + assert torch.dist(loaded.memory.prototypes["C0"], saved) < 0.05 # 1 of 31 examples + assert torch.dist(loaded.memory.prototypes["C0"], live.memory.prototypes["C0"]) < 0.01 + + +def test_a_reloaded_prototype_follows_new_examples_in_proportion(new_classifier, data, tmp_path): + texts, labels = data.classes(20, n_classes=2, noise=0.1, seed=1) + live = new_classifier(head_steps=20) + live.add_examples(texts, labels) + live.save(str(tmp_path), include_onnx=False) + loaded = AdaptiveClassifier.load(str(tmp_path), device="cpu", use_onnx=False) + more = [data.add(f"m{i}", 0, 0.1, 500 + i) for i in range(20)] + live.add_examples(more, ["C0"] * 20) + loaded.add_examples(more, ["C0"] * 20) + assert torch.dist(loaded.memory.prototypes["C0"], live.memory.prototypes["C0"]) < 0.01 + + +def test_removing_examples_from_a_reloaded_class_recomputes_its_prototype_from_what_is_held(new_classifier, data, tmp_path): + texts, labels = data.classes(20, n_classes=2, noise=0.1, seed=1) + live = new_classifier(head_steps=20) + live.add_examples(texts, labels) + live.save(str(tmp_path), include_onnx=False) + loaded = AdaptiveClassifier.load(str(tmp_path), device="cpu", use_onnx=False) + held = [ex.text for ex in loaded.memory.examples["C0"]] + loaded.remove_examples(held[:1], label="C0") + assert "C0" not in loaded.memory.prototype_base + assert loaded.predict("tr0_0", k=1)[0][0] == "C0" + + +# --- ONNX round trip and batching ---------------------------------------------------------------------- + +def test_resaving_an_onnx_loaded_classifier_keeps_the_original_model_name(base_model, tmp_path): + pytest.importorskip("optimum.onnxruntime") + import json + clf = AdaptiveClassifier(base_model, config={"pooling": "mean"}, device="cpu", use_onnx=False) + clf.add_examples(["great love", "good fast", "terrible hate", "awful bad"] * 3, ["p", "p", "n", "n"] * 3) + first, second = tmp_path / "first", tmp_path / "second" + clf.save(str(first), include_onnx=True, quantize_onnx=False) + + loaded = AdaptiveClassifier.load(str(first), use_onnx=True, prefer_quantized=False, device="cpu") + loaded.save(str(second), include_onnx=False) + + assert json.loads((second / "config.json").read_text())["model_name"] == base_model + + +def test_texts_are_encoded_in_batches_of_the_configured_size(new_classifier): + c = new_classifier(batch_size=4) + sizes = [] + real = c.model.forward + + def spy(*args, **kwargs): + sizes.append(kwargs["input_ids"].shape[0]) + return real(*args, **kwargs) + + c.model.forward = spy + texts = ["great love", "good fast", "terrible hate"] * 4 # 12 texts + embeddings = c._get_embeddings(texts) + assert sizes == [4, 4, 4] and len(embeddings) == 12 + + c.config.batch_size = 100 + sizes.clear() + whole = c._get_embeddings(texts) + assert sizes == [12] + assert all(torch.allclose(a, b, atol=1e-5) for a, b in zip(embeddings, whole)) + + +def test_remove_examples_rejects_a_bare_string(clf): + with pytest.raises(ValueError, match="not a single string"): + clf.remove_examples("tr0_0") diff --git a/tests/test_strategic.py b/tests/test_strategic.py new file mode 100644 index 0000000..253a5cd --- /dev/null +++ b/tests/test_strategic.py @@ -0,0 +1,355 @@ +"""Strategic (anti-gaming) mode: set-up, prediction invariants, determinism and persistence. + +Uses hand-made embeddings (synthetic.py, 32 dimensions to match the tiny test encoder). +""" + +import logging +import threading + +import pytest +import torch + +from adaptive_classifier import AdaptiveClassifier +from adaptive_classifier.strategic import ( + CostFunctionFactory, LinearCostFunction, SeparableCostFunction, StrategicEvaluator, +) +from .synthetic import DIM, Dataset + + +def _config(cost=0.3, **extra): + return { + "enable_strategic_mode": True, + "cost_function_type": "linear", + "cost_coefficients": [cost] * DIM, + "strategic_training_frequency": 1000, # keep fitting fast; training is tested separately + **extra, + } + + +@pytest.fixture +def data(monkeypatch, base_model): + return Dataset().patch(monkeypatch) + + +@pytest.fixture +def trained(data, base_model): + def build(config=None, n_per=10, n_classes=3, noise=0.1): + clf = AdaptiveClassifier(base_model, config=config or _config(), device="cpu", use_onnx=False) + texts, labels = data.classes(n_per, n_classes, noise=noise, seed=1) + clf.add_examples(texts, labels) + return clf, texts, labels + return build + + +# --- set-up ------------------------------------------------------------------- + +@pytest.mark.parametrize("kind", ["linear", "separable"]) +def test_list_coefficients_enable_strategic_mode(base_model, kind): + clf = AdaptiveClassifier(base_model, config=_config(cost_function_type=kind), device="cpu", use_onnx=False) + assert clf.strategic_mode + + +def test_strategic_mode_is_off_by_default(base_model): + assert not AdaptiveClassifier(base_model, device="cpu", use_onnx=False).strategic_mode + + +@pytest.mark.parametrize("coefficients, why", [ + ({"sentiment_words": 0.5, "length_change": 0.1}, "dict"), # the form the README once showed + ([0.3] * 5, "length"), # not one per embedding dimension + ([], "empty"), +]) +def test_unusable_coefficients_leave_strategic_mode_off_and_say_why(base_model, caplog, coefficients, why): + config = _config() + config["cost_coefficients"] = coefficients + with caplog.at_level(logging.WARNING): + clf = AdaptiveClassifier(base_model, config=config, device="cpu", use_onnx=False) + assert not clf.strategic_mode + assert caplog.records, "no explanation was logged" + message = " ".join(r.getMessage() for r in caplog.records) + assert "cost_coefficients" in message + + +def test_unknown_cost_function_type_is_reported(base_model, caplog): + with caplog.at_level(logging.WARNING): + clf = AdaptiveClassifier(base_model, config=_config(cost_function_type="quadratic"), + device="cpu", use_onnx=False) + assert not clf.strategic_mode + assert "quadratic" in " ".join(r.getMessage() for r in caplog.records) + + +def test_factory_rejects_unknown_types_and_dicts_without_names(): + with pytest.raises(ValueError): + CostFunctionFactory.create_cost_function("nope", [0.1] * 4) + with pytest.raises(ValueError): + LinearCostFunction({"a": 1.0}) + + +# --- cost functions ----------------------------------------------------------------- + +def test_linear_cost_is_zero_for_moves_that_do_not_raise_the_score(): + cost = LinearCostFunction(torch.tensor([1.0, 0.0, 0.0])) + x = torch.tensor([0.5, 0.0, 0.0]) + assert cost.compute_cost(x, torch.tensor([0.2, 0.0, 0.0])).item() == 0.0 # cheaper direction + assert cost.compute_cost(x, torch.tensor([0.9, 0.0, 0.0])).item() == pytest.approx(0.4) + assert cost.compute_cost(x, x).item() == 0.0 + + +def test_best_response_can_use_any_dimension_not_just_the_first_few(): + """Moving along the only free dimension (the last of 32) is the cheapest way to raise the score.""" + alpha = torch.ones(DIM) + alpha[-1] = 0.0 # free to change + cost = SeparableCostFunction(alpha, alpha) + x = torch.zeros(DIM) + + def score(batch): # the classifier likes a high last feature + return torch.sigmoid(batch[:, -1:] * 4) + + response = cost.compute_best_response(x, score) + assert response[-1] > 0, "best response never explored the last dimension" + + +def test_best_response_is_deterministic(): + cost = LinearCostFunction(torch.full((DIM,), 0.2)) + x = torch.linspace(-1, 1, DIM) + + def score(batch): + return torch.softmax(batch[:, :3], dim=-1) + + assert torch.equal(cost.compute_best_response(x, score), cost.compute_best_response(x, score)) + + +def test_a_prohibitive_cost_stops_the_moves_that_would_raise_the_score(): + """Costs only apply to moves that raise ; with a huge alpha no such move pays off.""" + cost = LinearCostFunction(torch.full((DIM,), 1000.0)) + x = torch.zeros(DIM) + + def likes_a_high_sum(batch): # the classifier is happier the larger the sum + return torch.sigmoid(batch.sum(dim=-1, keepdim=True)) + + assert cost.compute_best_response(x, likes_a_high_sum).sum().item() <= 1e-6 + + +def test_a_free_direction_is_taken(): + cost = LinearCostFunction(torch.zeros(DIM)) + x = torch.zeros(DIM) + response = cost.compute_best_response(x, lambda b: torch.sigmoid(b.sum(dim=-1, keepdim=True))) + assert response.sum().item() > 0 + + +# --- predictions ---------------------------------------------------------------------- + +@pytest.mark.parametrize("method", ["predict", "predict_strategic", "predict_robust"]) +def test_predictions_are_ranked_probability_distributions(trained, method): + clf, texts, _ = trained() + result = getattr(clf, method)(texts[0], k=3) + scores = [s for _, s in result] + assert {label for label, _ in result} <= set(clf.label_to_id) + assert scores == sorted(scores, reverse=True) + assert sum(scores) == pytest.approx(1.0, abs=1e-5) + assert all(0.0 <= s <= 1.0 for s in scores) + + +@pytest.mark.parametrize("method", ["predict", "predict_strategic", "predict_robust"]) +def test_scores_do_not_depend_on_how_many_results_are_requested(trained, method): + clf, texts, _ = trained() + full = dict(getattr(clf, method)(texts[0], k=3)) + top = getattr(clf, method)(texts[0], k=1) + assert top[0][0] == max(full, key=full.get) + assert top[0][1] == pytest.approx(full[top[0][0]], abs=1e-5) + + +@pytest.mark.parametrize("method", ["predict", "predict_strategic", "predict_robust"]) +def test_k_is_respected(trained, method): + clf, texts, _ = trained() + assert len(getattr(clf, method)(texts[0], k=2)) == 2 + assert len(getattr(clf, method)(texts[0], k=100)) == 3 + + +def test_predictions_are_repeatable(trained): + clf, texts, _ = trained() + assert clf.predict(texts[0], k=3) == clf.predict(texts[0], k=3) + assert clf.predict_strategic(texts[0], k=3) == clf.predict_strategic(texts[0], k=3) + + +def test_separable_data_is_still_classified_correctly(trained, data): + clf, _, _ = trained() + texts, labels = data.classes(8, 3, noise=0.1, seed=9, prefix="v") + hits = sum(clf.predict(t)[0][0] == l for t, l in zip(texts, labels)) + assert hits / len(texts) >= 0.9 + + +def test_with_strategic_mode_off_every_method_matches_regular_prediction(data, base_model): + clf = AdaptiveClassifier(base_model, device="cpu", use_onnx=False) + texts, labels = data.classes(8, 3, seed=1) + clf.add_examples(texts, labels) + regular = clf.predict(texts[0], k=3) + assert clf.predict_strategic(texts[0], k=3) == regular + assert clf.predict_robust(texts[0], k=3) == regular + + +def test_robust_prediction_follows_the_prototypes_when_the_head_has_no_weight(trained): + clf, texts, _ = trained(_config(strategic_robust_proto_weight=1.0, strategic_robust_head_weight=0.0)) + embedding = clf._get_embeddings([texts[0]])[0] + proto = [label for label, _ in clf.memory.get_nearest_prototypes(embedding, k=3)] + assert [label for label, _ in clf.predict_robust(texts[0], k=3)] == proto + + +def test_dual_prediction_blends_the_regular_and_strategic_ones(trained): + config = _config(strategic_blend_regular_weight=1.0, strategic_blend_strategic_weight=0.0) + clf, texts, _ = trained(config) + dual = dict(clf.predict(texts[0], k=3)) + regular = dict(clf._predict_regular(texts[0], 3)) + assert dual.keys() == regular.keys() + assert all(dual[l] == pytest.approx(regular[l], abs=1e-5) for l in dual) + + +def test_bad_input_is_rejected_like_regular_prediction(trained): + clf, texts, _ = trained() + with pytest.raises(ValueError): + clf.predict("") + with pytest.raises(ValueError): + clf.predict(texts[0], k=-1) + + +# --- training with the strategic loss ------------------------------------------------------- + +def test_strategic_training_steps_run_and_keep_the_classifier_accurate(data, base_model): + config = _config(strategic_training_frequency=1) # every update trains strategically + clf = AdaptiveClassifier(base_model, config=config, device="cpu", use_onnx=False) + texts, labels = data.classes(6, 3, noise=0.1, seed=1) + clf.add_examples(texts, labels) + clf.add_examples(*data.classes(6, 3, noise=0.1, seed=2, prefix="u")) + test_x, test_y = data.classes(8, 3, noise=0.1, seed=9, prefix="v") + hits = sum(clf.predict(t)[0][0] == l for t, l in zip(test_x, test_y)) + assert hits / len(test_x) >= 0.9 + + +def test_a_new_class_can_be_added_in_strategic_mode(trained, data): + clf, _, _ = trained(n_classes=2) + new_x, new_y = data.classes(10, 3, seed=5, prefix="n") + clf.add_examples([t for t, l in zip(new_x, new_y) if l == "C2"], ["C2"] * 10) + assert "C2" in clf.label_to_id + assert clf.predict_strategic(new_x[-1], k=3) + + +# --- robustness evaluation -------------------------------------------------------------------- + +def test_evaluate_strategic_robustness_reports_sane_numbers(trained, data): + clf, _, _ = trained() + texts, labels = data.classes(6, 3, seed=9, prefix="e") + report = clf.evaluate_strategic_robustness(texts, labels) + for level in (0.0, 0.5, 1.0): + assert 0.0 <= report[f"accuracy_gaming_{level}"] <= 1.0 + assert report["accuracy_gaming_0.0"] >= 0.9 + assert report["robustness_score"] == pytest.approx(report["accuracy_gaming_0.0"] - report["accuracy_gaming_1.0"]) + + +def test_evaluate_strategic_robustness_is_repeatable(trained, data): + clf, _, _ = trained() + texts, labels = data.classes(6, 3, seed=9, prefix="e") + assert clf.evaluate_strategic_robustness(texts, labels) == clf.evaluate_strategic_robustness(texts, labels) + + +def test_evaluate_strategic_robustness_needs_strategic_mode(data, base_model): + clf = AdaptiveClassifier(base_model, device="cpu", use_onnx=False) + with pytest.raises(ValueError): + clf.evaluate_strategic_robustness(["a"], ["b"]) + + +def test_evaluator_survives_a_classifier_that_gets_everything_wrong(): + """Accuracy of zero at no gaming used to divide by zero.""" + evaluator = StrategicEvaluator(LinearCostFunction(torch.full((4,), 0.1))) + always_class_1 = lambda x: torch.tensor([[0.0, 1.0]]).expand(len(x), 2) + report = evaluator.evaluate_robustness(always_class_1, torch.zeros(5, 4), torch.zeros(5, dtype=torch.long)) + assert report["accuracy_gaming_0.0"] == 0.0 + assert report["relative_robustness"] != report["relative_robustness"] # nan, not an exception + + +def test_evaluate_strategic_robustness_rejects_unknown_labels(trained, data): + clf, _, _ = trained() + with pytest.raises(ValueError): + clf.evaluate_strategic_robustness(["C0_0"], ["not-a-class"]) + + +# --- persistence and concurrency ---------------------------------------------------------------- + +def test_save_and_load_keep_strategic_mode_and_predictions(trained, tmp_path): + clf, texts, _ = trained(_config(strategic_robust_proto_weight=0.9, strategic_robust_head_weight=0.1)) + clf.save(str(tmp_path), include_onnx=False) + loaded = AdaptiveClassifier.load(str(tmp_path), device="cpu", use_onnx=False) + + assert loaded.strategic_mode + assert loaded.config.strategic_robust_proto_weight == 0.9 + for method in ("predict", "predict_strategic", "predict_robust"): + a, b = dict(getattr(clf, method)(texts[0], k=3)), dict(getattr(loaded, method)(texts[0], k=3)) + assert a.keys() == b.keys() + assert all(a[l] == pytest.approx(b[l], abs=1e-4) for l in a) + + +def test_strategic_predictions_survive_concurrent_updates(trained, data): + clf, texts, _ = trained() + extra_x, extra_y = data.classes(6, 4, seed=3, prefix="x") + extra = [(t, l) for t, l in zip(extra_x, extra_y) if l == "C3"] + stop, errors = threading.Event(), [] + + def reader(): + while not stop.is_set(): + try: + clf.predict(texts[0], k=3) + clf.predict_robust(texts[1], k=3) + except Exception as error: # noqa: BLE001 + errors.append(error) + return + + threads = [threading.Thread(target=reader) for _ in range(2)] + for t in threads: + t.start() + try: + for _ in range(3): + clf.add_examples([t for t, _ in extra], [l for _, l in extra]) + clf.forget("C3") + finally: + stop.set() + for t in threads: + t.join() + assert not errors, errors[0] + + +def test_predict_batch_agrees_with_predict(trained): + clf, texts, _ = trained() + for text, batch in zip(texts[::6], clf.predict_batch(texts[::6], k=3)): + single = clf.predict(text, k=3) + assert [l for l, _ in batch] == [l for l, _ in single] + assert all(b == pytest.approx(s, abs=1e-5) for (_, b), (_, s) in zip(batch, single)) + + +# --- input checks on the strategic entry points ---------------------------------------------------- + +@pytest.mark.parametrize("method", ["predict_strategic", "predict_robust"]) +def test_strategic_entry_points_validate_input_like_predict(trained, method): + clf, texts, _ = trained() + with pytest.raises(ValueError): + getattr(clf, method)("") + with pytest.raises(ValueError): + getattr(clf, method)(texts[0], k=-1) + with pytest.raises(ValueError): + getattr(clf, method)(texts[0], k="x") + assert getattr(clf, method)(texts[0], k=0) == [] + + +@pytest.mark.parametrize("levels", [[0.5], [0.25, 0.75], (0.0, 1.0), [0.5, 0.5]]) +def test_robustness_evaluation_always_includes_the_baselines(trained, data, levels): + clf, _, _ = trained() + texts, labels = data.classes(4, 3, seed=9, prefix="e") + report = clf.evaluate_strategic_robustness(texts, labels, gaming_levels=levels) + assert {"accuracy_gaming_0.0", "accuracy_gaming_1.0", "robustness_score"} <= set(report) + for level in levels: + assert f"accuracy_gaming_{float(level)}" in report + + +@pytest.mark.parametrize("levels", [[], [-0.1], [1.5], ["half"], [None], [True]]) +def test_robustness_evaluation_rejects_bad_levels(trained, data, levels): + clf, _, _ = trained() + texts, labels = data.classes(4, 3, seed=9, prefix="e") + with pytest.raises(ValueError, match="gaming_levels"): + clf.evaluate_strategic_robustness(texts, labels, gaming_levels=levels)