Stronger testing foundation for v1.0

bertopic/representation/_prompts.py

@@ -32,7 +32,7 @@
 """
 
 DEFAULT_JSON_PROMPT = """You will extract various topic details from a number of given documents and keywords.
-The documents are merely a subset of all documents related to the topic, but they are representative of the overall topic. 
+The documents are merely a subset of all documents related to the topic, but they are representative of the overall topic.
 The keywords are the most relevant keywords for this topic.
 
 # Texts

tests/conftest.py

@@ -1,4 +1,6 @@
 import copy
+import importlib.util
+
 import pytest
 from umap import UMAP
 from hdbscan import HDBSCAN
@@ -13,6 +15,32 @@
 from sklearn.linear_model import LogisticRegression
 
 
+def cuml_available():
+    """Check whether cuML is installed, since the GPU fixtures require CUDA."""
+    try:
+        return importlib.util.find_spec("cuml") is not None
+    except ImportError:
+        return False
+
+
+# Every fitted model fixture, for tests that must hold across all pipeline variants
+ALL_MODEL_FIXTURES = [
+    "base_topic_model",
+    "kmeans_pca_topic_model",
+    "custom_topic_model",
+    "merged_topic_model",
+    "reduced_topic_model",
+    "online_topic_model",
+    "supervised_topic_model",
+    "representation_topic_model",
+    "zeroshot_topic_model",
+    pytest.param(
+        "cuml_base_topic_model",
+        marks=pytest.mark.skipif(not cuml_available(), reason="cuML not available"),
+    ),
+]
+
+
 @pytest.fixture(scope="session")
 def embedding_model():
     model = SentenceTransformer("all-MiniLM-L6-v2")
@@ -46,6 +74,13 @@ def targets():
     return y
 
 
+@pytest.fixture(scope="session")
+def classes(targets):
+    """Human-readable class name per document, aligned with the `documents` fixture."""
+    data = fetch_20newsgroups(subset="all", remove=("headers", "footers", "quotes"))
+    return [data["target_names"][target] for target in targets]
+
+
 @pytest.fixture(scope="session")
 def base_topic_model(documents, document_embeddings, embedding_model):
     model = BERTopic(embedding_model=embedding_model, calculate_probabilities=True)
@@ -188,3 +223,21 @@ def cuml_base_topic_model(documents, document_embeddings, embedding_model):
     )
     model.fit(documents, document_embeddings)
     return model
+
+
+@pytest.fixture(scope="session")
+def image_paths(tmp_path_factory):
+    """Write small solid-colour images to disk and return their file paths."""
+    from PIL import Image
+
+    directory = tmp_path_factory.mktemp("images")
+    paths = []
+
+    for index in range(30):
+        # Cycle through red, green, and blue so the images form three clear groups
+        color = [(200, 40, 40), (40, 200, 40), (40, 40, 200)][index % 3]
+        path = directory / f"image_{index}.png"
+        Image.new("RGB", (32, 32), color).save(path)
+        paths.append(str(path))
+
+    return paths

tests/test_bertopic.py

@@ -1,35 +1,12 @@
 import copy
 import pytest
 from bertopic import BERTopic
-import importlib.util
 import polars as pl
 
+from tests.conftest import ALL_MODEL_FIXTURES
 
-def cuml_available():
-    try:
-        return importlib.util.find_spec("cuml") is not None
-    except ImportError:
-        return False
-
-
-@pytest.mark.parametrize(
-    "model",
-    [
-        ("base_topic_model"),
-        ("kmeans_pca_topic_model"),
-        ("custom_topic_model"),
-        ("merged_topic_model"),
-        ("reduced_topic_model"),
-        ("online_topic_model"),
-        ("supervised_topic_model"),
-        ("representation_topic_model"),
-        ("zeroshot_topic_model"),
-        pytest.param(
-            "cuml_base_topic_model",
-            marks=pytest.mark.skipif(not cuml_available(), reason="cuML not available"),
-        ),
-    ],
-)
+
+@pytest.mark.parametrize("model", ALL_MODEL_FIXTURES)
 def test_full_model(model, documents, request):
     """Tests the entire pipeline in one go. This serves as a sanity check to see if the default
     settings result in a good separation of topics.

tests/test_corpus.py

@@ -0,0 +1,75 @@
+"""Contract tests for the Corpus container.
+
+`Corpus` is the value object that carries documents, embeddings, and assignments
+through the pipeline. These tests pin down what it accepts, which matters because
+its `__post_init__` validation is what currently rejects image-only input.
+"""
+
+import importlib.util
+
+import numpy as np
+import pytest
+
+from bertopic._corpus import Corpus
+
+
+def pillow_available():
+    """Check whether Pillow is installed, since it is only in the `vision` extra."""
+    try:
+        return importlib.util.find_spec("PIL") is not None
+    except ImportError:
+        return False
+
+
+def test_documents_are_normalised_to_a_list():
+    """A single document may be passed as a bare string during inference."""
+    corpus = Corpus(documents="a single document")
+
+    assert corpus.documents == ["a single document"]
+
+
+def test_numpy_documents_are_converted_to_a_list():
+    """Document arrays are accepted and stored as a plain list."""
+    corpus = Corpus(documents=np.array(["first", "second"]))
+
+    assert corpus.documents == ["first", "second"]
+
+
+def test_original_indices_default_to_positions():
+    """Without explicit indices each document is identified by its position."""
+    corpus = Corpus(documents=["first", "second", "third"])
+
+    assert list(corpus.original_indices) == [0, 1, 2]
+
+
+def test_mismatched_embeddings_are_rejected():
+    """An embedding matrix must carry exactly one row per document."""
+    with pytest.raises(ValueError):
+        Corpus(documents=["first", "second"], embeddings=np.zeros((3, 8)))
+
+
+def test_mismatched_topics_are_rejected():
+    """Assignments must line up with documents, which the length guard enforces."""
+    corpus = Corpus(documents=["first", "second"])
+
+    with pytest.raises(ValueError):
+        corpus.topics = np.array([0, 1, 2])
+
+
+@pytest.mark.skipif(not pillow_available(), reason="Pillow not available")
+@pytest.mark.xfail(
+    strict=True,
+    reason="Bug 1: image-only input raises in __post_init__; fixed in unit 2",
+)
+def test_images_may_be_supplied_without_documents(image_paths):
+    """Multimodal input has no documents, which is the documented API for images.
+
+    `docs/getting_started/multimodal/multimodal.md` calls
+    `fit_transform(documents=None, images=images)`, but `check_documents_type` rejects
+    `None` before any embedding happens, so `has_only_images` can never be True and
+    `_images_to_text` is unreachable.
+    """
+    corpus = Corpus(documents=None, images=image_paths)
+
+    assert corpus.has_only_images
+    assert len(corpus.images) == len(image_paths)

tests/test_invariants.py

@@ -0,0 +1,162 @@
+"""Structural invariants that must hold for every BERTopic pipeline variant.
+
+These are deliberately not snapshots. Snapshotting the pipeline variant matrix is
+impractical, and UMAP is not bit-reproducible across platforms and versions, so such
+snapshots would be flaky and expensive to maintain. Invariants sidestep both problems:
+they survive intentional changes, and a failure names the rule that broke rather than
+just reporting that something moved.
+
+Every test runs against all nine model fixtures, so a change that holds for HDBSCAN but
+breaks KMeans, zero-shot, online, or supervised modelling is caught here.
+"""
+
+import copy
+
+import numpy as np
+import pytest
+
+from bertopic import BERTopic
+from tests.conftest import ALL_MODEL_FIXTURES
+
+
+@pytest.mark.parametrize("model", ALL_MODEL_FIXTURES)
+def test_every_document_has_exactly_one_topic(model, documents, request):
+    """There is one topic assignment per document, no more and no fewer."""
+    topic_model = request.getfixturevalue(model)
+
+    assert len(topic_model.topics_) == len(documents)
+
+
+@pytest.mark.parametrize("model", ALL_MODEL_FIXTURES)
+def test_topic_sizes_account_for_every_document(model, documents, request):
+    """Document counts across all topics add up to the size of the corpus."""
+    topic_model = request.getfixturevalue(model)
+
+    assert sum(topic_model.topic_sizes_.values()) == len(documents)
+
+
+@pytest.mark.parametrize("model", ALL_MODEL_FIXTURES)
+def test_assigned_topics_and_known_topics_agree(model, request):
+    """Every topic that documents point at exists, and every topic holds documents."""
+    topic_model = request.getfixturevalue(model)
+
+    assert set(topic_model.topics_) == set(topic_model.topic_sizes_)
+
+
+@pytest.mark.parametrize("model", ALL_MODEL_FIXTURES)
+def test_every_prediction_refers_to_a_known_topic(model, request):
+    """No document points at a topic that was deleted or merged away."""
+    topic_model = request.getfixturevalue(model)
+    known_topics = set(topic_model.topic_sizes_)
+
+    assert all(prediction in known_topics for prediction in topic_model.topics_)
+
+
+@pytest.mark.parametrize("model", ALL_MODEL_FIXTURES)
+def test_topic_ids_are_contiguous(model, request):
+    """Topic IDs run from 0 upwards with no gaps, preceded by -1 when outliers exist."""
+    topic_model = request.getfixturevalue(model)
+    topic_ids = topic_model._topics.topic_ids()
+
+    expected_start = -1 if -1 in topic_ids else 0
+    assert topic_ids == list(range(expected_start, len(topic_ids) + expected_start))
+
+
+@pytest.mark.parametrize("model", ALL_MODEL_FIXTURES)
+def test_topic_matrices_have_one_row_per_topic(model, request):
+    """c-TF-IDF and topic embeddings stay aligned with the set of topics."""
+    topic_model = request.getfixturevalue(model)
+    nr_topics = len(topic_model.topic_sizes_)
+
+    assert topic_model.c_tf_idf_.shape[0] == nr_topics
+    assert topic_model.topic_embeddings_.shape[0] == nr_topics
+
+
+@pytest.mark.parametrize("model", ALL_MODEL_FIXTURES)
+def test_every_topic_has_a_representation(model, request):
+    """No topic is left without a Main representation to describe it."""
+    topic_model = request.getfixturevalue(model)
+
+    for topic in topic_model._topics:
+        assert "Main" in topic.representations
+
+
+@pytest.mark.parametrize("model", ALL_MODEL_FIXTURES)
+def test_probability_matrix_has_one_column_per_topic(model, request):
+    """The probability matrix carries a column for every topic, outlier included.
+
+    The width is already correct everywhere, because the mapping sizes the matrix from
+    the number of distinct target topics. What is wrong today is the *content*: columns
+    are shifted by one and the outlier's mass is dropped. That is specified separately
+    in `tests/test_topics.py`, which is where the fix will be verified.
+    """
+    topic_model = request.getfixturevalue(model)
+    probabilities = topic_model.probabilities_
+
+    if probabilities is None or probabilities.ndim == 1:
+        pytest.skip("This model does not produce a full probability distribution")
+
+    assert probabilities.shape[1] == len(topic_model._topics.topic_ids())
+
+
+@pytest.mark.parametrize("model", ALL_MODEL_FIXTURES)
+def test_probability_rows_match_the_document_count(model, documents, request):
+    """However wide the probability matrix is, it has one row per document."""
+    topic_model = request.getfixturevalue(model)
+    probabilities = topic_model.probabilities_
+
+    if probabilities is None:
+        pytest.skip("This model does not produce probabilities")
+
+    assert probabilities.shape[0] == len(documents)
+
+
+@pytest.mark.parametrize("model", ALL_MODEL_FIXTURES)
+def test_save_and_load_preserves_the_invariants(model, tmp_path, request):
+    """A round-trip through disk changes nothing structural.
+
+    Pickle is used rather than the safetensors default because it is the format that
+    is meant to be lossless; what safetensors deliberately omits is a separate concern.
+    """
+    topic_model = copy.deepcopy(request.getfixturevalue(model))
+    path = tmp_path / "model"
+
+    topic_model.save(str(path), serialization="pickle")
+    loaded = BERTopic.load(str(path))
+
+    assert loaded.topics_ == topic_model.topics_
+    assert loaded.topic_sizes_ == topic_model.topic_sizes_
+    assert loaded._topics.topic_ids() == topic_model._topics.topic_ids()
+    assert loaded.topic_labels_ == topic_model.topic_labels_
+    assert np.array_equal(loaded.c_tf_idf_.toarray(), topic_model.c_tf_idf_.toarray())
+
+
+# The models that `test_representation/test_representations.py::test_topic_reduction`
+# reduces down to 10 topics
+REDUCIBLE_MODEL_FIXTURES = [
+    "base_topic_model",
+    "kmeans_pca_topic_model",
+    "custom_topic_model",
+    "merged_topic_model",
+    "reduced_topic_model",
+    "online_topic_model",
+]
+
+
+@pytest.mark.parametrize("model", REDUCIBLE_MODEL_FIXTURES)
+def test_reduction_fixtures_have_more_topics_than_they_are_reduced_to(model, request):
+    """Fixtures must hold more topics than the reduction tests reduce them to.
+
+    `test_topic_reduction` reduces to 10 and then asserts the assignments changed. If a
+    fixture already holds 10 topics or fewer, `reduce_topics` takes its no-op branch and
+    that test fails for a reason unrelated to the code under test. Asserting it here
+    reports the actual topic count, which turns a confusing downstream failure into a
+    direct statement about the fixture.
+    """
+    topic_model = request.getfixturevalue(model)
+    nr_topics = len(topic_model.topic_sizes_)
+
+    assert nr_topics > 10, (
+        f"{model} produced only {nr_topics} topics; the reduction tests reduce to 10 "
+        "and need more than that to exercise a real reduction"
+    )