diff --git a/integrations/valkey/src/haystack_integrations/document_stores/valkey/document_store.py b/integrations/valkey/src/haystack_integrations/document_stores/valkey/document_store.py index 7299dec251..5a3daff07d 100644 --- a/integrations/valkey/src/haystack_integrations/document_stores/valkey/document_store.py +++ b/integrations/valkey/src/haystack_integrations/document_stores/valkey/document_store.py @@ -1039,10 +1039,8 @@ def get_metadata_field_unique_values( :param size: Number of values to return (default 10). :param filters: Optional filters to restrict the documents considered. :return: Tuple of (list of unique values for the requested page, total count of unique values). - :raises ValueError: If the field is not configured for filtering. :raises ValkeyDocumentStoreError: If the operation fails. """ - self._validate_metadata_field_names([metadata_field]) try: docs = self.filter_documents(filters=filters) return ValkeyDocumentStore._get_metadata_field_unique_values_impl( @@ -1072,10 +1070,8 @@ async def get_metadata_field_unique_values_async( :param size: Number of values to return (default 10). :param filters: Optional filters to restrict the documents considered. :return: Tuple of (list of unique values for the requested page, total count of unique values). - :raises ValueError: If the field is not configured for filtering. :raises ValkeyDocumentStoreError: If the operation fails. """ - self._validate_metadata_field_names([metadata_field]) try: docs = await self.filter_documents_async(filters=filters) return ValkeyDocumentStore._get_metadata_field_unique_values_impl( @@ -1329,8 +1325,10 @@ def _prepare_document_dict(self, doc: Document) -> dict[str, Any]: if isinstance(value, bool): value = int(value) elif not isinstance(value, (int, float)): - msg = f"Field '{field_name}' expects numeric value but got {type(value).__name__}" - raise ValueError(msg) + # Not indexable as numeric; omit from the index. The original value is still + # preserved in `payload` above, so type-preserving reads (e.g. + # get_metadata_field_unique_values) remain correct. + value = None doc_dict[field_name_with_prefix] = value @@ -1491,17 +1489,20 @@ def _get_metadata_field_unique_values_impl( ) -> tuple[list[Any], int]: """Extract unique values for a metadata field with optional search and pagination.""" unique_vals: list[Any] = [] - seen: set[str] = set() + # Key on (type, str value): values that share a string form (e.g. int 1 and str "1") + # must not collapse into a single entry. + seen: set[tuple[type, str]] = set() for doc in documents: val = (doc.meta or {}).get(ValkeyDocumentStore._metadata_field_to_doc_meta_key(metadata_field)) if val is None: continue str_val = str(val) - if str_val in seen: + dedup_key = (type(val), str_val) + if dedup_key in seen: continue if search_term is not None and search_term.lower() not in str_val.lower(): continue - seen.add(str_val) + seen.add(dedup_key) unique_vals.append(val) unique_vals.sort(key=str) total = len(unique_vals) diff --git a/integrations/valkey/tests/test_document_store.py b/integrations/valkey/tests/test_document_store.py index 66b33fa610..836218b56e 100644 --- a/integrations/valkey/tests/test_document_store.py +++ b/integrations/valkey/tests/test_document_store.py @@ -738,82 +738,6 @@ def test_get_metadata_field_min_max_unknown_field_raises(self, document_store): with pytest.raises(ValueError, match="not configured for filtering"): document_store.get_metadata_field_min_max("unknown_field") - def test_get_metadata_field_unique_values(self, document_store): - """Test get_metadata_field_unique_values returns distinct values and total count.""" - docs = [ - Document(id="gmv1", content="doc 1", embedding=[0.1, 0.2, 0.3], meta={"category": "apple", "priority": 1}), - Document(id="gmv2", content="doc 2", embedding=[0.2, 0.3, 0.4], meta={"category": "banana", "priority": 2}), - Document(id="gmv3", content="doc 3", embedding=[0.3, 0.4, 0.5], meta={"category": "apple", "priority": 3}), - ] - document_store.write_documents(docs) - values, total = document_store.get_metadata_field_unique_values("category", from_=0, size=10) - assert total == 2 - assert set(values) == {"apple", "banana"} - assert len(values) == 2 - - def test_get_metadata_field_unique_values_pagination(self, document_store): - """Test get_metadata_field_unique_values with from_ and size.""" - docs = [ - Document(id=f"gmvp{i}", content=f"doc {i}", embedding=[0.1, 0.2, 0.3], meta={"category": f"cat_{i}"}) - for i in range(5) - ] - document_store.write_documents(docs) - values, total = document_store.get_metadata_field_unique_values("category", from_=1, size=2) - assert total == 5 - assert len(values) == 2 - assert sorted(values)[0] >= "cat_0" - - def test_get_metadata_field_unique_values_with_search_term(self, document_store): - """Test get_metadata_field_unique_values with search_term filter.""" - docs = [ - Document(id="gmvs1", content="doc 1", embedding=[0.1, 0.2, 0.3], meta={"category": "apple_pie"}), - Document(id="gmvs2", content="doc 2", embedding=[0.2, 0.3, 0.4], meta={"category": "banana"}), - Document(id="gmvs3", content="doc 3", embedding=[0.3, 0.4, 0.5], meta={"category": "apple_jam"}), - ] - document_store.write_documents(docs) - values, total = document_store.get_metadata_field_unique_values( - "category", search_term="apple", from_=0, size=10 - ) - assert total == 2 - assert set(values) == {"apple_pie", "apple_jam"} - - def test_get_metadata_field_unique_values_with_filters(self, document_store): - """Test get_metadata_field_unique_values restricts documents using the filters param.""" - docs = [ - Document( - id="gmvf1", content="doc 1", embedding=[0.1, 0.2, 0.3], meta={"category": "A", "status": "active"} - ), - Document( - id="gmvf2", content="doc 2", embedding=[0.2, 0.3, 0.4], meta={"category": "B", "status": "active"} - ), - Document( - id="gmvf3", content="doc 3", embedding=[0.3, 0.4, 0.5], meta={"category": "C", "status": "inactive"} - ), - ] - document_store.write_documents(docs) - - filters = {"field": "meta.status", "operator": "==", "value": "active"} - values, total = document_store.get_metadata_field_unique_values("category", filters=filters) - assert set(values) == {"A", "B"} - assert total == 2 - - def test_get_metadata_field_unique_values_preserves_non_string_types(self, document_store): - """Non-string metadata values (e.g. ints) are returned in their original type, not stringified.""" - docs = [ - Document(id="gmvt1", content="doc 1", embedding=[0.1, 0.2, 0.3], meta={"priority": 1}), - Document(id="gmvt2", content="doc 2", embedding=[0.2, 0.3, 0.4], meta={"priority": 2}), - Document(id="gmvt3", content="doc 3", embedding=[0.3, 0.4, 0.5], meta={"priority": 1}), - ] - document_store.write_documents(docs) - values, total = document_store.get_metadata_field_unique_values("priority") - assert total == 2 - assert set(values) == {1, 2} - - def test_get_metadata_field_unique_values_unknown_field_raises(self, document_store): - """Test get_metadata_field_unique_values raises for unconfigured field.""" - with pytest.raises(ValueError, match="not configured for filtering"): - document_store.get_metadata_field_unique_values("unknown_field") - def test_count_unique_metadata_by_filter_invalid_field_raises(self, document_store): """Test count_unique_metadata_by_filter raises for unconfigured field.""" document_store.write_documents( @@ -1256,8 +1180,8 @@ def test_prepare_document_dict_validates_tag_field_type(): store._prepare_document_dict(doc) -def test_prepare_document_dict_validates_numeric_field_type(): - """Test that numeric fields reject non-numeric values.""" +def test_prepare_document_dict_omits_non_numeric_value_from_index(): + """Test that a non-numeric value for a numeric field is omitted from the index but kept in the payload.""" store = ValkeyDocumentStore( index_name="test_validation", embedding_dim=3, @@ -1265,8 +1189,10 @@ def test_prepare_document_dict_validates_numeric_field_type(): ) doc = Document(content="test", embedding=[0.1, 0.2, 0.3], meta={"priority": "high"}) - with pytest.raises(ValueError, match="Field 'priority' expects numeric value but got str"): - store._prepare_document_dict(doc) + doc_dict = store._prepare_document_dict(doc) + + assert doc_dict["meta_priority"] is None + assert doc_dict["payload"]["meta"]["priority"] == "high" @pytest.fixture diff --git a/integrations/valkey/tests/test_document_store_async.py b/integrations/valkey/tests/test_document_store_async.py index e2c1f136c3..c264fe0931 100644 --- a/integrations/valkey/tests/test_document_store_async.py +++ b/integrations/valkey/tests/test_document_store_async.py @@ -565,105 +565,3 @@ async def test_get_metadata_field_min_max_empty_store_async(self, document_store result = await document_store.get_metadata_field_min_max_async("priority") assert result["min"] is None assert result["max"] is None - - async def test_get_metadata_field_unique_values_async(self, document_store): - """Test async get_metadata_field_unique_values returns distinct values and total count.""" - test_id = str(uuid.uuid4())[:8] - docs = [ - Document( - id=f"gmv1_{test_id}", - content="doc 1", - embedding=[0.1, 0.2, 0.3], - meta={"category": "apple", "priority": 1}, - ), - Document( - id=f"gmv2_{test_id}", - content="doc 2", - embedding=[0.2, 0.3, 0.4], - meta={"category": "banana", "priority": 2}, - ), - Document( - id=f"gmv3_{test_id}", - content="doc 3", - embedding=[0.3, 0.4, 0.5], - meta={"category": "apple", "priority": 3}, - ), - ] - await document_store.write_documents_async(docs) - values, total = await document_store.get_metadata_field_unique_values_async("category", from_=0, size=10) - assert total == 2 - assert set(values) == {"apple", "banana"} - assert len(values) == 2 - - async def test_get_metadata_field_unique_values_with_search_term_async(self, document_store): - """Test async get_metadata_field_unique_values with search_term filter.""" - test_id = str(uuid.uuid4())[:8] - docs = [ - Document( - id=f"gmvs1_{test_id}", - content="doc 1", - embedding=[0.1, 0.2, 0.3], - meta={"category": "apple_pie"}, - ), - Document( - id=f"gmvs2_{test_id}", - content="doc 2", - embedding=[0.2, 0.3, 0.4], - meta={"category": "banana"}, - ), - Document( - id=f"gmvs3_{test_id}", - content="doc 3", - embedding=[0.3, 0.4, 0.5], - meta={"category": "apple_jam"}, - ), - ] - await document_store.write_documents_async(docs) - values, total = await document_store.get_metadata_field_unique_values_async( - "category", search_term="apple", from_=0, size=10 - ) - assert total == 2 - assert set(values) == {"apple_pie", "apple_jam"} - - async def test_get_metadata_field_unique_values_with_filters_async(self, document_store): - """Test async get_metadata_field_unique_values restricts documents using the filters param.""" - test_id = str(uuid.uuid4())[:8] - docs = [ - Document( - id=f"gmvf1_{test_id}", - content="doc 1", - embedding=[0.1, 0.2, 0.3], - meta={"category": "A", "status": "active"}, - ), - Document( - id=f"gmvf2_{test_id}", - content="doc 2", - embedding=[0.2, 0.3, 0.4], - meta={"category": "B", "status": "active"}, - ), - Document( - id=f"gmvf3_{test_id}", - content="doc 3", - embedding=[0.3, 0.4, 0.5], - meta={"category": "C", "status": "inactive"}, - ), - ] - await document_store.write_documents_async(docs) - - filters = {"field": "meta.status", "operator": "==", "value": "active"} - values, total = await document_store.get_metadata_field_unique_values_async("category", filters=filters) - assert set(values) == {"A", "B"} - assert total == 2 - - async def test_get_metadata_field_unique_values_async_preserves_non_string_types(self, document_store): - """Non-string metadata values (e.g. ints) are returned in their original type, not stringified.""" - test_id = str(uuid.uuid4())[:8] - docs = [ - Document(id=f"gmvt1_{test_id}", content="doc 1", embedding=[0.1, 0.2, 0.3], meta={"priority": 1}), - Document(id=f"gmvt2_{test_id}", content="doc 2", embedding=[0.2, 0.3, 0.4], meta={"priority": 2}), - Document(id=f"gmvt3_{test_id}", content="doc 3", embedding=[0.3, 0.4, 0.5], meta={"priority": 1}), - ] - await document_store.write_documents_async(docs) - values, total = await document_store.get_metadata_field_unique_values_async("priority") - assert total == 2 - assert set(values) == {1, 2}