diff --git a/integrations/opensearch/tests/test_document_store.py b/integrations/opensearch/tests/test_document_store.py index 93817e29b2..b074d830e4 100644 --- a/integrations/opensearch/tests/test_document_store.py +++ b/integrations/opensearch/tests/test_document_store.py @@ -1070,156 +1070,24 @@ def test_delete_all_documents_index_recreation(self, document_store: OpenSearchD assert len(results) == 1 assert results[0].content == "New document after delete all" - def test_get_metadata_field_unique_values(self, document_store: OpenSearchDocumentStore): - # Test with string values + def test_get_metadata_field_unique_values_pagination_beyond_total(self, document_store: OpenSearchDocumentStore): + """ + Edge case not covered by the shared GetMetadataFieldUniqueValuesTest mixin's pagination test: + requesting a page starting beyond the total count must return an empty page with the correct total. + """ docs = [ - Document(content="Python programming", meta={"category": "A", "language": "Python"}), - Document(content="Java programming", meta={"category": "B", "language": "Java"}), - Document(content="Python scripting", meta={"category": "A", "language": "Python"}), - Document(content="JavaScript development", meta={"category": "C", "language": "JavaScript"}), - Document(content="Python data science", meta={"category": "A", "language": "Python"}), - Document(content="Java backend", meta={"category": "B", "language": "Java"}), + Document(content="Doc 1", meta={"category": "A"}), + Document(content="Doc 2", meta={"category": "B"}), + Document(content="Doc 3", meta={"category": "C"}), ] document_store.write_documents(docs) - # Test getting all unique values without search term - unique_values, total_count = document_store.get_metadata_field_unique_values( - metadata_field="category", search_term=None, from_=0, size=10 - ) - assert set(unique_values) == {"A", "B", "C"} - assert total_count == 3 - - # Test field name normalization - the "meta." prefix is optional and must give identical results - unique_values_prefixed, total_count_prefixed = document_store.get_metadata_field_unique_values( - metadata_field="meta.category", search_term=None, from_=0, size=10 - ) - assert set(unique_values_prefixed) == set(unique_values) - assert total_count_prefixed == total_count - - unique_languages, total_languages = document_store.get_metadata_field_unique_values( - metadata_field="meta.language", search_term=None, from_=0, size=10 - ) - assert set(unique_languages) == {"Python", "Java", "JavaScript"} - assert total_languages == 3 - - # Test pagination - first page - unique_values_page1, total_count_page1 = document_store.get_metadata_field_unique_values( - metadata_field="meta.category", search_term=None, from_=0, size=2 - ) - assert len(unique_values_page1) == 2 - assert all(val in ["A", "B", "C"] for val in unique_values_page1) - assert total_count_page1 == 3 - - # Test pagination - second page, via from_ (triggers the offset-walk internally) - unique_values_page2, total_count_page2 = document_store.get_metadata_field_unique_values( - metadata_field="meta.category", search_term=None, from_=2, size=2 - ) - assert len(unique_values_page2) == 1 - assert unique_values_page2[0] in ["A", "B", "C"] - assert total_count_page2 == 3 - - # Pages don't overlap and together cover all values - assert not set(unique_values_page1).intersection(set(unique_values_page2)) - assert set(unique_values_page1) | set(unique_values_page2) == {"A", "B", "C"} - - # Test pagination - from_ beyond total count (should return empty, but a valid total_count) unique_values_beyond, total_beyond = document_store.get_metadata_field_unique_values( metadata_field="meta.category", search_term=None, from_=10, size=10 ) assert len(unique_values_beyond) == 0 assert total_beyond == 3 - # Test with search term - filter by the metadata field's own VALUE matching "Python" - # ("language" values are "Python"/"Java"/"JavaScript", so searching "Python" against - # the "category" field's values ("A"/"B"/"C") should match nothing) - unique_values_filtered, total_filtered = document_store.get_metadata_field_unique_values( - metadata_field="meta.category", search_term="Python", from_=0, size=10 - ) - assert set(unique_values_filtered) == set() - assert total_filtered == 0 - - # Searching "language" values themselves for the substring "Java" must match both - # "Java" and "JavaScript" (substring match on the field's own value). - unique_languages_filtered, total_java = document_store.get_metadata_field_unique_values( - metadata_field="meta.language", search_term="Java", from_=0, size=10 - ) - assert set(unique_languages_filtered) == {"Java", "JavaScript"} - assert total_java == 2 - - # Case-insensitivity: a lowercase search term must still match the differently-cased values above. - unique_languages_lower, total_lower = document_store.get_metadata_field_unique_values( - metadata_field="meta.language", search_term="java", from_=0, size=10 - ) - assert set(unique_languages_lower) == {"Java", "JavaScript"} - assert total_lower == 2 - - # Test with integer values - int_docs = [ - Document(content="Doc 1", meta={"priority": 1}), - Document(content="Doc 2", meta={"priority": 2}), - Document(content="Doc 3", meta={"priority": 1}), - Document(content="Doc 4", meta={"priority": 3}), - ] - document_store.write_documents(int_docs) - unique_priorities, total_priorities = document_store.get_metadata_field_unique_values( - metadata_field="meta.priority", search_term=None, from_=0, size=10 - ) - assert set(unique_priorities) == {1, 2, 3} - assert total_priorities == 3 - - # search_term now matches against the field's own value, not the content, so searching - # for content text ("Doc 1") against the "priority" field's values ("1"/"2"/"3") matches nothing. - unique_priorities_filtered, total_priorities_filtered = document_store.get_metadata_field_unique_values( - metadata_field="meta.priority", search_term="Doc 1", from_=0, size=10 - ) - assert set(unique_priorities_filtered) == set() - assert total_priorities_filtered == 0 - - # search_term matching the field's own (stringified) value (e.g. "1") does match, but the - # returned value itself keeps its original type (int here). - unique_priorities_by_value, total_priorities_by_value = document_store.get_metadata_field_unique_values( - metadata_field="meta.priority", search_term="1", from_=0, size=10 - ) - assert set(unique_priorities_by_value) == {1} - assert total_priorities_by_value == 1 - - # Prove the semantic change explicitly with a document whose CONTENT contains the search - # term but whose target metadata field value does NOT: it must now be EXCLUDED. - content_match_docs = [ - Document(content="This mentions needle in the text", meta={"topic": "unrelated"}), - ] - document_store.write_documents(content_match_docs) - unique_topics_content_only, total_topics_content_only = document_store.get_metadata_field_unique_values( - metadata_field="meta.topic", search_term="needle", from_=0, size=10 - ) - assert set(unique_topics_content_only) == set() - assert total_topics_content_only == 0 - - # And a document whose metadata field VALUE contains the search term but whose content does - # NOT: it must now be INCLUDED. - value_match_docs = [ - Document(content="Nothing special here", meta={"topic": "needle-in-haystack"}), - ] - document_store.write_documents(value_match_docs) - unique_topics_value_only, total_topics_value_only = document_store.get_metadata_field_unique_values( - metadata_field="meta.topic", search_term="needle", from_=0, size=10 - ) - assert set(unique_topics_value_only) == {"needle-in-haystack"} - assert total_topics_value_only == 1 - - def test_get_metadata_field_unique_values_with_filters(self, document_store: OpenSearchDocumentStore): - docs = [ - Document(content="Doc 1", meta={"category": "A", "status": "active"}), - Document(content="Doc 2", meta={"category": "B", "status": "active"}), - Document(content="Doc 3", meta={"category": "C", "status": "inactive"}), - ] - document_store.write_documents(docs) - - filters = {"field": "meta.status", "operator": "==", "value": "active"} - values, total = document_store.get_metadata_field_unique_values("meta.category", filters=filters) - assert set(values) == {"A", "B"} - assert total == 2 - def test_write_with_routing(self, document_store: OpenSearchDocumentStore): """Test writing documents with routing metadata""" docs = [ diff --git a/integrations/opensearch/tests/test_document_store_async.py b/integrations/opensearch/tests/test_document_store_async.py index 3c72e9f9c8..6e0269578f 100644 --- a/integrations/opensearch/tests/test_document_store_async.py +++ b/integrations/opensearch/tests/test_document_store_async.py @@ -913,156 +913,22 @@ async def test_get_metadata_fields_info_async_with_alias(self, document_store: O await alias_store._async_client.close() @pytest.mark.asyncio - async def test_get_metadata_field_unique_values_async(self, document_store: OpenSearchDocumentStore): - # Test with string values + async def test_get_metadata_field_unique_values_pagination_beyond_total_async( + self, document_store: OpenSearchDocumentStore + ): + """ + Edge case not covered by the shared GetMetadataFieldUniqueValuesAsyncTest mixin's pagination test: + requesting a page starting beyond the total count must return an empty page with the correct total. + """ docs = [ - Document(content="Python programming", meta={"category": "A", "language": "Python"}), - Document(content="Java programming", meta={"category": "B", "language": "Java"}), - Document(content="Python scripting", meta={"category": "A", "language": "Python"}), - Document(content="JavaScript development", meta={"category": "C", "language": "JavaScript"}), - Document(content="Python data science", meta={"category": "A", "language": "Python"}), - Document(content="Java backend", meta={"category": "B", "language": "Java"}), + Document(content="Doc 1", meta={"category": "A"}), + Document(content="Doc 2", meta={"category": "B"}), + Document(content="Doc 3", meta={"category": "C"}), ] await document_store.write_documents_async(docs) - # Test getting all unique values without search term - unique_values, total_count = await document_store.get_metadata_field_unique_values_async( - metadata_field="category", search_term=None, from_=0, size=10 - ) - assert set(unique_values) == {"A", "B", "C"} - assert total_count == 3 - - # Test field name normalization - the "meta." prefix is optional and must give identical results - unique_values_prefixed, total_count_prefixed = await document_store.get_metadata_field_unique_values_async( - metadata_field="meta.category", search_term=None, from_=0, size=10 - ) - assert set(unique_values_prefixed) == set(unique_values) - assert total_count_prefixed == total_count - - unique_languages, total_languages = await document_store.get_metadata_field_unique_values_async( - metadata_field="meta.language", search_term=None, from_=0, size=10 - ) - assert set(unique_languages) == {"Python", "Java", "JavaScript"} - assert total_languages == 3 - - # Test pagination - first page - unique_values_page1, total_count_page1 = await document_store.get_metadata_field_unique_values_async( - metadata_field="meta.category", search_term=None, from_=0, size=2 - ) - assert len(unique_values_page1) == 2 - assert all(val in ["A", "B", "C"] for val in unique_values_page1) - assert total_count_page1 == 3 - - # Test pagination - second page, via from_ (triggers the offset-walk internally) - unique_values_page2, total_count_page2 = await document_store.get_metadata_field_unique_values_async( - metadata_field="meta.category", search_term=None, from_=2, size=2 - ) - assert len(unique_values_page2) == 1 - assert unique_values_page2[0] in ["A", "B", "C"] - assert total_count_page2 == 3 - - # Pages don't overlap and together cover all values - assert not set(unique_values_page1).intersection(set(unique_values_page2)) - assert set(unique_values_page1) | set(unique_values_page2) == {"A", "B", "C"} - - # Test pagination - from_ beyond total count (should return empty, but a valid total_count) unique_values_beyond, total_beyond = await document_store.get_metadata_field_unique_values_async( metadata_field="meta.category", search_term=None, from_=10, size=10 ) assert len(unique_values_beyond) == 0 assert total_beyond == 3 - - # Test with search term - filter by the metadata field's own VALUE matching "Python" - # ("language" values are "Python"/"Java"/"JavaScript", so searching "Python" against - # the "category" field's values ("A"/"B"/"C") should match nothing) - unique_values_filtered, total_filtered = await document_store.get_metadata_field_unique_values_async( - metadata_field="meta.category", search_term="Python", from_=0, size=10 - ) - assert set(unique_values_filtered) == set() - assert total_filtered == 0 - - # Searching "language" values themselves for the substring "Java" must match both - # "Java" and "JavaScript" (substring match on the field's own value). - unique_languages_filtered, total_java = await document_store.get_metadata_field_unique_values_async( - metadata_field="meta.language", search_term="Java", from_=0, size=10 - ) - assert set(unique_languages_filtered) == {"Java", "JavaScript"} - assert total_java == 2 - - # Case-insensitivity: a lowercase search term must still match the differently-cased values above. - unique_languages_lower, total_lower = await document_store.get_metadata_field_unique_values_async( - metadata_field="meta.language", search_term="java", from_=0, size=10 - ) - assert set(unique_languages_lower) == {"Java", "JavaScript"} - assert total_lower == 2 - - # Test with integer values - int_docs = [ - Document(content="Doc 1", meta={"priority": 1}), - Document(content="Doc 2", meta={"priority": 2}), - Document(content="Doc 3", meta={"priority": 1}), - Document(content="Doc 4", meta={"priority": 3}), - ] - await document_store.write_documents_async(int_docs) - unique_priorities, total_priorities = await document_store.get_metadata_field_unique_values_async( - metadata_field="meta.priority", search_term=None, from_=0, size=10 - ) - assert set(unique_priorities) == {1, 2, 3} - assert total_priorities == 3 - - # search_term now matches against the field's own value, not the content, so searching - # for content text ("Doc 1") against the "priority" field's values ("1"/"2"/"3") matches nothing. - priorities_filtered = await document_store.get_metadata_field_unique_values_async( - metadata_field="meta.priority", search_term="Doc 1", from_=0, size=10 - ) - unique_priorities_filtered, total_priorities_filtered = priorities_filtered - assert set(unique_priorities_filtered) == set() - assert total_priorities_filtered == 0 - - # search_term matching the field's own (stringified) value (e.g. "1") does match, but the - # returned value itself keeps its original type (int here). - priorities_by_value = await document_store.get_metadata_field_unique_values_async( - metadata_field="meta.priority", search_term="1", from_=0, size=10 - ) - unique_priorities_by_value, total_priorities_by_value = priorities_by_value - assert set(unique_priorities_by_value) == {1} - assert total_priorities_by_value == 1 - - # Prove the semantic change explicitly with a document whose CONTENT contains the search - # term but whose target metadata field value does NOT: it must now be EXCLUDED. - content_match_docs = [ - Document(content="This mentions needle in the text", meta={"topic": "unrelated"}), - ] - await document_store.write_documents_async(content_match_docs) - topics_content_only = await document_store.get_metadata_field_unique_values_async( - metadata_field="meta.topic", search_term="needle", from_=0, size=10 - ) - unique_topics_content_only, total_topics_content_only = topics_content_only - assert set(unique_topics_content_only) == set() - assert total_topics_content_only == 0 - - # And a document whose metadata field VALUE contains the search term but whose content does - # NOT: it must now be INCLUDED. - value_match_docs = [ - Document(content="Nothing special here", meta={"topic": "needle-in-haystack"}), - ] - await document_store.write_documents_async(value_match_docs) - unique_topics_value_only, total_topics_value_only = await document_store.get_metadata_field_unique_values_async( - metadata_field="meta.topic", search_term="needle", from_=0, size=10 - ) - assert set(unique_topics_value_only) == {"needle-in-haystack"} - assert total_topics_value_only == 1 - - @pytest.mark.asyncio - async def test_get_metadata_field_unique_values_with_filters_async(self, document_store: OpenSearchDocumentStore): - docs = [ - Document(content="Doc 1", meta={"category": "A", "status": "active"}), - Document(content="Doc 2", meta={"category": "B", "status": "active"}), - Document(content="Doc 3", meta={"category": "C", "status": "inactive"}), - ] - await document_store.write_documents_async(docs) - - filters = {"field": "meta.status", "operator": "==", "value": "active"} - values, total = await document_store.get_metadata_field_unique_values_async("meta.category", filters=filters) - assert set(values) == {"A", "B"} - assert total == 2