Skip to content

test: ValkeyDocumentStore removing duplicated tests already covered by haystack Mixin tests + allow non-schema-typed metadata values in Valkey get_metadata_field_unique_value #3786

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
davidsbatista wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

test: ValkeyDocumentStore removing duplicated tests already covered by haystack Mixin tests + allow non-schema-typed metadata values in Valkey get_metadata_field_unique_value #3786

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e49bd9f
removing duplicated tests - already in Mixin
davidsbatista Aug 14, 2026
9954d94
fix: allow non-schema-typed metadata values in Valkey get_metadata_fi…
davidsbatista Aug 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 10 additions & 9 deletions integrations/valkey/src/haystack_integrations/document_stores/valkey/document_store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1039,10 +1039,8 @@ def get_metadata_field_unique_values(
:param size: Number of values to return (default 10).
:param filters: Optional filters to restrict the documents considered.
:return: Tuple of (list of unique values for the requested page, total count of unique values).
:raises ValueError: If the field is not configured for filtering.
:raises ValkeyDocumentStoreError: If the operation fails.
"""
self._validate_metadata_field_names([metadata_field])
try:
docs = self.filter_documents(filters=filters)
return ValkeyDocumentStore._get_metadata_field_unique_values_impl(
Expand Down Expand Up @@ -1072,10 +1070,8 @@ async def get_metadata_field_unique_values_async(
:param size: Number of values to return (default 10).
:param filters: Optional filters to restrict the documents considered.
:return: Tuple of (list of unique values for the requested page, total count of unique values).
:raises ValueError: If the field is not configured for filtering.
:raises ValkeyDocumentStoreError: If the operation fails.
"""
self._validate_metadata_field_names([metadata_field])
try:
docs = await self.filter_documents_async(filters=filters)
return ValkeyDocumentStore._get_metadata_field_unique_values_impl(
Expand Down Expand Up @@ -1329,8 +1325,10 @@ def _prepare_document_dict(self, doc: Document) -> dict[str, Any]:
if isinstance(value, bool):
value = int(value)
elif not isinstance(value, (int, float)):
msg = f"Field '{field_name}' expects numeric value but got {type(value).__name__}"
raise ValueError(msg)
# Not indexable as numeric; omit from the index. The original value is still
# preserved in `payload` above, so type-preserving reads (e.g.
# get_metadata_field_unique_values) remain correct.
value = None

doc_dict[field_name_with_prefix] = value

Expand Down Expand Up @@ -1491,17 +1489,20 @@ def _get_metadata_field_unique_values_impl(
) -> tuple[list[Any], int]:
"""Extract unique values for a metadata field with optional search and pagination."""
unique_vals: list[Any] = []
seen: set[str] = set()
# Key on (type, str value): values that share a string form (e.g. int 1 and str "1")
# must not collapse into a single entry.
seen: set[tuple[type, str]] = set()
for doc in documents:
val = (doc.meta or {}).get(ValkeyDocumentStore._metadata_field_to_doc_meta_key(metadata_field))
if val is None:
continue
str_val = str(val)
if str_val in seen:
dedup_key = (type(val), str_val)
if dedup_key in seen:
continue
if search_term is not None and search_term.lower() not in str_val.lower():
continue
seen.add(str_val)
seen.add(dedup_key)
unique_vals.append(val)
unique_vals.sort(key=str)
total = len(unique_vals)
Expand Down
86 changes: 6 additions & 80 deletions integrations/valkey/tests/test_document_store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -738,82 +738,6 @@ def test_get_metadata_field_min_max_unknown_field_raises(self, document_store):
with pytest.raises(ValueError, match="not configured for filtering"):
document_store.get_metadata_field_min_max("unknown_field")

def test_get_metadata_field_unique_values(self, document_store):
"""Test get_metadata_field_unique_values returns distinct values and total count."""
docs = [
Document(id="gmv1", content="doc 1", embedding=[0.1, 0.2, 0.3], meta={"category": "apple", "priority": 1}),
Document(id="gmv2", content="doc 2", embedding=[0.2, 0.3, 0.4], meta={"category": "banana", "priority": 2}),
Document(id="gmv3", content="doc 3", embedding=[0.3, 0.4, 0.5], meta={"category": "apple", "priority": 3}),
]
document_store.write_documents(docs)
values, total = document_store.get_metadata_field_unique_values("category", from_=0, size=10)
assert total == 2
assert set(values) == {"apple", "banana"}
assert len(values) == 2

def test_get_metadata_field_unique_values_pagination(self, document_store):
"""Test get_metadata_field_unique_values with from_ and size."""
docs = [
Document(id=f"gmvp{i}", content=f"doc {i}", embedding=[0.1, 0.2, 0.3], meta={"category": f"cat_{i}"})
for i in range(5)
]
document_store.write_documents(docs)
values, total = document_store.get_metadata_field_unique_values("category", from_=1, size=2)
assert total == 5
assert len(values) == 2
assert sorted(values)[0] >= "cat_0"

def test_get_metadata_field_unique_values_with_search_term(self, document_store):
"""Test get_metadata_field_unique_values with search_term filter."""
docs = [
Document(id="gmvs1", content="doc 1", embedding=[0.1, 0.2, 0.3], meta={"category": "apple_pie"}),
Document(id="gmvs2", content="doc 2", embedding=[0.2, 0.3, 0.4], meta={"category": "banana"}),
Document(id="gmvs3", content="doc 3", embedding=[0.3, 0.4, 0.5], meta={"category": "apple_jam"}),
]
document_store.write_documents(docs)
values, total = document_store.get_metadata_field_unique_values(
"category", search_term="apple", from_=0, size=10
)
assert total == 2
assert set(values) == {"apple_pie", "apple_jam"}

def test_get_metadata_field_unique_values_with_filters(self, document_store):
"""Test get_metadata_field_unique_values restricts documents using the filters param."""
docs = [
Document(
id="gmvf1", content="doc 1", embedding=[0.1, 0.2, 0.3], meta={"category": "A", "status": "active"}
),
Document(
id="gmvf2", content="doc 2", embedding=[0.2, 0.3, 0.4], meta={"category": "B", "status": "active"}
),
Document(
id="gmvf3", content="doc 3", embedding=[0.3, 0.4, 0.5], meta={"category": "C", "status": "inactive"}
),
]
document_store.write_documents(docs)

filters = {"field": "meta.status", "operator": "==", "value": "active"}
values, total = document_store.get_metadata_field_unique_values("category", filters=filters)
assert set(values) == {"A", "B"}
assert total == 2

def test_get_metadata_field_unique_values_preserves_non_string_types(self, document_store):
"""Non-string metadata values (e.g. ints) are returned in their original type, not stringified."""
docs = [
Document(id="gmvt1", content="doc 1", embedding=[0.1, 0.2, 0.3], meta={"priority": 1}),
Document(id="gmvt2", content="doc 2", embedding=[0.2, 0.3, 0.4], meta={"priority": 2}),
Document(id="gmvt3", content="doc 3", embedding=[0.3, 0.4, 0.5], meta={"priority": 1}),
]
document_store.write_documents(docs)
values, total = document_store.get_metadata_field_unique_values("priority")
assert total == 2
assert set(values) == {1, 2}

def test_get_metadata_field_unique_values_unknown_field_raises(self, document_store):
"""Test get_metadata_field_unique_values raises for unconfigured field."""
with pytest.raises(ValueError, match="not configured for filtering"):
document_store.get_metadata_field_unique_values("unknown_field")

def test_count_unique_metadata_by_filter_invalid_field_raises(self, document_store):
"""Test count_unique_metadata_by_filter raises for unconfigured field."""
document_store.write_documents(
Expand Down Expand Up @@ -1256,17 +1180,19 @@ def test_prepare_document_dict_validates_tag_field_type():
store._prepare_document_dict(doc)


def test_prepare_document_dict_validates_numeric_field_type():
"""Test that numeric fields reject non-numeric values."""
def test_prepare_document_dict_omits_non_numeric_value_from_index():
"""Test that a non-numeric value for a numeric field is omitted from the index but kept in the payload."""
store = ValkeyDocumentStore(
index_name="test_validation",
embedding_dim=3,
metadata_fields={"priority": int},
)
doc = Document(content="test", embedding=[0.1, 0.2, 0.3], meta={"priority": "high"})

with pytest.raises(ValueError, match="Field 'priority' expects numeric value but got str"):
store._prepare_document_dict(doc)
doc_dict = store._prepare_document_dict(doc)

assert doc_dict["meta_priority"] is None
assert doc_dict["payload"]["meta"]["priority"] == "high"


@pytest.fixture
Expand Down
102 changes: 0 additions & 102 deletions integrations/valkey/tests/test_document_store_async.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -565,105 +565,3 @@ async def test_get_metadata_field_min_max_empty_store_async(self, document_store
result = await document_store.get_metadata_field_min_max_async("priority")
assert result["min"] is None
assert result["max"] is None

async def test_get_metadata_field_unique_values_async(self, document_store):
"""Test async get_metadata_field_unique_values returns distinct values and total count."""
test_id = str(uuid.uuid4())[:8]
docs = [
Document(
id=f"gmv1_{test_id}",
content="doc 1",
embedding=[0.1, 0.2, 0.3],
meta={"category": "apple", "priority": 1},
),
Document(
id=f"gmv2_{test_id}",
content="doc 2",
embedding=[0.2, 0.3, 0.4],
meta={"category": "banana", "priority": 2},
),
Document(
id=f"gmv3_{test_id}",
content="doc 3",
embedding=[0.3, 0.4, 0.5],
meta={"category": "apple", "priority": 3},
),
]
await document_store.write_documents_async(docs)
values, total = await document_store.get_metadata_field_unique_values_async("category", from_=0, size=10)
assert total == 2
assert set(values) == {"apple", "banana"}
assert len(values) == 2

async def test_get_metadata_field_unique_values_with_search_term_async(self, document_store):
"""Test async get_metadata_field_unique_values with search_term filter."""
test_id = str(uuid.uuid4())[:8]
docs = [
Document(
id=f"gmvs1_{test_id}",
content="doc 1",
embedding=[0.1, 0.2, 0.3],
meta={"category": "apple_pie"},
),
Document(
id=f"gmvs2_{test_id}",
content="doc 2",
embedding=[0.2, 0.3, 0.4],
meta={"category": "banana"},
),
Document(
id=f"gmvs3_{test_id}",
content="doc 3",
embedding=[0.3, 0.4, 0.5],
meta={"category": "apple_jam"},
),
]
await document_store.write_documents_async(docs)
values, total = await document_store.get_metadata_field_unique_values_async(
"category", search_term="apple", from_=0, size=10
)
assert total == 2
assert set(values) == {"apple_pie", "apple_jam"}

async def test_get_metadata_field_unique_values_with_filters_async(self, document_store):
"""Test async get_metadata_field_unique_values restricts documents using the filters param."""
test_id = str(uuid.uuid4())[:8]
docs = [
Document(
id=f"gmvf1_{test_id}",
content="doc 1",
embedding=[0.1, 0.2, 0.3],
meta={"category": "A", "status": "active"},
),
Document(
id=f"gmvf2_{test_id}",
content="doc 2",
embedding=[0.2, 0.3, 0.4],
meta={"category": "B", "status": "active"},
),
Document(
id=f"gmvf3_{test_id}",
content="doc 3",
embedding=[0.3, 0.4, 0.5],
meta={"category": "C", "status": "inactive"},
),
]
await document_store.write_documents_async(docs)

filters = {"field": "meta.status", "operator": "==", "value": "active"}
values, total = await document_store.get_metadata_field_unique_values_async("category", filters=filters)
assert set(values) == {"A", "B"}
assert total == 2

async def test_get_metadata_field_unique_values_async_preserves_non_string_types(self, document_store):
"""Non-string metadata values (e.g. ints) are returned in their original type, not stringified."""
test_id = str(uuid.uuid4())[:8]
docs = [
Document(id=f"gmvt1_{test_id}", content="doc 1", embedding=[0.1, 0.2, 0.3], meta={"priority": 1}),
Document(id=f"gmvt2_{test_id}", content="doc 2", embedding=[0.2, 0.3, 0.4], meta={"priority": 2}),
Document(id=f"gmvt3_{test_id}", content="doc 3", embedding=[0.3, 0.4, 0.5], meta={"priority": 1}),
]
await document_store.write_documents_async(docs)
values, total = await document_store.get_metadata_field_unique_values_async("priority")
assert total == 2
assert set(values) == {1, 2}
Loading