Skip to content

GH-51626: [C++][Parquet] Fix statistics of dictionary-encoded leaves under null parents #51627

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kita-renji wants to merge 1 commit into
base: main
Choose a base branch
from
+42 −6
Open

GH-51626: [C++][Parquet] Fix statistics of dictionary-encoded leaves under null parents #51627

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 33 additions & 1 deletion cpp/src/parquet/arrow/arrow_statistics_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include "arrow/array/builder_time.h"
#include "arrow/compute/api.h"
#include "arrow/table.h"
#include "arrow/testing/builder.h"
#include "arrow/testing/gtest_util.h"

#include "parquet/api/reader.h"
Expand Down Expand Up @@ -74,6 +75,21 @@ std::string GetManyEmptyLists() {
return many_empty_lists;
}

// A dictionary-encoded leaf under a struct with null rows. The child slots under the
// null rows are valid in the Arrow array (and hold "zzz"), but they are nulls in
// Parquet: they count as nulls and must not end up in min/max.
std::shared_ptr<::arrow::Table> GetDictionaryUnderNullStruct() {
auto dict_type = ::arrow::dictionary(::arrow::int32(), ::arrow::utf8());
auto values = ::arrow::DictArrayFromJSON(dict_type, "[0, 1, 0, 1]", R"(["b", "zzz"])");
std::shared_ptr<Buffer> null_bitmap;
ABORT_NOT_OK(
::arrow::GetBitmapFromVector<bool>({true, false, true, false}, &null_bitmap));
auto array = ::arrow::StructArray::Make({values}, {::arrow::field("x", dict_type)},
std::move(null_bitmap))
.ValueOrDie();
return Table::Make(::arrow::schema({::arrow::field("a", array->type())}), {array});
}

// PARQUET-2067: Tests that nulls from parent fields are included in null statistics.
TEST_P(ParameterizedStatisticsTest, NoNullCountWrittenForRepeatedFields) {
std::shared_ptr<::arrow::ResizableBuffer> serialized_data = AllocateBuffer();
Expand Down Expand Up @@ -159,7 +175,23 @@ INSTANTIATE_TEST_SUITE_P(
/*expected_null_count=*/5,
/*expected_value_count=*/2,
/*expected_min=*/"z",
/*expected_max=*/"z"}));
/*expected_max=*/"z"},
StatisticsTestParam{/*table=*/GetDictionaryUnderNullStruct(),
/*expected_null_count=*/2,
/*expected_value_count=*/2,
/*expected_min=*/"b",
/*expected_max=*/"b"},
StatisticsTestParam{
// "zzz" is in the dictionary but no index references it
/*table=*/Table::Make(
::arrow::schema({::arrow::field("a", dictionary(::arrow::int32(),
::arrow::utf8()))}),
{::arrow::DictArrayFromJSON(dictionary(::arrow::int32(), ::arrow::utf8()),
"[0, null, 0]", R"(["b", "zzz"])")}),
/*expected_null_count=*/1,
/*expected_value_count=*/2,
/*expected_min=*/"b",
/*expected_max=*/"b"}));

TEST(StatisticsTest, FixedWidthLeafUnderListStructNullCount) {
// Null counts for leaves under list<struct<...>> must include null and empty
Expand Down
14 changes: 9 additions & 5 deletions cpp/src/parquet/column_writer.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2002,8 +2002,10 @@ Status TypedColumnWriterImpl<ParquetType>::WriteArrowDictionary(
PARQUET_ASSIGN_OR_THROW(::arrow::Datum referenced_indices,
::arrow::compute::Unique(*chunk_indices, &exec_ctx));

// On first run, we might be able to re-use the existing dictionary
if (referenced_indices.length() == dictionary->length()) {
// On first run, we might be able to re-use the existing dictionary.
// Null indices show up as one null in referenced_indices: don't count it.
if (referenced_indices.length() - referenced_indices.null_count() ==
dictionary->length()) {
referenced_dictionary = dictionary;
} else {
PARQUET_ASSIGN_OR_THROW(
Expand Down Expand Up @@ -2043,12 +2045,14 @@ Status TypedColumnWriterImpl<ParquetType>::WriteArrowDictionary(
AddIfNotNull(rep_levels, offset));
std::shared_ptr<Array> writeable_indices =
indices->Slice(value_offset, batch_num_spaced_values);
if (page_statistics_ || bloom_filter_writer_) {
update_stats(/*num_chunk_levels=*/batch_size, writeable_indices);
}
// Statistics need the recomputed validity too: an index under a null parent may
// be valid in the leaf array.
PARQUET_ASSIGN_OR_THROW(
writeable_indices,
MaybeReplaceValidity(writeable_indices, null_count, ctx->memory_pool));
if (page_statistics_ || bloom_filter_writer_) {
update_stats(/*num_chunk_levels=*/batch_size, writeable_indices);
}
dict_encoder->PutIndices(*writeable_indices);
// Update unencoded byte array data size to size statistics
UpdateUnencodedDataBytes();
Expand Down
Loading