From 074619b111a3271bcd2a438a0793f8480c735421 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Tue, 25 Aug 2026 14:00:51 +0100 Subject: [PATCH 1/5] Document updates --- r/NEWS.md | 8 ++++++++ r/R/type.R | 4 +++- r/man/data-type.Rd | 4 +++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/r/NEWS.md b/r/NEWS.md index 37ed6c6de66a..751acea4d32b 100644 --- a/r/NEWS.md +++ b/r/NEWS.md @@ -19,6 +19,14 @@ # arrow 25.0.1.9000 +## Minor improvements and fixes + +- When converting Arrow list-columns to R (e.g. via `as.data.frame()` or + `collect()`), every element now has the same type as the column's `ptype`: + factors share one set of levels and `int64`/`uint32` values are either all + integers or all `integer64`/doubles, so `tidyr::unnest()` works on them + (#50514). + # arrow 25.0.1 ## Minor improvements and fixes diff --git a/r/R/type.R b/r/R/type.R index da60b027defd..26eda942f005 100644 --- a/r/R/type.R +++ b/r/R/type.R @@ -356,7 +356,9 @@ NestedType <- R6Class("NestedType", inherit = DataType) #' to `double` ("numeric") and `int64` is converted to `bit64::integer64`. For #' `int64` types, this conversion can be disabled (so that `int64` always yields #' a `bit64::integer64` object) by setting `options(arrow.int64_downcast = -#' FALSE)`. +#' FALSE)`. For values nested inside a list type, this decision is made once +#' for all values in the column, so every element of the list uses the same R +#' type. #' #' `decimal128()` creates a `Decimal128Type`. Arrow decimals are fixed-point #' decimal numbers encoded as a scalar integer. The `precision` is the number of diff --git a/r/man/data-type.Rd b/r/man/data-type.Rd index 29fb667ed6ba..67bea0d5425b 100644 --- a/r/man/data-type.Rd +++ b/r/man/data-type.Rd @@ -178,7 +178,9 @@ signed integer) types may contain values that exceed the range of R's \code{integer} type (32-bit signed integer). When they do, \code{uint32} is converted to \code{double} ("numeric") and \code{int64} is converted to \code{bit64::integer64}. For \code{int64} types, this conversion can be disabled (so that \code{int64} always yields -a \code{bit64::integer64} object) by setting \code{options(arrow.int64_downcast = FALSE)}. +a \code{bit64::integer64} object) by setting \code{options(arrow.int64_downcast = FALSE)}. For values nested inside a list type, this decision is made once +for all values in the column, so every element of the list uses the same R +type. \code{decimal128()} creates a \code{Decimal128Type}. Arrow decimals are fixed-point decimal numbers encoded as a scalar integer. The \code{precision} is the number of From 53fa85baf3f76c5dfbe1f08b322f1350bdb2b13b Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Tue, 25 Aug 2026 14:01:14 +0100 Subject: [PATCH 2/5] Add missing return() --- r/src/type_infer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/src/type_infer.cpp b/r/src/type_infer.cpp index b492cfde1c46..125b7e9a01c6 100644 --- a/r/src/type_infer.cpp +++ b/r/src/type_infer.cpp @@ -165,7 +165,7 @@ std::shared_ptr InferArrowTypeFromVector(SEXP x) { // Check attr(x, "ptype") for an appropriate R prototype SEXP ptype = Rf_getAttrib(x, symbols::ptype); if (!Rf_isNull(ptype)) { - arrow::list(InferArrowType(ptype)); + return arrow::list(InferArrowType(ptype)); } // If unspecified, iterate through the vector until we get a non-null result From c35269a19963d69b6b881a46bca76f0c4313142c Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Thu, 3 Sep 2026 14:26:55 +0100 Subject: [PATCH 3/5] Convert values based on whole thing --- r/src/array_to_vector.cpp | 167 +++++++++++++++---- r/tests/testthat/test-Array.R | 222 ++++++++++++++++++++++++++ r/tests/testthat/test-chunked-array.R | 15 ++ 3 files changed, 371 insertions(+), 33 deletions(-) diff --git a/r/src/array_to_vector.cpp b/r/src/array_to_vector.cpp index 8b617e6709d6..f5f16433f230 100644 --- a/r/src/array_to_vector.cpp +++ b/r/src/array_to_vector.cpp @@ -48,6 +48,12 @@ class Converter { // Allocate a vector of the right R type for this converter virtual SEXP Allocate(R_xlen_t n) const = 0; + // Allocate a vector of the right R type for a slice of the data this converter + // was built for, i.e. never as an altrep vector shadowing the whole chunked array. + // Only converters whose Allocate() may hand out altrep vectors (e.g. Converter_Struct) + // need to override this. + virtual SEXP AllocateSlice(R_xlen_t n) const { return Allocate(n); } + // data[ start:(start + n) ] = NA virtual Status Ingest_all_nulls(SEXP data, R_xlen_t start, R_xlen_t n) const = 0; @@ -97,6 +103,34 @@ class Converter { return out; } + // Convert a slice of the data this converter was built for (e.g. one element of a + // list array) to a standalone R vector, reusing the decisions this converter made + // over all chunks (factor levels, integer vs double, ...). + // + // chunk_index is the index of the chunk that `slice` was taken from, so that + // per-chunk state (e.g. dictionary transposition) is applied correctly. + virtual SEXP ConvertSlice(const std::shared_ptr& slice, + size_t chunk_index) const { + // Dictionary slices must go through this converter so that they get the + // unified levels; other types can shadow the slice with altrep as before + if (slice->type_id() != Type::DICTIONARY) { + SEXP alt = altrep::MakeAltrepVector(std::make_shared(slice)); + if (!Rf_isNull(alt)) { + return alt; + } + } + + R_xlen_t n = slice->length(); + SEXP out = PROTECT(AllocateSlice(n)); + if (slice->null_count() == n) { + StopIfNotOk(Ingest_all_nulls(out, 0, n)); + } else { + StopIfNotOk(Ingest_some_nulls(out, slice, 0, n, chunk_index)); + } + UNPROTECT(1); + return out; + } + // Converter factory static std::shared_ptr Make( const std::shared_ptr& chunked_array); @@ -761,24 +795,36 @@ class Converter_Struct : public Converter { SEXP Allocate(R_xlen_t n) const { // allocate a data frame column to host each array // If possible, a column is dealt with directly with altrep - auto type = - checked_cast(this->chunked_array_->type().get()); - auto out = - arrow::r::to_r_list(converters, [n](const std::shared_ptr& converter) { - SEXP out = converter->MaybeAltrep(); - if (Rf_isNull(out)) { - out = converter->Allocate(n); - } - return out; - }); - auto colnames = arrow::r::to_r_strings( - type->fields(), - [](const std::shared_ptr& field) { return field->name(); }); - out.attr(symbols::row_names) = arrow::r::short_row_names(static_cast(n)); - out.attr(R_NamesSymbol) = colnames; - out.attr(R_ClassSymbol) = arrow::r::data::classes_tbl_df; + return AllocateDataFrame(n, [n](const std::shared_ptr& converter) { + SEXP out = converter->MaybeAltrep(); + if (Rf_isNull(out)) { + out = converter->Allocate(n); + } + return out; + }); + } - return out; + SEXP AllocateSlice(R_xlen_t n) const { + // altrep would shadow the whole chunked array rather than the slice + return AllocateDataFrame(n, [n](const std::shared_ptr& converter) { + return converter->AllocateSlice(n); + }); + } + + // Convert each child of the slice through its own converter, rather than + // allocating then ingesting: that lets children that can only convert whole + // arrays (extension types) work, and children that can be altrep be altrep + SEXP ConvertSlice(const std::shared_ptr& slice, size_t chunk_index) const { + const auto& struct_array = checked_cast(*slice); + // Flatten() deals with merging of nulls + auto arrays = ValueOrStop(struct_array.Flatten(gc_memory_pool())); + int nf = static_cast(converters.size()); + + cpp11::writable::list out(nf); + for (int i = 0; i < nf; i++) { + out[i] = converters[i]->ConvertSlice(arrays[i], chunk_index); + } + return FinishDataFrame(out, slice->length()); } Status Ingest_all_nulls(SEXP data, R_xlen_t start, R_xlen_t n) const { @@ -824,6 +870,27 @@ class Converter_Struct : public Converter { private: std::vector> converters; + + template + SEXP AllocateDataFrame(R_xlen_t n, AllocateColumn&& allocate_column) const { + auto out = + arrow::r::to_r_list(converters, std::forward(allocate_column)); + return FinishDataFrame(out, n); + } + + // set the names, row names and class of a list of columns + SEXP FinishDataFrame(cpp11::writable::list& out, R_xlen_t n) const { + auto type = + checked_cast(this->chunked_array_->type().get()); + auto colnames = arrow::r::to_r_strings( + type->fields(), + [](const std::shared_ptr& field) { return field->name(); }); + out.attr(symbols::row_names) = arrow::r::short_row_names(static_cast(n)); + out.attr(R_NamesSymbol) = colnames; + out.attr(R_ClassSymbol) = arrow::r::data::classes_tbl_df; + + return out; + } }; double ms_to_seconds(int64_t ms) { return static_cast(ms) / 1000; } @@ -1021,15 +1088,36 @@ class Converter_Decimal : public Converter { } }; +// Build a converter for the values of all chunks of a list-like chunked array, so +// that decisions such as factor levels or whether integers fit are made once for +// the whole column rather than once per list element (GH-50514, GH-50339) +template +std::shared_ptr MakeListValuesConverter( + const std::shared_ptr& chunked_array, + const std::shared_ptr& value_type) { + ArrayVector values; + values.reserve(chunked_array->num_chunks()); + for (const auto& chunk : chunked_array->chunks()) { + // Flatten() rather than values() so that only the values that are logically + // part of the list (respecting the offset of a sliced array and null lists) + // take part in the decisions + values.push_back(ValueOrStop( + checked_cast(*chunk).Flatten(gc_memory_pool()))); + } + return Converter::Make(std::make_shared(std::move(values), value_type)); +} + template class Converter_List : public Converter { private: - std::shared_ptr value_type_; + std::shared_ptr values_converter_; public: explicit Converter_List(const std::shared_ptr& chunked_array, const std::shared_ptr& value_type) - : Converter(chunked_array), value_type_(value_type) {} + : Converter(chunked_array), + values_converter_( + MakeListValuesConverter(chunked_array, value_type)) {} SEXP Allocate(R_xlen_t n) const { cpp11::writable::list res(n); @@ -1042,10 +1130,8 @@ class Converter_List : public Converter { res.attr(R_ClassSymbol) = arrow::r::data::classes_arrow_large_list; } - std::shared_ptr array = CreateEmptyArray(value_type_); - - // convert to an R object to store as the list' ptype - res.attr(arrow::r::symbols::ptype) = Converter::Convert(array); + // an empty R object of the type of the elements, stored as the list's ptype + res.attr(arrow::r::symbols::ptype) = values_converter_->AllocateSlice(0); return res; } @@ -1058,11 +1144,11 @@ class Converter_List : public Converter { Status Ingest_some_nulls(SEXP data, const std::shared_ptr& array, R_xlen_t start, R_xlen_t n, size_t chunk_index) const { auto list_array = checked_cast(array.get()); - auto values_array = list_array->values(); auto ingest_one = [&](R_xlen_t i) { auto slice = list_array->value_slice(i); - SET_VECTOR_ELT(data, i + start, Converter::Convert(slice)); + SET_VECTOR_ELT(data, i + start, + values_converter_->ConvertSlice(slice, chunk_index)); return Status::OK(); }; @@ -1074,24 +1160,25 @@ class Converter_List : public Converter { class Converter_FixedSizeList : public Converter { private: - std::shared_ptr value_type_; + std::shared_ptr values_converter_; int list_size_; public: explicit Converter_FixedSizeList(const std::shared_ptr& chunked_array, const std::shared_ptr& value_type, int list_size) - : Converter(chunked_array), value_type_(value_type), list_size_(list_size) {} + : Converter(chunked_array), + values_converter_( + MakeListValuesConverter(chunked_array, value_type)), + list_size_(list_size) {} SEXP Allocate(R_xlen_t n) const { cpp11::writable::list res(n); Rf_classgets(res, arrow::r::data::classes_arrow_fixed_size_list); res.attr(arrow::r::symbols::list_size) = Rf_ScalarInteger(list_size_); - std::shared_ptr array = CreateEmptyArray(value_type_); - - // convert to an R object to store as the list' ptype - res.attr(arrow::r::symbols::ptype) = Converter::Convert(array); + // an empty R object of the type of the elements, stored as the list's ptype + res.attr(arrow::r::symbols::ptype) = values_converter_->AllocateSlice(0); return res; } @@ -1104,11 +1191,11 @@ class Converter_FixedSizeList : public Converter { Status Ingest_some_nulls(SEXP data, const std::shared_ptr& array, R_xlen_t start, R_xlen_t n, size_t chunk_index) const { const auto& fixed_size_list_array = checked_cast(*array); - auto values_array = fixed_size_list_array.values(); auto ingest_one = [&](R_xlen_t i) { auto slice = fixed_size_list_array.value_slice(i); - SET_VECTOR_ELT(data, i + start, Converter::Convert(slice)); + SET_VECTOR_ELT(data, i + start, + values_converter_->ConvertSlice(slice, chunk_index)); return Status::OK(); }; return IngestSome(array, n, ingest_one); @@ -1198,6 +1285,20 @@ class Converter_Extension : public Converter { return extension_type->Convert(chunked_array_); } + // The conversion happens in Allocate() over the whole chunked array, so a slice + // can't reuse it: convert an empty slice / the slice itself on its own instead. + // Non-empty slices always go through ConvertSlice() below. + SEXP AllocateSlice(R_xlen_t n) const { + if (n != 0) { + cpp11::stop("Cannot allocate a non-empty slice of an extension array"); + } + return Converter::Convert(chunked_array_->Slice(0, 0), false); + } + + SEXP ConvertSlice(const std::shared_ptr& slice, size_t chunk_index) const { + return Converter::Convert(slice); + } + // At this point we have already done the conversion Status Ingest_all_nulls(SEXP data, R_xlen_t start, R_xlen_t n) const { return Status::OK(); diff --git a/r/tests/testthat/test-Array.R b/r/tests/testthat/test-Array.R index e7a6ce5d2410..3544c98a6943 100644 --- a/r/tests/testthat/test-Array.R +++ b/r/tests/testthat/test-Array.R @@ -1466,3 +1466,225 @@ test_that("uint64 inside list columns always converts to double (GH-50339)", { expect_type(result[[1]], "double") expect_type(result[[2]], "double") }) + +test_that("list of dictionary: ptype and elements share the dictionary's levels (GH-50514)", { + arr <- arrow_array( + list(factor(c("a", "b"), levels = c("a", "b", "c")), factor("c", levels = c("a", "b", "c"))), + type = list_of(dictionary(int8(), utf8())) + ) + result <- as.vector(arr) + + ptype <- attr(result, "ptype") + expect_s3_class(ptype, "factor") + expect_identical(levels(ptype), c("a", "b", "c")) + expect_identical(levels(result[[1]]), c("a", "b", "c")) + expect_identical(levels(result[[2]]), c("a", "b", "c")) + expect_identical(result[[1]], factor(c("a", "b"), levels = c("a", "b", "c"))) +}) + +test_that("list> unifies factor levels across chunks (GH-50514)", { + skip_if_not_installed("tidyr") + + batch1 <- record_batch( + id = 1:2, + resources = list( + data.frame(type = factor(c("river", "lake"))), + data.frame(type = factor("river")) + ) + ) + batch2 <- record_batch( + id = 3L, + resources = list(data.frame(type = factor(c("sea", "lake")))) + ) + # Each batch carries its own dictionary for resources$type + dict1 <- batch1$resources$values()$GetFieldByName("type")$dictionary() + dict2 <- batch2$resources$values()$GetFieldByName("type")$dictionary() + expect_false(dict1$Equals(dict2)) + + buf <- write_to_raw(Table$create(batch1, batch2), format = "stream") + df <- as.data.frame(read_ipc_stream(buf)) + + ptype_levels <- levels(attr(df$resources, "ptype")$type) + expect_identical(sort(ptype_levels), c("lake", "river", "sea")) + expect_identical( + lapply(df$resources, function(element) levels(element$type)), + rep(list(ptype_levels), 3) + ) + expect_identical(as.character(df$resources[[3]]$type), c("sea", "lake")) + + unnested <- tidyr::unnest(df, "resources", names_sep = "_") + expect_s3_class(unnested$resources_type, "factor") + expect_identical( + as.character(unnested$resources_type), + c("river", "lake", "river", "sea", "lake") + ) +}) + +test_that("list of ordered dictionary keeps ordered class across chunks (GH-50514)", { + arr1 <- arrow_array(list(factor("lo", levels = c("lo", "hi"), ordered = TRUE))) + arr2 <- arrow_array(list(factor("hi", levels = c("hi", "max"), ordered = TRUE))) + result <- as.vector(chunked_array(arr1, arr2)) + + expect_s3_class(attr(result, "ptype"), "ordered") + expect_s3_class(result[[1]], "ordered") + expect_identical(levels(result[[1]]), levels(result[[2]])) + expect_identical(as.character(result[[1]]), "lo") + expect_identical(as.character(result[[2]]), "hi") +}) + +test_that("large_list, fixed_size_list and map of dictionary unify levels across chunks (GH-50514)", { + f1 <- list(factor(c("a", "b"))) + f2 <- list(factor(c("c", "b"))) + + large <- chunked_array( + arrow_array(f1, type = large_list_of(dictionary())), + arrow_array(f2, type = large_list_of(dictionary())) + ) + result <- as.vector(large) + expect_identical(levels(result[[1]]), levels(result[[2]])) + expect_identical(levels(attr(result, "ptype")), levels(result[[1]])) + expect_identical(as.character(result[[2]]), c("c", "b")) + + fixed <- chunked_array( + arrow_array(f1, type = fixed_size_list_of(dictionary(), 2L)), + arrow_array(f2, type = fixed_size_list_of(dictionary(), 2L)) + ) + result <- as.vector(fixed) + expect_identical(levels(result[[1]]), levels(result[[2]])) + expect_identical(levels(attr(result, "ptype")), levels(result[[1]])) + expect_identical(as.character(result[[2]]), c("c", "b")) + + map_type <- map_of(utf8(), dictionary()) + m1 <- arrow_array(list(data.frame(key = "k", value = factor("a"))), type = map_type) + m2 <- arrow_array(list(data.frame(key = "k", value = factor("b"))), type = map_type) + result <- as.vector(chunked_array(m1, m2)) + expect_identical(levels(result[[1]]$value), levels(result[[2]]$value)) + expect_identical(levels(attr(result, "ptype")$value), levels(result[[1]]$value)) + expect_identical(as.character(result[[2]]$value), "b") +}) + +test_that("int64 inside list columns converts to one type for the whole column (GH-50514)", { + small <- arrow_array(list(bit64::as.integer64(1:2)), type = list_of(int64())) + big <- arrow_array(list(bit64::as.integer64(2)^40), type = list_of(int64())) + + # all values fit: integer, including the ptype + result <- as.vector(small) + expect_type(result[[1]], "integer") + expect_type(attr(result, "ptype"), "integer") + + # one element doesn't fit: integer64 everywhere, including the ptype + result <- as.vector(chunked_array(small, big)) + expect_s3_class(result[[1]], "integer64") + expect_s3_class(result[[2]], "integer64") + expect_s3_class(attr(result, "ptype"), "integer64") + expect_identical(result[[1]], bit64::as.integer64(1:2)) + + # and the option is respected inside lists + withr::with_options(list(arrow.int64_downcast = FALSE), { + result <- as.vector(small) + expect_s3_class(result[[1]], "integer64") + expect_s3_class(attr(result, "ptype"), "integer64") + }) +}) + +test_that("uint32 inside list columns converts to one type for the whole column (GH-50514)", { + small <- arrow_array(list(1:2), type = list_of(uint32())) + big <- arrow_array(list(.Machine$integer.max + 1), type = list_of(uint32())) + + result <- as.vector(small) + expect_type(result[[1]], "integer") + expect_type(attr(result, "ptype"), "integer") + + result <- as.vector(chunked_array(small, big)) + expect_type(result[[1]], "double") + expect_type(result[[2]], "double") + expect_type(attr(result, "ptype"), "double") + expect_identical(result[[2]], .Machine$integer.max + 1) +}) + +test_that("nested lists unify inner element types across chunks (GH-50514)", { + # list> + a <- arrow_array(list(list(factor("a"))), type = list_of(list_of(dictionary()))) + b <- arrow_array(list(list(factor("b"))), type = list_of(list_of(dictionary()))) + result <- as.vector(chunked_array(a, b)) + expect_identical(levels(result[[1]][[1]]), levels(result[[2]][[1]])) + expect_identical(levels(attr(attr(result, "ptype"), "ptype")), levels(result[[1]][[1]])) + expect_identical(as.character(result[[2]][[1]]), "b") + + # list>> + type <- list_of(struct(a = list_of(int64()))) + a <- arrow_array(list(data.frame(a = I(list(bit64::as.integer64(1))))), type = type) + b <- arrow_array(list(data.frame(a = I(list(bit64::as.integer64(2)^40)))), type = type) + result <- as.vector(chunked_array(a, b)) + expect_s3_class(result[[1]]$a[[1]], "integer64") + expect_s3_class(result[[2]]$a[[1]], "integer64") + expect_s3_class(attr(attr(result, "ptype")$a, "ptype"), "integer64") +}) + +test_that("empty and all-null list of dictionary columns still convert (GH-50514)", { + type <- list_of(dictionary()) + + empty <- as.vector(arrow_array(list(), type = type)) + expect_length(empty, 0) + expect_s3_class(attr(empty, "ptype"), "factor") + + nulls <- as.vector(arrow_array(list(NULL, NULL), type = type)) + expect_identical(nulls[[1]], NULL) + expect_s3_class(attr(nulls, "ptype"), "factor") + + one_empty_chunk <- ChunkedArray$create(type = type) + result <- as.vector(one_empty_chunk) + expect_length(result, 0) + expect_s3_class(attr(result, "ptype"), "factor") + + zero_chunks <- one_empty_chunk$Filter(ChunkedArray$create(type = bool())) + expect_equal(zero_chunks$num_chunks, 0) + result <- as.vector(zero_chunks) + expect_length(result, 0) + expect_s3_class(attr(result, "ptype"), "factor") +}) + +test_that("string list elements are still altrep vectors (GH-50514)", { + skip_if_not(getOption("arrow.use_altrep", TRUE)) + result <- as.vector(arrow_array(list(c("a", "b"), "c"))) + expect_true(is_arrow_altrep(result[[1]])) + expect_identical(result[[1]], c("a", "b")) +}) + +test_that("list of extension type still converts (GH-50514)", { + vctr <- vctrs::new_vctr(1:3, class = "custom_vctr") + # list can't be built directly from R, but can be cast to + arr <- arrow_array(list(1:3, 1L))$cast(list_of(vctrs_extension_type(vctr))) + expect_r6_class(arr$type$value_type, "ExtensionType") + + result <- as.vector(arr) + expect_identical(result[[1]], vctr) + expect_identical(result[[2]], vctr[1]) + expect_identical(attr(result, "ptype"), vctr[0]) +}) + +test_that("only the logical values of a sliced list array take part in conversion decisions (GH-50514)", { + arr <- arrow_array( + list(bit64::as.integer64(2)^40, bit64::as.integer64(1:2)), + type = list_of(int64()) + ) + # the whole array needs integer64 + expect_s3_class(as.vector(arr)[[2]], "integer64") + + # but a slice that leaves out the large value doesn't + sliced <- as.vector(arr$Slice(1)) + expect_type(sliced[[1]], "integer") + expect_type(attr(sliced, "ptype"), "integer") +}) + +test_that("list of struct with an extension type column converts (GH-50514)", { + vctr <- vctrs::new_vctr(1:3, class = "custom_vctr") + arr <- arrow_array(list(data.frame(x = 1:3), data.frame(x = 1L)))$cast( + list_of(struct(x = vctrs_extension_type(vctr))) + ) + + result <- as.vector(arr) + expect_identical(result[[1]]$x, vctr) + expect_identical(result[[2]]$x, vctr[1]) + expect_identical(attr(result, "ptype")$x, vctr[0]) +}) diff --git a/r/tests/testthat/test-chunked-array.R b/r/tests/testthat/test-chunked-array.R index e5fcfefe9045..d654b3a92da6 100644 --- a/r/tests/testthat/test-chunked-array.R +++ b/r/tests/testthat/test-chunked-array.R @@ -561,3 +561,18 @@ test_that("float16 values roundtrip to R correctly", { expect_as_vector(a$chunk(1), x[5:7]) expect_as_vector(a$Slice(1), x[-1]) }) + +test_that("Converting a chunked array of lists unifies nested factors (GH-50514)", { + a <- chunked_array( + list(factor(c("a", "b"))), + list(factor("c")), + list(factor("d"), factor("a")) + ) + expect_r6_class(a$type$value_type, "DictionaryType") + + result <- as.vector(a) + unified <- c("a", "b", "c", "d") + expect_identical(levels(attr(result, "ptype")), unified) + expect_identical(lapply(result, levels), rep(list(unified), 4)) + expect_identical(lapply(result, as.character), list(c("a", "b"), "c", "d", "a")) +}) From ac1dae317ae8a2929b6b98e92f2eb68b2b2d155c Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Thu, 3 Sep 2026 14:30:46 +0100 Subject: [PATCH 4/5] Rephrase NEWS --- r/NEWS.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/r/NEWS.md b/r/NEWS.md index 751acea4d32b..185316beb8e6 100644 --- a/r/NEWS.md +++ b/r/NEWS.md @@ -21,11 +21,11 @@ ## Minor improvements and fixes -- When converting Arrow list-columns to R (e.g. via `as.data.frame()` or - `collect()`), every element now has the same type as the column's `ptype`: - factors share one set of levels and `int64`/`uint32` values are either all - integers or all `integer64`/doubles, so `tidyr::unnest()` works on them - (#50514). +- Factor levels inside list columns are now unified across the whole column + when converting to R, so data read in multiple batches (e.g. via + `read_ipc_stream()` or `open_dataset()`) produces valid factors that can be + unnested. Similarly, `int64` and `uint32` values inside list columns are + converted to a single R type across the column (#50514). # arrow 25.0.1 From 9a962751547de1a2407b0138b20c1c945b0f0397 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Thu, 3 Sep 2026 17:23:07 +0100 Subject: [PATCH 5/5] remove errant tidyr ref --- r/tests/testthat/test-Array.R | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/r/tests/testthat/test-Array.R b/r/tests/testthat/test-Array.R index 3544c98a6943..ea3bff782663 100644 --- a/r/tests/testthat/test-Array.R +++ b/r/tests/testthat/test-Array.R @@ -1483,8 +1483,6 @@ test_that("list of dictionary: ptype and elements share the dictionary's levels }) test_that("list> unifies factor levels across chunks (GH-50514)", { - skip_if_not_installed("tidyr") - batch1 <- record_batch( id = 1:2, resources = list( @@ -1512,10 +1510,11 @@ test_that("list> unifies factor levels across chunks (GH-5051 ) expect_identical(as.character(df$resources[[3]]$type), c("sea", "lake")) - unnested <- tidyr::unnest(df, "resources", names_sep = "_") - expect_s3_class(unnested$resources_type, "factor") + # tidyr::unnest() combines the elements via vctrs using the list's ptype + unnested <- vctrs::vec_rbind(!!!df$resources, .ptype = attr(df$resources, "ptype")) + expect_s3_class(unnested$type, "factor") expect_identical( - as.character(unnested$resources_type), + as.character(unnested$type), c("river", "lake", "river", "sea", "lake") ) })