From f3e92370f68267ba18097e59ea103cea3c640aef Mon Sep 17 00:00:00 2001 From: Yifan Chen Date: Thu, 10 Sep 2026 00:18:02 -0700 Subject: [PATCH 1/5] GH-36010: [GLib][Ruby][Parquet] Add buffered reader properties Expose buffered stream controls and additive properties-aware reader constructors. Copy native properties and retain the input stream; disable whole-column read-ahead when buffered streams are selected. Generated-by: OpenAI Codex --- c_glib/parquet-glib/arrow-file-reader.cpp | 232 ++++++++++++++++++ c_glib/parquet-glib/arrow-file-reader.h | 47 ++++ c_glib/parquet-glib/arrow-file-reader.hpp | 2 + c_glib/test/parquet/test-arrow-file-reader.rb | 74 ++++++ c_glib/test/parquet/test-reader-properties.rb | 38 +++ .../test/test-arrow-file-reader.rb | 28 +++ 6 files changed, 421 insertions(+) create mode 100644 c_glib/test/parquet/test-reader-properties.rb diff --git a/c_glib/parquet-glib/arrow-file-reader.cpp b/c_glib/parquet-glib/arrow-file-reader.cpp index 86bf284d1236..150baf2a5b76 100644 --- a/c_glib/parquet-glib/arrow-file-reader.cpp +++ b/c_glib/parquet-glib/arrow-file-reader.cpp @@ -25,6 +25,33 @@ #include +namespace { + GParquetArrowFileReader * + open_reader_with_properties(std::shared_ptr source, + GParquetReaderProperties *properties, + GError **error, + const char *tag) + { + auto parquet_properties = properties ? gparquet_reader_properties_get_raw(properties) + : parquet::default_reader_properties(); + parquet::arrow::FileReaderBuilder builder; + if (!garrow::check(error, builder.Open(source, parquet_properties), tag)) { + return NULL; + } + if (parquet_properties.is_buffered_stream_enabled()) { + // Read-ahead would bypass the buffered stream by caching whole column chunks. + auto arrow_properties = parquet::default_arrow_reader_properties(); + arrow_properties.set_pre_buffer(false); + builder.properties(arrow_properties); + } + auto result = builder.Build(); + if (!garrow::check(error, result, tag)) { + return NULL; + } + return gparquet_arrow_file_reader_new_raw(result->release()); + } +} // namespace + G_BEGIN_DECLS /** @@ -32,13 +59,145 @@ G_BEGIN_DECLS * @short_description: Arrow file reader class * @include: parquet-glib/parquet-glib.h * + * #GParquetReaderProperties is a class for configuring Parquet reads. + * * #GParquetArrowFileReader is a class for reading Apache Parquet data * from file and returns them as Apache Arrow data. */ +typedef struct GParquetReaderPropertiesPrivate_ +{ + parquet::ReaderProperties properties; +} GParquetReaderPropertiesPrivate; + +G_DEFINE_TYPE_WITH_PRIVATE(GParquetReaderProperties, + gparquet_reader_properties, + G_TYPE_OBJECT) + +#define GPARQUET_READER_PROPERTIES_GET_PRIVATE(object) \ + static_cast( \ + gparquet_reader_properties_get_instance_private(GPARQUET_READER_PROPERTIES(object))) + +static void +gparquet_reader_properties_finalize(GObject *object) +{ + auto priv = GPARQUET_READER_PROPERTIES_GET_PRIVATE(object); + priv->properties.~ReaderProperties(); + G_OBJECT_CLASS(gparquet_reader_properties_parent_class)->finalize(object); +} + +static void +gparquet_reader_properties_init(GParquetReaderProperties *object) +{ + auto priv = GPARQUET_READER_PROPERTIES_GET_PRIVATE(object); + new (&priv->properties) parquet::ReaderProperties(parquet::default_reader_properties()); +} + +static void +gparquet_reader_properties_class_init(GParquetReaderPropertiesClass *klass) +{ + G_OBJECT_CLASS(klass)->finalize = gparquet_reader_properties_finalize; +} + +/** + * gparquet_reader_properties_new: + * + * Returns: A newly created #GParquetReaderProperties. + * + * Since: 26.0.0 + */ +GParquetReaderProperties * +gparquet_reader_properties_new(void) +{ + return GPARQUET_READER_PROPERTIES(g_object_new(GPARQUET_TYPE_READER_PROPERTIES, NULL)); +} + +/** + * gparquet_reader_properties_enable_buffered_stream: + * @properties: A #GParquetReaderProperties. + * + * Enable buffered stream reading. Readers constructed with these properties + * disable read-ahead of whole column chunks to use buffered streams instead. + * This does not impose a limit on the memory used by decoded data. + * + * Since: 26.0.0 + */ +void +gparquet_reader_properties_enable_buffered_stream(GParquetReaderProperties *properties) +{ + auto priv = GPARQUET_READER_PROPERTIES_GET_PRIVATE(properties); + priv->properties.enable_buffered_stream(); +} + +/** + * gparquet_reader_properties_disable_buffered_stream: + * @properties: A #GParquetReaderProperties. + * + * Disable buffered stream reading. This is the default. + * + * Since: 26.0.0 + */ +void +gparquet_reader_properties_disable_buffered_stream(GParquetReaderProperties *properties) +{ + auto priv = GPARQUET_READER_PROPERTIES_GET_PRIVATE(properties); + priv->properties.disable_buffered_stream(); +} + +/** + * gparquet_reader_properties_is_buffered_stream_enabled: + * @properties: A #GParquetReaderProperties. + * + * Returns: %TRUE if buffered stream reading is enabled, %FALSE otherwise. + * + * Since: 26.0.0 + */ +gboolean +gparquet_reader_properties_is_buffered_stream_enabled( + GParquetReaderProperties *properties) +{ + auto priv = GPARQUET_READER_PROPERTIES_GET_PRIVATE(properties); + return priv->properties.is_buffered_stream_enabled(); +} + +/** + * gparquet_reader_properties_set_buffer_size: + * @properties: A #GParquetReaderProperties. + * @buffer_size: The buffer size in bytes. This must be positive when buffering is + * enabled. + * + * Set the buffered stream size. This does not enable buffered stream reading. + * Reads required for a data page may exceed this size. + * + * Since: 26.0.0 + */ +void +gparquet_reader_properties_set_buffer_size(GParquetReaderProperties *properties, + gint64 buffer_size) +{ + auto priv = GPARQUET_READER_PROPERTIES_GET_PRIVATE(properties); + priv->properties.set_buffer_size(buffer_size); +} + +/** + * gparquet_reader_properties_get_buffer_size: + * @properties: A #GParquetReaderProperties. + * + * Returns: The buffered stream size in bytes. + * + * Since: 26.0.0 + */ +gint64 +gparquet_reader_properties_get_buffer_size(GParquetReaderProperties *properties) +{ + auto priv = GPARQUET_READER_PROPERTIES_GET_PRIVATE(properties); + return priv->properties.buffer_size(); +} + typedef struct GParquetArrowFileReaderPrivate_ { parquet::arrow::FileReader *arrow_file_reader; + GArrowSeekableInputStream *source; } GParquetArrowFileReaderPrivate; enum { @@ -54,6 +213,14 @@ G_DEFINE_TYPE_WITH_PRIVATE(GParquetArrowFileReader, static_cast( \ gparquet_arrow_file_reader_get_instance_private(GPARQUET_ARROW_FILE_READER(obj))) +static void +gparquet_arrow_file_reader_dispose(GObject *object) +{ + auto priv = GPARQUET_ARROW_FILE_READER_GET_PRIVATE(object); + g_clear_object(&priv->source); + G_OBJECT_CLASS(gparquet_arrow_file_reader_parent_class)->dispose(object); +} + static void gparquet_arrow_file_reader_finalize(GObject *object) { @@ -108,6 +275,7 @@ gparquet_arrow_file_reader_class_init(GParquetArrowFileReaderClass *klass) auto gobject_class = G_OBJECT_CLASS(klass); + gobject_class->dispose = gparquet_arrow_file_reader_dispose; gobject_class->finalize = gparquet_arrow_file_reader_finalize; gobject_class->set_property = gparquet_arrow_file_reader_set_property; gobject_class->get_property = gparquet_arrow_file_reader_get_property; @@ -181,6 +349,63 @@ gparquet_arrow_file_reader_new_path(const gchar *path, GError **error) } } +/** + * gparquet_arrow_file_reader_new_arrow_with_properties: + * @source: Arrow source to be read. + * @properties: (nullable): Reader properties or %NULL for the defaults. + * @error: (nullable): Return location for a #GError or %NULL. + * + * The reader copies @properties at construction. Later changes to @properties + * do not affect the reader. The native source is retained by the reader. + * + * Returns: (nullable): A newly created #GParquetArrowFileReader. + * + * Since: 26.0.0 + */ +GParquetArrowFileReader * +gparquet_arrow_file_reader_new_arrow_with_properties(GArrowSeekableInputStream *source, + GParquetReaderProperties *properties, + GError **error) +{ + auto reader = open_reader_with_properties( + garrow_seekable_input_stream_get_raw(source), + properties, + error, + "[parquet][arrow][file-reader][new-arrow-with-properties]"); + if (reader) { + auto priv = GPARQUET_ARROW_FILE_READER_GET_PRIVATE(reader); + priv->source = GARROW_SEEKABLE_INPUT_STREAM(g_object_ref(source)); + } + return reader; +} + +/** + * gparquet_arrow_file_reader_new_path_with_properties: + * @path: Path to be read. + * @properties: (nullable): Reader properties or %NULL for the defaults. + * @error: (nullable): Return location for a #GError or %NULL. + * + * The reader copies @properties at construction. Later changes to @properties + * do not affect the reader. The file is memory mapped, as with + * gparquet_arrow_file_reader_new_path(). + * + * Returns: (nullable): A newly created #GParquetArrowFileReader. + * + * Since: 26.0.0 + */ +GParquetArrowFileReader * +gparquet_arrow_file_reader_new_path_with_properties(const gchar *path, + GParquetReaderProperties *properties, + GError **error) +{ + const char *tag = "[parquet][arrow][file-reader][new-path-with-properties]"; + auto source = arrow::io::MemoryMappedFile::Open(path, arrow::io::FileMode::READ); + if (!garrow::check(error, source, tag)) { + return NULL; + } + return open_reader_with_properties(*source, properties, error, tag); +} + /** * gparquet_arrow_file_reader_close: * @reader: A #GParquetArrowFileReader. @@ -426,3 +651,10 @@ gparquet_arrow_file_reader_get_raw(GParquetArrowFileReader *arrow_file_reader) auto priv = GPARQUET_ARROW_FILE_READER_GET_PRIVATE(arrow_file_reader); return priv->arrow_file_reader; } + +parquet::ReaderProperties +gparquet_reader_properties_get_raw(GParquetReaderProperties *properties) +{ + auto priv = GPARQUET_READER_PROPERTIES_GET_PRIVATE(properties); + return priv->properties; +} diff --git a/c_glib/parquet-glib/arrow-file-reader.h b/c_glib/parquet-glib/arrow-file-reader.h index 04df24b87d31..306f6204e533 100644 --- a/c_glib/parquet-glib/arrow-file-reader.h +++ b/c_glib/parquet-glib/arrow-file-reader.h @@ -23,6 +23,41 @@ G_BEGIN_DECLS +#define GPARQUET_TYPE_READER_PROPERTIES (gparquet_reader_properties_get_type()) +GPARQUET_AVAILABLE_IN_26_0 +G_DECLARE_DERIVABLE_TYPE(GParquetReaderProperties, + gparquet_reader_properties, + GPARQUET, + READER_PROPERTIES, + GObject) +struct _GParquetReaderPropertiesClass +{ + GObjectClass parent_class; +}; + +GPARQUET_AVAILABLE_IN_26_0 +GParquetReaderProperties * +gparquet_reader_properties_new(void); + +GPARQUET_AVAILABLE_IN_26_0 +void +gparquet_reader_properties_enable_buffered_stream(GParquetReaderProperties *properties); +GPARQUET_AVAILABLE_IN_26_0 +void +gparquet_reader_properties_disable_buffered_stream(GParquetReaderProperties *properties); +GPARQUET_AVAILABLE_IN_26_0 +gboolean +gparquet_reader_properties_is_buffered_stream_enabled( + GParquetReaderProperties *properties); + +GPARQUET_AVAILABLE_IN_26_0 +void +gparquet_reader_properties_set_buffer_size(GParquetReaderProperties *properties, + gint64 buffer_size); +GPARQUET_AVAILABLE_IN_26_0 +gint64 +gparquet_reader_properties_get_buffer_size(GParquetReaderProperties *properties); + #define GPARQUET_TYPE_ARROW_FILE_READER (gparquet_arrow_file_reader_get_type()) GPARQUET_AVAILABLE_IN_0_11 G_DECLARE_DERIVABLE_TYPE(GParquetArrowFileReader, @@ -43,6 +78,18 @@ GPARQUET_AVAILABLE_IN_0_11 GParquetArrowFileReader * gparquet_arrow_file_reader_new_path(const gchar *path, GError **error); +GPARQUET_AVAILABLE_IN_26_0 +GParquetArrowFileReader * +gparquet_arrow_file_reader_new_arrow_with_properties(GArrowSeekableInputStream *source, + GParquetReaderProperties *properties, + GError **error); + +GPARQUET_AVAILABLE_IN_26_0 +GParquetArrowFileReader * +gparquet_arrow_file_reader_new_path_with_properties(const gchar *path, + GParquetReaderProperties *properties, + GError **error); + GPARQUET_AVAILABLE_IN_23_0 void gparquet_arrow_file_reader_close(GParquetArrowFileReader *reader); diff --git a/c_glib/parquet-glib/arrow-file-reader.hpp b/c_glib/parquet-glib/arrow-file-reader.hpp index 172dcccb0d45..c240e9f26953 100644 --- a/c_glib/parquet-glib/arrow-file-reader.hpp +++ b/c_glib/parquet-glib/arrow-file-reader.hpp @@ -27,3 +27,5 @@ GParquetArrowFileReader * gparquet_arrow_file_reader_new_raw(parquet::arrow::FileReader *parquet_arrow_file_reader); parquet::arrow::FileReader * gparquet_arrow_file_reader_get_raw(GParquetArrowFileReader *arrow_file_reader); +parquet::ReaderProperties +gparquet_reader_properties_get_raw(GParquetReaderProperties *properties); diff --git a/c_glib/test/parquet/test-arrow-file-reader.rb b/c_glib/test/parquet/test-arrow-file-reader.rb index eff5ad966aea..304d3c14efb8 100644 --- a/c_glib/test/parquet/test-arrow-file-reader.rb +++ b/c_glib/test/parquet/test-arrow-file-reader.rb @@ -39,6 +39,80 @@ def setup end end + sub_test_case(".new with properties") do + data("path" => :path, "stream" => :stream) + test("read") do |source_type| + properties = Parquet::ReaderProperties.new + properties.enable_buffered_stream + properties.buffer_size = 4096 + source = if source_type == :path + @file.path + else + Arrow::FileInputStream.new(@file.path) + end + reader = Parquet::ArrowFileReader.new(source, properties) + begin + # The reader owns a copy of the properties and the native source. + properties.disable_buffered_stream + properties.buffer_size = 0 + properties.unref + source.unref if source_type == :stream + assert_equal(@table, reader.read_table) + assert_equal(build_table("a" => @a_array.slice(1, 1), + "b" => @b_array.slice(1, 1)), + reader.read_row_group(1)) + ensure + reader.close + reader.unref + end + end + + data("path" => :path, "stream" => :stream) + test("default properties") do |source_type| + source = if source_type == :path + @file.path + else + Arrow::FileInputStream.new(@file.path) + end + reader = Parquet::ArrowFileReader.new(source, nil) + begin + assert_equal(@table, reader.read_table) + ensure + reader.close + reader.unref + source.unref if source_type == :stream + end + end + + test("missing path") do + properties = Parquet::ReaderProperties.new + assert_raise(Arrow::Error::Io) do + Parquet::ArrowFileReader.new("#{@file.path}.missing", properties) + end + end + + data("path" => :path, "stream" => :stream) + test("invalid file") do |source_type| + Tempfile.create("invalid-parquet") do |file| + file.write("not a parquet file") + file.flush + source = if source_type == :path + file.path + else + Arrow::FileInputStream.new(file.path) + end + properties = Parquet::ReaderProperties.new + begin + assert_raise(Arrow::Error::Invalid) do + Parquet::ArrowFileReader.new(source, properties) + end + ensure + source.unref if source_type == :stream + end + end + end + end + def test_schema assert_equal(<<-SCHEMA.chomp, @reader.schema.to_s) a: string diff --git a/c_glib/test/parquet/test-reader-properties.rb b/c_glib/test/parquet/test-reader-properties.rb new file mode 100644 index 000000000000..16d117339a5b --- /dev/null +++ b/c_glib/test/parquet/test-reader-properties.rb @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestParquetReaderProperties < Test::Unit::TestCase + def setup + omit("Parquet is required") unless defined?(::Parquet) + @properties = Parquet::ReaderProperties.new + end + + def test_buffered_stream + assert_false(@properties.buffered_stream_enabled?) + @properties.enable_buffered_stream + assert_true(@properties.buffered_stream_enabled?) + @properties.disable_buffered_stream + assert_false(@properties.buffered_stream_enabled?) + end + + def test_buffer_size + assert_equal(16 * 1024, @properties.buffer_size) + @properties.buffer_size = 32 * 1024 + assert_equal(32 * 1024, @properties.buffer_size) + assert_false(@properties.buffered_stream_enabled?) + end +end diff --git a/ruby/red-parquet/test/test-arrow-file-reader.rb b/ruby/red-parquet/test/test-arrow-file-reader.rb index 61db74b899ef..e11615a3b2ab 100644 --- a/ruby/red-parquet/test/test-arrow-file-reader.rb +++ b/ruby/red-parquet/test/test-arrow-file-reader.rb @@ -29,6 +29,34 @@ def setup end end + sub_test_case(".open with properties") do + data("path" => :path, "stream" => :stream) + test("row groups") do |source_type| + properties = Parquet::ReaderProperties.new + properties.buffer_size = 4096 + properties.enable_buffered_stream + assert_true(properties.buffered_stream_enabled?) + source = if source_type == :path + @file.path + else + Arrow::FileInputStream.new(@file.path) + end + begin + Parquet::ArrowFileReader.open(source, properties) do |reader| + properties.disable_buffered_stream + properties.buffer_size = 0 + assert_equal([ + Arrow::Table.new(@schema, [[true]]), + Arrow::Table.new(@schema, [[false]]) + ], + reader.each_row_group.to_a) + end + ensure + source.close if source_type == :stream + end + end + end + sub_test_case("#each_row_group") do test("block") do Arrow::FileInputStream.open(@file.path) do |input| From b39e1a5d142f57dd9848754f3f4dabb42f795abe Mon Sep 17 00:00:00 2001 From: Yifan Chen Date: Thu, 10 Sep 2026 00:46:34 -0700 Subject: [PATCH 2/5] GH-36010: [Ruby][Parquet] Release mapped reader in buffered test Free the reader after the .open block so Windows can remove the memory-mapped temporary file without waiting for garbage collection. Generated-by: OpenAI Codex --- ruby/red-parquet/test/test-arrow-file-reader.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ruby/red-parquet/test/test-arrow-file-reader.rb b/ruby/red-parquet/test/test-arrow-file-reader.rb index e11615a3b2ab..37488abfad66 100644 --- a/ruby/red-parquet/test/test-arrow-file-reader.rb +++ b/ruby/red-parquet/test/test-arrow-file-reader.rb @@ -41,8 +41,10 @@ def setup else Arrow::FileInputStream.new(@file.path) end + reader = nil begin - Parquet::ArrowFileReader.open(source, properties) do |reader| + Parquet::ArrowFileReader.open(source, properties) do |opened_reader| + reader = opened_reader properties.disable_buffered_stream properties.buffer_size = 0 assert_equal([ @@ -52,6 +54,8 @@ def setup reader.each_row_group.to_a) end ensure + # Release the memory-mapped path before Tempfile removes it on Windows. + reader&.unref source.close if source_type == :stream end end From bd192d6070a7324b260ecda3c3b0be28f415dbef Mon Sep 17 00:00:00 2001 From: Yifan Chen <30335308+emecii@users.noreply.github.com> Date: Wed, 23 Sep 2026 20:33:20 -0700 Subject: [PATCH 3/5] GH-36010: [GLib][Parquet] Address buffered reader API review Use full constructor names and retain the source through a construct-only GObject property. Separate property-copy and source-lifetime regressions from normal reading, and use the existing nonexistent-path terminology. Generated-by: OpenAI Codex --- c_glib/parquet-glib/arrow-file-reader.cpp | 71 +++++++++++++------ c_glib/parquet-glib/arrow-file-reader.h | 12 ++-- c_glib/test/parquet/test-arrow-file-reader.rb | 40 +++++++++-- 3 files changed, 87 insertions(+), 36 deletions(-) diff --git a/c_glib/parquet-glib/arrow-file-reader.cpp b/c_glib/parquet-glib/arrow-file-reader.cpp index 150baf2a5b76..0aa81453d60e 100644 --- a/c_glib/parquet-glib/arrow-file-reader.cpp +++ b/c_glib/parquet-glib/arrow-file-reader.cpp @@ -28,6 +28,7 @@ namespace { GParquetArrowFileReader * open_reader_with_properties(std::shared_ptr source, + GArrowSeekableInputStream *source_object, GParquetReaderProperties *properties, GError **error, const char *tag) @@ -48,7 +49,12 @@ namespace { if (!garrow::check(error, result, tag)) { return NULL; } - return gparquet_arrow_file_reader_new_raw(result->release()); + return GPARQUET_ARROW_FILE_READER(g_object_new(GPARQUET_TYPE_ARROW_FILE_READER, + "arrow-file-reader", + result->release(), + "source", + source_object, + NULL)); } } // namespace @@ -202,7 +208,8 @@ typedef struct GParquetArrowFileReaderPrivate_ enum { PROP_0, - PROP_ARROW_FILE_READER + PROP_ARROW_FILE_READER, + PROP_SOURCE }; G_DEFINE_TYPE_WITH_PRIVATE(GParquetArrowFileReader, @@ -240,6 +247,9 @@ gparquet_arrow_file_reader_set_property(GObject *object, auto priv = GPARQUET_ARROW_FILE_READER_GET_PRIVATE(object); switch (prop_id) { + case PROP_SOURCE: + priv->source = GARROW_SEEKABLE_INPUT_STREAM(g_value_dup_object(value)); + break; case PROP_ARROW_FILE_READER: priv->arrow_file_reader = static_cast(g_value_get_pointer(value)); @@ -256,7 +266,12 @@ gparquet_arrow_file_reader_get_property(GObject *object, GValue *value, GParamSpec *pspec) { + auto priv = GPARQUET_ARROW_FILE_READER_GET_PRIVATE(object); + switch (prop_id) { + case PROP_SOURCE: + g_value_set_object(value, priv->source); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); break; @@ -286,6 +301,21 @@ gparquet_arrow_file_reader_class_init(GParquetArrowFileReaderClass *klass) "The raw parquet::arrow::FileReader *", static_cast(G_PARAM_WRITABLE | G_PARAM_CONSTRUCT_ONLY)); g_object_class_install_property(gobject_class, PROP_ARROW_FILE_READER, spec); + + /** + * GParquetArrowFileReader:source: + * + * The source stream for this reader. + * + * Since: 26.0.0 + */ + spec = g_param_spec_object( + "source", + "Source", + "The source stream for this reader", + GARROW_TYPE_SEEKABLE_INPUT_STREAM, + static_cast(G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY)); + g_object_class_install_property(gobject_class, PROP_SOURCE, spec); } /** @@ -350,7 +380,7 @@ gparquet_arrow_file_reader_new_path(const gchar *path, GError **error) } /** - * gparquet_arrow_file_reader_new_arrow_with_properties: + * gparquet_arrow_file_reader_new_arrow_full: * @source: Arrow source to be read. * @properties: (nullable): Reader properties or %NULL for the defaults. * @error: (nullable): Return location for a #GError or %NULL. @@ -363,24 +393,19 @@ gparquet_arrow_file_reader_new_path(const gchar *path, GError **error) * Since: 26.0.0 */ GParquetArrowFileReader * -gparquet_arrow_file_reader_new_arrow_with_properties(GArrowSeekableInputStream *source, - GParquetReaderProperties *properties, - GError **error) -{ - auto reader = open_reader_with_properties( - garrow_seekable_input_stream_get_raw(source), - properties, - error, - "[parquet][arrow][file-reader][new-arrow-with-properties]"); - if (reader) { - auto priv = GPARQUET_ARROW_FILE_READER_GET_PRIVATE(reader); - priv->source = GARROW_SEEKABLE_INPUT_STREAM(g_object_ref(source)); - } - return reader; +gparquet_arrow_file_reader_new_arrow_full(GArrowSeekableInputStream *source, + GParquetReaderProperties *properties, + GError **error) +{ + return open_reader_with_properties(garrow_seekable_input_stream_get_raw(source), + source, + properties, + error, + "[parquet][arrow][file-reader][new-arrow-full]"); } /** - * gparquet_arrow_file_reader_new_path_with_properties: + * gparquet_arrow_file_reader_new_path_full: * @path: Path to be read. * @properties: (nullable): Reader properties or %NULL for the defaults. * @error: (nullable): Return location for a #GError or %NULL. @@ -394,16 +419,16 @@ gparquet_arrow_file_reader_new_arrow_with_properties(GArrowSeekableInputStream * * Since: 26.0.0 */ GParquetArrowFileReader * -gparquet_arrow_file_reader_new_path_with_properties(const gchar *path, - GParquetReaderProperties *properties, - GError **error) +gparquet_arrow_file_reader_new_path_full(const gchar *path, + GParquetReaderProperties *properties, + GError **error) { - const char *tag = "[parquet][arrow][file-reader][new-path-with-properties]"; + const char *tag = "[parquet][arrow][file-reader][new-path-full]"; auto source = arrow::io::MemoryMappedFile::Open(path, arrow::io::FileMode::READ); if (!garrow::check(error, source, tag)) { return NULL; } - return open_reader_with_properties(*source, properties, error, tag); + return open_reader_with_properties(*source, nullptr, properties, error, tag); } /** diff --git a/c_glib/parquet-glib/arrow-file-reader.h b/c_glib/parquet-glib/arrow-file-reader.h index 306f6204e533..ecb8592706ac 100644 --- a/c_glib/parquet-glib/arrow-file-reader.h +++ b/c_glib/parquet-glib/arrow-file-reader.h @@ -80,15 +80,15 @@ gparquet_arrow_file_reader_new_path(const gchar *path, GError **error); GPARQUET_AVAILABLE_IN_26_0 GParquetArrowFileReader * -gparquet_arrow_file_reader_new_arrow_with_properties(GArrowSeekableInputStream *source, - GParquetReaderProperties *properties, - GError **error); +gparquet_arrow_file_reader_new_arrow_full(GArrowSeekableInputStream *source, + GParquetReaderProperties *properties, + GError **error); GPARQUET_AVAILABLE_IN_26_0 GParquetArrowFileReader * -gparquet_arrow_file_reader_new_path_with_properties(const gchar *path, - GParquetReaderProperties *properties, - GError **error); +gparquet_arrow_file_reader_new_path_full(const gchar *path, + GParquetReaderProperties *properties, + GError **error); GPARQUET_AVAILABLE_IN_23_0 void diff --git a/c_glib/test/parquet/test-arrow-file-reader.rb b/c_glib/test/parquet/test-arrow-file-reader.rb index 304d3c14efb8..be5e3ddcc74c 100644 --- a/c_glib/test/parquet/test-arrow-file-reader.rb +++ b/c_glib/test/parquet/test-arrow-file-reader.rb @@ -52,15 +52,41 @@ def setup end reader = Parquet::ArrowFileReader.new(source, properties) begin - # The reader owns a copy of the properties and the native source. - properties.disable_buffered_stream - properties.buffer_size = 0 - properties.unref - source.unref if source_type == :stream assert_equal(@table, reader.read_table) assert_equal(build_table("a" => @a_array.slice(1, 1), "b" => @b_array.slice(1, 1)), reader.read_row_group(1)) + ensure + reader.close + reader.unref + source.unref if source_type == :stream + end + end + + test("copies properties") do + properties = Parquet::ReaderProperties.new + properties.enable_buffered_stream + properties.buffer_size = 4096 + reader = Parquet::ArrowFileReader.new(@file.path, properties) + begin + properties.disable_buffered_stream + properties.buffer_size = 0 + properties.unref + assert_equal(@table, reader.read_table) + ensure + reader.close + reader.unref + end + end + + test("retains source") do + properties = Parquet::ReaderProperties.new + source = Arrow::FileInputStream.new(@file.path) + reader = Parquet::ArrowFileReader.new(source, properties) + begin + assert_equal(source, reader.source) + source.unref + assert_equal(@table, reader.read_table) ensure reader.close reader.unref @@ -84,10 +110,10 @@ def setup end end - test("missing path") do + test("nonexistent path") do properties = Parquet::ReaderProperties.new assert_raise(Arrow::Error::Io) do - Parquet::ArrowFileReader.new("#{@file.path}.missing", properties) + Parquet::ArrowFileReader.new("#{@file.path}.nonexistent", properties) end end From 3b468be4fd22e12e89a28f904cb5794841c1239f Mon Sep 17 00:00:00 2001 From: Yifan Chen <30335308+emecii@users.noreply.github.com> Date: Thu, 24 Sep 2026 09:08:06 -0700 Subject: [PATCH 4/5] GH-36010: [GLib][Parquet] Make reader pre-buffering explicit Expose independent pre-buffer controls backed by ArrowReaderProperties. Reuse open_reader for both stream constructors and construct readers through new_raw with the retained source. Generated-by: OpenAI Codex --- c_glib/parquet-glib/arrow-file-reader.cpp | 112 ++++++++++++------ c_glib/parquet-glib/arrow-file-reader.h | 8 ++ c_glib/parquet-glib/arrow-file-reader.hpp | 5 + c_glib/test/parquet/test-arrow-file-reader.rb | 3 + c_glib/test/parquet/test-reader-properties.rb | 13 ++ .../test/test-arrow-file-reader.rb | 2 + 6 files changed, 107 insertions(+), 36 deletions(-) diff --git a/c_glib/parquet-glib/arrow-file-reader.cpp b/c_glib/parquet-glib/arrow-file-reader.cpp index 0aa81453d60e..ed719cafb767 100644 --- a/c_glib/parquet-glib/arrow-file-reader.cpp +++ b/c_glib/parquet-glib/arrow-file-reader.cpp @@ -27,11 +27,11 @@ namespace { GParquetArrowFileReader * - open_reader_with_properties(std::shared_ptr source, - GArrowSeekableInputStream *source_object, - GParquetReaderProperties *properties, - GError **error, - const char *tag) + open_reader(std::shared_ptr source, + GArrowSeekableInputStream *source_object, + GParquetReaderProperties *properties, + GError **error, + const char *tag) { auto parquet_properties = properties ? gparquet_reader_properties_get_raw(properties) : parquet::default_reader_properties(); @@ -39,22 +39,14 @@ namespace { if (!garrow::check(error, builder.Open(source, parquet_properties), tag)) { return NULL; } - if (parquet_properties.is_buffered_stream_enabled()) { - // Read-ahead would bypass the buffered stream by caching whole column chunks. - auto arrow_properties = parquet::default_arrow_reader_properties(); - arrow_properties.set_pre_buffer(false); - builder.properties(arrow_properties); + if (properties) { + builder.properties(gparquet_reader_properties_get_arrow_raw(properties)); } auto result = builder.Build(); if (!garrow::check(error, result, tag)) { return NULL; } - return GPARQUET_ARROW_FILE_READER(g_object_new(GPARQUET_TYPE_ARROW_FILE_READER, - "arrow-file-reader", - result->release(), - "source", - source_object, - NULL)); + return gparquet_arrow_file_reader_new_raw(result->release(), source_object); } } // namespace @@ -74,6 +66,7 @@ G_BEGIN_DECLS typedef struct GParquetReaderPropertiesPrivate_ { parquet::ReaderProperties properties; + parquet::ArrowReaderProperties arrow_properties; } GParquetReaderPropertiesPrivate; G_DEFINE_TYPE_WITH_PRIVATE(GParquetReaderProperties, @@ -88,6 +81,7 @@ static void gparquet_reader_properties_finalize(GObject *object) { auto priv = GPARQUET_READER_PROPERTIES_GET_PRIVATE(object); + priv->arrow_properties.~ArrowReaderProperties(); priv->properties.~ReaderProperties(); G_OBJECT_CLASS(gparquet_reader_properties_parent_class)->finalize(object); } @@ -97,6 +91,8 @@ gparquet_reader_properties_init(GParquetReaderProperties *object) { auto priv = GPARQUET_READER_PROPERTIES_GET_PRIVATE(object); new (&priv->properties) parquet::ReaderProperties(parquet::default_reader_properties()); + new (&priv->arrow_properties) + parquet::ArrowReaderProperties(parquet::default_arrow_reader_properties()); } static void @@ -122,8 +118,8 @@ gparquet_reader_properties_new(void) * gparquet_reader_properties_enable_buffered_stream: * @properties: A #GParquetReaderProperties. * - * Enable buffered stream reading. Readers constructed with these properties - * disable read-ahead of whole column chunks to use buffered streams instead. + * Enable buffered stream reading. To avoid pre-buffering whole column chunks, + * also call gparquet_reader_properties_set_pre_buffer() with %FALSE. * This does not impose a limit on the memory used by decoded data. * * Since: 26.0.0 @@ -200,6 +196,41 @@ gparquet_reader_properties_get_buffer_size(GParquetReaderProperties *properties) return priv->properties.buffer_size(); } +/** + * gparquet_reader_properties_set_pre_buffer: + * @properties: A #GParquetReaderProperties. + * @pre_buffer: Whether to pre-buffer column chunks. + * + * Set whether to pre-buffer column chunks to coalesce reads. This is enabled + * by default to improve performance on high-latency filesystems. Set this to + * %FALSE to use buffered streams without pre-buffering whole column chunks. + * This does not enable or disable buffered stream reading. + * + * Since: 26.0.0 + */ +void +gparquet_reader_properties_set_pre_buffer(GParquetReaderProperties *properties, + gboolean pre_buffer) +{ + auto priv = GPARQUET_READER_PROPERTIES_GET_PRIVATE(properties); + priv->arrow_properties.set_pre_buffer(pre_buffer); +} + +/** + * gparquet_reader_properties_get_pre_buffer: + * @properties: A #GParquetReaderProperties. + * + * Returns: %TRUE if pre-buffering is enabled, %FALSE otherwise. + * + * Since: 26.0.0 + */ +gboolean +gparquet_reader_properties_get_pre_buffer(GParquetReaderProperties *properties) +{ + auto priv = GPARQUET_READER_PROPERTIES_GET_PRIVATE(properties); + return priv->arrow_properties.pre_buffer(); +} + typedef struct GParquetArrowFileReaderPrivate_ { parquet::arrow::FileReader *arrow_file_reader; @@ -330,18 +361,11 @@ gparquet_arrow_file_reader_class_init(GParquetArrowFileReaderClass *klass) GParquetArrowFileReader * gparquet_arrow_file_reader_new_arrow(GArrowSeekableInputStream *source, GError **error) { - auto arrow_random_access_file = garrow_seekable_input_stream_get_raw(source); - auto arrow_memory_pool = arrow::default_memory_pool(); - auto parquet_arrow_file_reader_result = - parquet::arrow::OpenFile(arrow_random_access_file, arrow_memory_pool); - if (garrow::check(error, - parquet_arrow_file_reader_result, - "[parquet][arrow][file-reader][new-arrow]")) { - return gparquet_arrow_file_reader_new_raw( - parquet_arrow_file_reader_result->release()); - } else { - return NULL; - } + return open_reader(garrow_seekable_input_stream_get_raw(source), + source, + nullptr, + error, + "[parquet][arrow][file-reader][new-arrow]"); } /** @@ -397,11 +421,11 @@ gparquet_arrow_file_reader_new_arrow_full(GArrowSeekableInputStream *source, GParquetReaderProperties *properties, GError **error) { - return open_reader_with_properties(garrow_seekable_input_stream_get_raw(source), - source, - properties, - error, - "[parquet][arrow][file-reader][new-arrow-full]"); + return open_reader(garrow_seekable_input_stream_get_raw(source), + source, + properties, + error, + "[parquet][arrow][file-reader][new-arrow-full]"); } /** @@ -428,7 +452,7 @@ gparquet_arrow_file_reader_new_path_full(const gchar *path, if (!garrow::check(error, source, tag)) { return NULL; } - return open_reader_with_properties(*source, nullptr, properties, error, tag); + return open_reader(*source, nullptr, properties, error, tag); } /** @@ -661,11 +685,20 @@ G_END_DECLS GParquetArrowFileReader * gparquet_arrow_file_reader_new_raw(parquet::arrow::FileReader *parquet_arrow_file_reader) +{ + return gparquet_arrow_file_reader_new_raw(parquet_arrow_file_reader, nullptr); +} + +GParquetArrowFileReader * +gparquet_arrow_file_reader_new_raw(parquet::arrow::FileReader *parquet_arrow_file_reader, + GArrowSeekableInputStream *source) { auto arrow_file_reader = GPARQUET_ARROW_FILE_READER(g_object_new(GPARQUET_TYPE_ARROW_FILE_READER, "arrow-file-reader", parquet_arrow_file_reader, + "source", + source, NULL)); return arrow_file_reader; } @@ -683,3 +716,10 @@ gparquet_reader_properties_get_raw(GParquetReaderProperties *properties) auto priv = GPARQUET_READER_PROPERTIES_GET_PRIVATE(properties); return priv->properties; } + +parquet::ArrowReaderProperties +gparquet_reader_properties_get_arrow_raw(GParquetReaderProperties *properties) +{ + auto priv = GPARQUET_READER_PROPERTIES_GET_PRIVATE(properties); + return priv->arrow_properties; +} diff --git a/c_glib/parquet-glib/arrow-file-reader.h b/c_glib/parquet-glib/arrow-file-reader.h index ecb8592706ac..93fdba586959 100644 --- a/c_glib/parquet-glib/arrow-file-reader.h +++ b/c_glib/parquet-glib/arrow-file-reader.h @@ -58,6 +58,14 @@ GPARQUET_AVAILABLE_IN_26_0 gint64 gparquet_reader_properties_get_buffer_size(GParquetReaderProperties *properties); +GPARQUET_AVAILABLE_IN_26_0 +void +gparquet_reader_properties_set_pre_buffer(GParquetReaderProperties *properties, + gboolean pre_buffer); +GPARQUET_AVAILABLE_IN_26_0 +gboolean +gparquet_reader_properties_get_pre_buffer(GParquetReaderProperties *properties); + #define GPARQUET_TYPE_ARROW_FILE_READER (gparquet_arrow_file_reader_get_type()) GPARQUET_AVAILABLE_IN_0_11 G_DECLARE_DERIVABLE_TYPE(GParquetArrowFileReader, diff --git a/c_glib/parquet-glib/arrow-file-reader.hpp b/c_glib/parquet-glib/arrow-file-reader.hpp index c240e9f26953..afcfa606d1e0 100644 --- a/c_glib/parquet-glib/arrow-file-reader.hpp +++ b/c_glib/parquet-glib/arrow-file-reader.hpp @@ -25,7 +25,12 @@ GParquetArrowFileReader * gparquet_arrow_file_reader_new_raw(parquet::arrow::FileReader *parquet_arrow_file_reader); +GParquetArrowFileReader * +gparquet_arrow_file_reader_new_raw(parquet::arrow::FileReader *parquet_arrow_file_reader, + GArrowSeekableInputStream *source); parquet::arrow::FileReader * gparquet_arrow_file_reader_get_raw(GParquetArrowFileReader *arrow_file_reader); parquet::ReaderProperties gparquet_reader_properties_get_raw(GParquetReaderProperties *properties); +parquet::ArrowReaderProperties +gparquet_reader_properties_get_arrow_raw(GParquetReaderProperties *properties); diff --git a/c_glib/test/parquet/test-arrow-file-reader.rb b/c_glib/test/parquet/test-arrow-file-reader.rb index be5e3ddcc74c..90f9253ef0ce 100644 --- a/c_glib/test/parquet/test-arrow-file-reader.rb +++ b/c_glib/test/parquet/test-arrow-file-reader.rb @@ -44,6 +44,7 @@ def setup test("read") do |source_type| properties = Parquet::ReaderProperties.new properties.enable_buffered_stream + properties.pre_buffer = false properties.buffer_size = 4096 source = if source_type == :path @file.path @@ -66,9 +67,11 @@ def setup test("copies properties") do properties = Parquet::ReaderProperties.new properties.enable_buffered_stream + properties.pre_buffer = false properties.buffer_size = 4096 reader = Parquet::ArrowFileReader.new(@file.path, properties) begin + properties.pre_buffer = true properties.disable_buffered_stream properties.buffer_size = 0 properties.unref diff --git a/c_glib/test/parquet/test-reader-properties.rb b/c_glib/test/parquet/test-reader-properties.rb index 16d117339a5b..0e7a9dbfdd33 100644 --- a/c_glib/test/parquet/test-reader-properties.rb +++ b/c_glib/test/parquet/test-reader-properties.rb @@ -23,12 +23,25 @@ def setup def test_buffered_stream assert_false(@properties.buffered_stream_enabled?) + assert_true(@properties.pre_buffer?) @properties.enable_buffered_stream + assert_true(@properties.pre_buffer?) assert_true(@properties.buffered_stream_enabled?) + @properties.pre_buffer = false @properties.disable_buffered_stream + assert_false(@properties.pre_buffer?) assert_false(@properties.buffered_stream_enabled?) end + def test_pre_buffer + assert_true(@properties.pre_buffer?) + @properties.pre_buffer = false + assert_false(@properties.pre_buffer?) + assert_false(@properties.buffered_stream_enabled?) + @properties.pre_buffer = true + assert_true(@properties.pre_buffer?) + end + def test_buffer_size assert_equal(16 * 1024, @properties.buffer_size) @properties.buffer_size = 32 * 1024 diff --git a/ruby/red-parquet/test/test-arrow-file-reader.rb b/ruby/red-parquet/test/test-arrow-file-reader.rb index 37488abfad66..83980c0fbcd3 100644 --- a/ruby/red-parquet/test/test-arrow-file-reader.rb +++ b/ruby/red-parquet/test/test-arrow-file-reader.rb @@ -35,6 +35,7 @@ def setup properties = Parquet::ReaderProperties.new properties.buffer_size = 4096 properties.enable_buffered_stream + properties.pre_buffer = false assert_true(properties.buffered_stream_enabled?) source = if source_type == :path @file.path @@ -45,6 +46,7 @@ def setup begin Parquet::ArrowFileReader.open(source, properties) do |opened_reader| reader = opened_reader + properties.pre_buffer = true properties.disable_buffered_stream properties.buffer_size = 0 assert_equal([ From bce0231ffa5c6dce055669dc7b63c507a9152f95 Mon Sep 17 00:00:00 2001 From: Yifan Chen <30335308+emecii@users.noreply.github.com> Date: Sun, 27 Sep 2026 09:03:43 -0700 Subject: [PATCH 5/5] GH-36010: [GLib][Parquet] Simplify reader construction and tests Share the path constructor helper, return native property pointers, use the requested default source argument, and remove redundant tests following review. Generated-by: OpenAI Codex --- c_glib/parquet-glib/arrow-file-reader.cpp | 41 ++++----------- c_glib/parquet-glib/arrow-file-reader.hpp | 8 ++- c_glib/test/parquet/test-arrow-file-reader.rb | 51 +------------------ c_glib/test/parquet/test-reader-properties.rb | 30 ++++++----- .../test/test-arrow-file-reader.rb | 34 ------------- 5 files changed, 32 insertions(+), 132 deletions(-) diff --git a/c_glib/parquet-glib/arrow-file-reader.cpp b/c_glib/parquet-glib/arrow-file-reader.cpp index ed719cafb767..2db6b313f5b4 100644 --- a/c_glib/parquet-glib/arrow-file-reader.cpp +++ b/c_glib/parquet-glib/arrow-file-reader.cpp @@ -33,14 +33,14 @@ namespace { GError **error, const char *tag) { - auto parquet_properties = properties ? gparquet_reader_properties_get_raw(properties) + auto parquet_properties = properties ? *gparquet_reader_properties_get_raw(properties) : parquet::default_reader_properties(); parquet::arrow::FileReaderBuilder builder; if (!garrow::check(error, builder.Open(source, parquet_properties), tag)) { return NULL; } if (properties) { - builder.properties(gparquet_reader_properties_get_arrow_raw(properties)); + builder.properties(*gparquet_reader_properties_get_arrow_raw(properties)); } auto result = builder.Build(); if (!garrow::check(error, result, tag)) { @@ -380,27 +380,12 @@ gparquet_arrow_file_reader_new_arrow(GArrowSeekableInputStream *source, GError * GParquetArrowFileReader * gparquet_arrow_file_reader_new_path(const gchar *path, GError **error) { - auto arrow_memory_mapped_file = - arrow::io::MemoryMappedFile::Open(path, arrow::io::FileMode::READ); - if (!garrow::check(error, - arrow_memory_mapped_file, - "[parquet][arrow][file-reader][new-path]")) { - return NULL; - } - - std::shared_ptr arrow_random_access_file = - arrow_memory_mapped_file.ValueOrDie(); - auto arrow_memory_pool = arrow::default_memory_pool(); - auto parquet_arrow_file_reader_result = - parquet::arrow::OpenFile(arrow_random_access_file, arrow_memory_pool); - if (garrow::check(error, - parquet_arrow_file_reader_result, - "[parquet][arrow][file-reader][new-path]")) { - return gparquet_arrow_file_reader_new_raw( - parquet_arrow_file_reader_result->release()); - } else { + const char *tag = "[parquet][arrow][file-reader][new-path]"; + auto source = arrow::io::MemoryMappedFile::Open(path, arrow::io::FileMode::READ); + if (!garrow::check(error, source, tag)) { return NULL; } + return open_reader(*source, nullptr, nullptr, error, tag); } /** @@ -683,12 +668,6 @@ gparquet_arrow_file_reader_get_metadata(GParquetArrowFileReader *reader) G_END_DECLS -GParquetArrowFileReader * -gparquet_arrow_file_reader_new_raw(parquet::arrow::FileReader *parquet_arrow_file_reader) -{ - return gparquet_arrow_file_reader_new_raw(parquet_arrow_file_reader, nullptr); -} - GParquetArrowFileReader * gparquet_arrow_file_reader_new_raw(parquet::arrow::FileReader *parquet_arrow_file_reader, GArrowSeekableInputStream *source) @@ -710,16 +689,16 @@ gparquet_arrow_file_reader_get_raw(GParquetArrowFileReader *arrow_file_reader) return priv->arrow_file_reader; } -parquet::ReaderProperties +parquet::ReaderProperties * gparquet_reader_properties_get_raw(GParquetReaderProperties *properties) { auto priv = GPARQUET_READER_PROPERTIES_GET_PRIVATE(properties); - return priv->properties; + return &(priv->properties); } -parquet::ArrowReaderProperties +parquet::ArrowReaderProperties * gparquet_reader_properties_get_arrow_raw(GParquetReaderProperties *properties) { auto priv = GPARQUET_READER_PROPERTIES_GET_PRIVATE(properties); - return priv->arrow_properties; + return &(priv->arrow_properties); } diff --git a/c_glib/parquet-glib/arrow-file-reader.hpp b/c_glib/parquet-glib/arrow-file-reader.hpp index afcfa606d1e0..451ccbb7c829 100644 --- a/c_glib/parquet-glib/arrow-file-reader.hpp +++ b/c_glib/parquet-glib/arrow-file-reader.hpp @@ -23,14 +23,12 @@ #include -GParquetArrowFileReader * -gparquet_arrow_file_reader_new_raw(parquet::arrow::FileReader *parquet_arrow_file_reader); GParquetArrowFileReader * gparquet_arrow_file_reader_new_raw(parquet::arrow::FileReader *parquet_arrow_file_reader, - GArrowSeekableInputStream *source); + GArrowSeekableInputStream *source = nullptr); parquet::arrow::FileReader * gparquet_arrow_file_reader_get_raw(GParquetArrowFileReader *arrow_file_reader); -parquet::ReaderProperties +parquet::ReaderProperties * gparquet_reader_properties_get_raw(GParquetReaderProperties *properties); -parquet::ArrowReaderProperties +parquet::ArrowReaderProperties * gparquet_reader_properties_get_arrow_raw(GParquetReaderProperties *properties); diff --git a/c_glib/test/parquet/test-arrow-file-reader.rb b/c_glib/test/parquet/test-arrow-file-reader.rb index 90f9253ef0ce..a088f91520bd 100644 --- a/c_glib/test/parquet/test-arrow-file-reader.rb +++ b/c_glib/test/parquet/test-arrow-file-reader.rb @@ -39,7 +39,7 @@ def setup end end - sub_test_case(".new with properties") do + sub_test_case(".new(properties)") do data("path" => :path, "stream" => :stream) test("read") do |source_type| properties = Parquet::ReaderProperties.new @@ -64,55 +64,6 @@ def setup end end - test("copies properties") do - properties = Parquet::ReaderProperties.new - properties.enable_buffered_stream - properties.pre_buffer = false - properties.buffer_size = 4096 - reader = Parquet::ArrowFileReader.new(@file.path, properties) - begin - properties.pre_buffer = true - properties.disable_buffered_stream - properties.buffer_size = 0 - properties.unref - assert_equal(@table, reader.read_table) - ensure - reader.close - reader.unref - end - end - - test("retains source") do - properties = Parquet::ReaderProperties.new - source = Arrow::FileInputStream.new(@file.path) - reader = Parquet::ArrowFileReader.new(source, properties) - begin - assert_equal(source, reader.source) - source.unref - assert_equal(@table, reader.read_table) - ensure - reader.close - reader.unref - end - end - - data("path" => :path, "stream" => :stream) - test("default properties") do |source_type| - source = if source_type == :path - @file.path - else - Arrow::FileInputStream.new(@file.path) - end - reader = Parquet::ArrowFileReader.new(source, nil) - begin - assert_equal(@table, reader.read_table) - ensure - reader.close - reader.unref - source.unref if source_type == :stream - end - end - test("nonexistent path") do properties = Parquet::ReaderProperties.new assert_raise(Arrow::Error::Io) do diff --git a/c_glib/test/parquet/test-reader-properties.rb b/c_glib/test/parquet/test-reader-properties.rb index 0e7a9dbfdd33..f1d7598aa30b 100644 --- a/c_glib/test/parquet/test-reader-properties.rb +++ b/c_glib/test/parquet/test-reader-properties.rb @@ -22,30 +22,36 @@ def setup end def test_buffered_stream - assert_false(@properties.buffered_stream_enabled?) - assert_true(@properties.pre_buffer?) + assert do + not @properties.buffered_stream_enabled? + end @properties.enable_buffered_stream - assert_true(@properties.pre_buffer?) - assert_true(@properties.buffered_stream_enabled?) - @properties.pre_buffer = false + assert do + @properties.buffered_stream_enabled? + end @properties.disable_buffered_stream - assert_false(@properties.pre_buffer?) - assert_false(@properties.buffered_stream_enabled?) + assert do + not @properties.buffered_stream_enabled? + end end def test_pre_buffer - assert_true(@properties.pre_buffer?) + assert do + @properties.pre_buffer? + end @properties.pre_buffer = false - assert_false(@properties.pre_buffer?) - assert_false(@properties.buffered_stream_enabled?) + assert do + not @properties.pre_buffer? + end @properties.pre_buffer = true - assert_true(@properties.pre_buffer?) + assert do + @properties.pre_buffer? + end end def test_buffer_size assert_equal(16 * 1024, @properties.buffer_size) @properties.buffer_size = 32 * 1024 assert_equal(32 * 1024, @properties.buffer_size) - assert_false(@properties.buffered_stream_enabled?) end end diff --git a/ruby/red-parquet/test/test-arrow-file-reader.rb b/ruby/red-parquet/test/test-arrow-file-reader.rb index 83980c0fbcd3..61db74b899ef 100644 --- a/ruby/red-parquet/test/test-arrow-file-reader.rb +++ b/ruby/red-parquet/test/test-arrow-file-reader.rb @@ -29,40 +29,6 @@ def setup end end - sub_test_case(".open with properties") do - data("path" => :path, "stream" => :stream) - test("row groups") do |source_type| - properties = Parquet::ReaderProperties.new - properties.buffer_size = 4096 - properties.enable_buffered_stream - properties.pre_buffer = false - assert_true(properties.buffered_stream_enabled?) - source = if source_type == :path - @file.path - else - Arrow::FileInputStream.new(@file.path) - end - reader = nil - begin - Parquet::ArrowFileReader.open(source, properties) do |opened_reader| - reader = opened_reader - properties.pre_buffer = true - properties.disable_buffered_stream - properties.buffer_size = 0 - assert_equal([ - Arrow::Table.new(@schema, [[true]]), - Arrow::Table.new(@schema, [[false]]) - ], - reader.each_row_group.to_a) - end - ensure - # Release the memory-mapped path before Tempfile removes it on Windows. - reader&.unref - source.close if source_type == :stream - end - end - end - sub_test_case("#each_row_group") do test("block") do Arrow::FileInputStream.open(@file.path) do |input|