From 9fbfb9e72fec392d2ac458d35c3e370b6c66fa40 Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Thu, 15 Oct 2026 20:55:54 -0400 Subject: [PATCH 1/2] feat(ipc): Allow the stream reader's validation level to be configured ArrowIpcArrayStreamReaderOptions gains a validation_level field, used for every record batch and dictionary batch the reader decodes. Passing NULL options keeps the current behaviour (NANOARROW_VALIDATION_LEVEL_FULL). Co-Authored-By: Claude Opus 5.5 (1M context) --- src/nanoarrow/ipc/reader.c | 17 ++++++-- src/nanoarrow/ipc/reader_test.cc | 68 ++++++++++++++++++++++++++++++++ src/nanoarrow/nanoarrow_ipc.h | 8 ++++ 3 files changed, 89 insertions(+), 4 deletions(-) diff --git a/src/nanoarrow/ipc/reader.c b/src/nanoarrow/ipc/reader.c index c699ca331..71e9a1196 100644 --- a/src/nanoarrow/ipc/reader.c +++ b/src/nanoarrow/ipc/reader.c @@ -187,6 +187,7 @@ struct ArrowIpcArrayStreamReaderPrivate { struct ArrowIpcInputStream input; struct ArrowIpcDecoder decoder; int use_shared_buffers; + enum ArrowValidationLevel validation_level; struct ArrowSchema out_schema; int64_t field_index; struct ArrowBuffer header; @@ -459,7 +460,7 @@ static int ArrowIpcArrayStreamReaderProcessRecordBatch( ArrowSharedBufferInit(&shared, &private_data->body), &private_data->error); ArrowErrorCode result = ArrowIpcDecoderDecodeArrayFromSharedWithDictionaries( &private_data->decoder, &shared, private_data->field_index, - &private_data->dictionaries, out, NANOARROW_VALIDATION_LEVEL_FULL, + &private_data->dictionaries, out, private_data->validation_level, &private_data->error); ArrowBufferReset(&shared); NANOARROW_RETURN_NOT_OK(result); @@ -470,7 +471,7 @@ static int ArrowIpcArrayStreamReaderProcessRecordBatch( NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderDecodeArrayWithDictionaries( &private_data->decoder, body_view, private_data->field_index, - &private_data->dictionaries, out, NANOARROW_VALIDATION_LEVEL_FULL, + &private_data->dictionaries, out, private_data->validation_level, &private_data->error)); } @@ -488,7 +489,7 @@ static int ArrowIpcArrayStreamReaderProcessDictionary( NANOARROW_RETURN_NOT_OK_WITH_ERROR( ArrowSharedBufferInit(&shared, &private_data->body), &private_data->error); int result = ArrowIpcDecoderDecodeDictionaryFromShared( - &private_data->decoder, &shared, NANOARROW_VALIDATION_LEVEL_FULL, + &private_data->decoder, &shared, private_data->validation_level, &private_data->dictionaries, &private_data->error); ArrowBufferReset(&shared); NANOARROW_RETURN_NOT_OK(result); @@ -497,7 +498,7 @@ static int ArrowIpcArrayStreamReaderProcessDictionary( body_view.data.data = private_data->body.data; body_view.size_bytes = private_data->body.size_bytes; NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderDecodeDictionary( - &private_data->decoder, body_view, NANOARROW_VALIDATION_LEVEL_FULL, + &private_data->decoder, body_view, private_data->validation_level, &private_data->dictionaries, &private_data->error)); } @@ -573,6 +574,12 @@ static const char* ArrowIpcArrayStreamReaderGetLastError( ArrowErrorCode ArrowIpcArrayStreamReaderInit( struct ArrowArrayStream* out, struct ArrowIpcInputStream* input_stream, struct ArrowIpcArrayStreamReaderOptions* options) { + if (options != NULL && + ((int)options->validation_level < (int)NANOARROW_VALIDATION_LEVEL_NONE || + (int)options->validation_level > (int)NANOARROW_VALIDATION_LEVEL_FULL)) { + return EINVAL; + } + struct ArrowIpcArrayStreamReaderPrivate* private_data = (struct ArrowIpcArrayStreamReaderPrivate*)ArrowMalloc( sizeof(struct ArrowIpcArrayStreamReaderPrivate)); @@ -596,9 +603,11 @@ ArrowErrorCode ArrowIpcArrayStreamReaderInit( if (options != NULL) { private_data->field_index = options->field_index; private_data->use_shared_buffers = options->use_shared_buffers; + private_data->validation_level = options->validation_level; } else { private_data->field_index = -1; private_data->use_shared_buffers = ArrowSharedBufferIsThreadSafe(); + private_data->validation_level = NANOARROW_VALIDATION_LEVEL_FULL; } out->private_data = private_data; diff --git a/src/nanoarrow/ipc/reader_test.cc b/src/nanoarrow/ipc/reader_test.cc index d257e111f..da5887769 100644 --- a/src/nanoarrow/ipc/reader_test.cc +++ b/src/nanoarrow/ipc/reader_test.cc @@ -19,6 +19,8 @@ #include +#include + #include "nanoarrow/nanoarrow_ipc.h" static uint8_t kSimpleSchema[] = { @@ -272,6 +274,7 @@ TEST(NanoarrowIpcReader, StreamReaderBasicNoSharedBuffers) { struct ArrowIpcArrayStreamReaderOptions options; options.field_index = -1; options.use_shared_buffers = 0; + options.validation_level = NANOARROW_VALIDATION_LEVEL_FULL; ASSERT_EQ(ArrowIpcArrayStreamReaderInit(&stream, &input, &options), NANOARROW_OK); struct ArrowSchema schema; @@ -495,6 +498,7 @@ TEST(NanoarrowIpcReader, StreamReaderUnsupportedFieldIndex) { struct ArrowIpcArrayStreamReaderOptions options; options.field_index = 0; options.use_shared_buffers = 0; + options.validation_level = NANOARROW_VALIDATION_LEVEL_FULL; ASSERT_EQ(ArrowIpcArrayStreamReaderInit(&stream, &input, &options), NANOARROW_OK); struct ArrowSchema schema; @@ -586,6 +590,70 @@ TEST(NanoarrowIpcReader, StreamReaderDictionary) { ArrowArrayStreamRelease(&stream); } +TEST(NanoarrowIpcReader, StreamReaderValidationLevel) { + // Point the last dictionary index past the end of the three-value dictionary. + // Only full validation inspects index values, so a lower level accepts the batch. + uint8_t record_batch[sizeof(kDictionaryRecordBatch)]; + memcpy(record_batch, kDictionaryRecordBatch, sizeof(kDictionaryRecordBatch)); + ASSERT_EQ(record_batch[sizeof(record_batch) - 6], 0x02); + record_batch[sizeof(record_batch) - 6] = 0x07; + + for (auto validation_level : + {NANOARROW_VALIDATION_LEVEL_FULL, NANOARROW_VALIDATION_LEVEL_DEFAULT}) { + SCOPED_TRACE(validation_level); + struct ArrowBuffer input_buffer; + ArrowBufferInit(&input_buffer); + ASSERT_EQ( + ArrowBufferAppend(&input_buffer, kDictionarySchema, sizeof(kDictionarySchema)), + NANOARROW_OK); + ASSERT_EQ( + ArrowBufferAppend(&input_buffer, kDictionaryBatch, sizeof(kDictionaryBatch)), + NANOARROW_OK); + ASSERT_EQ(ArrowBufferAppend(&input_buffer, record_batch, sizeof(record_batch)), + NANOARROW_OK); + + struct ArrowIpcInputStream input; + ASSERT_EQ(ArrowIpcInputStreamInitBuffer(&input, &input_buffer), NANOARROW_OK); + + struct ArrowArrayStream stream; + struct ArrowIpcArrayStreamReaderOptions options; + options.field_index = -1; + options.use_shared_buffers = ArrowSharedBufferIsThreadSafe(); + options.validation_level = validation_level; + ASSERT_EQ(ArrowIpcArrayStreamReaderInit(&stream, &input, &options), NANOARROW_OK); + + struct ArrowArray array; + if (validation_level == NANOARROW_VALIDATION_LEVEL_FULL) { + ASSERT_EQ(ArrowArrayStreamGetNext(&stream, &array, nullptr), EINVAL); + EXPECT_NE(std::string(ArrowArrayStreamGetLastError(&stream)) + .find("Expected dictionary index >= 0 and < 3 but found value 7"), + std::string::npos); + } else { + ASSERT_EQ(ArrowArrayStreamGetNext(&stream, &array, nullptr), NANOARROW_OK); + EXPECT_EQ(array.length, 3); + ArrowArrayRelease(&array); + } + + ArrowArrayStreamRelease(&stream); + } +} + +TEST(NanoarrowIpcReader, StreamReaderInvalidValidationLevel) { + struct ArrowBuffer input_buffer; + ArrowBufferInit(&input_buffer); + struct ArrowIpcInputStream input; + ASSERT_EQ(ArrowIpcInputStreamInitBuffer(&input, &input_buffer), NANOARROW_OK); + + struct ArrowArrayStream stream; + struct ArrowIpcArrayStreamReaderOptions options; + options.field_index = -1; + options.use_shared_buffers = 0; + options.validation_level = static_cast(4); + ASSERT_EQ(ArrowIpcArrayStreamReaderInit(&stream, &input, &options), EINVAL); + + input.release(&input); +} + TEST(NanoarrowIpcReader, StreamReaderDictionaryBatchWithoutDictionarySchema) { // Send a dictionary batch when the schema has no dictionaries struct ArrowBuffer input_buffer; diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index 7f5559759..38b57f97f 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -936,6 +936,14 @@ struct ArrowIpcArrayStreamReaderOptions { /// a single field there is probably no advantage to using shared buffers. /// Defaults to the value of ArrowSharedBufferIsThreadSafe(). int use_shared_buffers; + + /// \brief The level of validation to apply to each decoded array + /// + /// Defaults to NANOARROW_VALIDATION_LEVEL_FULL, which is appropriate for untrusted + /// input. Use a lower level (e.g., NANOARROW_VALIDATION_LEVEL_DEFAULT) to skip checks + /// that are O(n) in the length of the decoded buffers when the input comes from a + /// trusted producer. + enum ArrowValidationLevel validation_level; }; /// \brief Initialize an ArrowArrayStream from an input stream of bytes From 4fea4789f2c63edd8e0672c467f402c15f0de04b Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Fri, 16 Oct 2026 11:56:52 -0400 Subject: [PATCH 2/2] Validate the validation level with a switch; suppress clang-tidy for the invalid-value test Co-Authored-By: Claude Opus 5.5 (1M context) --- src/nanoarrow/ipc/reader.c | 14 ++++++++++---- src/nanoarrow/ipc/reader_test.cc | 1 + 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/nanoarrow/ipc/reader.c b/src/nanoarrow/ipc/reader.c index 71e9a1196..3348a9b60 100644 --- a/src/nanoarrow/ipc/reader.c +++ b/src/nanoarrow/ipc/reader.c @@ -574,10 +574,16 @@ static const char* ArrowIpcArrayStreamReaderGetLastError( ArrowErrorCode ArrowIpcArrayStreamReaderInit( struct ArrowArrayStream* out, struct ArrowIpcInputStream* input_stream, struct ArrowIpcArrayStreamReaderOptions* options) { - if (options != NULL && - ((int)options->validation_level < (int)NANOARROW_VALIDATION_LEVEL_NONE || - (int)options->validation_level > (int)NANOARROW_VALIDATION_LEVEL_FULL)) { - return EINVAL; + if (options != NULL) { + switch (options->validation_level) { + case NANOARROW_VALIDATION_LEVEL_NONE: + case NANOARROW_VALIDATION_LEVEL_MINIMAL: + case NANOARROW_VALIDATION_LEVEL_DEFAULT: + case NANOARROW_VALIDATION_LEVEL_FULL: + break; + default: + return EINVAL; + } } struct ArrowIpcArrayStreamReaderPrivate* private_data = diff --git a/src/nanoarrow/ipc/reader_test.cc b/src/nanoarrow/ipc/reader_test.cc index da5887769..13545a691 100644 --- a/src/nanoarrow/ipc/reader_test.cc +++ b/src/nanoarrow/ipc/reader_test.cc @@ -648,6 +648,7 @@ TEST(NanoarrowIpcReader, StreamReaderInvalidValidationLevel) { struct ArrowIpcArrayStreamReaderOptions options; options.field_index = -1; options.use_shared_buffers = 0; + // NOLINTNEXTLINE(clang-analyzer-optin.core.EnumCastOutOfRange): deliberately invalid options.validation_level = static_cast(4); ASSERT_EQ(ArrowIpcArrayStreamReaderInit(&stream, &input, &options), EINVAL);