Skip to content

feat(ipc): Allow the stream reader's validation level to be configured #964

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
paleolimbot merged 2 commits into from
Oct 17, 2026
Merged

feat(ipc): Allow the stream reader's validation level to be configured #964

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9fbfb9e
feat(ipc): Allow the stream reader's validation level to be configured
rustyconover Oct 16, 2026
4fea478
Validate the validation level with a switch; suppress clang-tidy for …
rustyconover Oct 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions src/nanoarrow/ipc/reader.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ struct ArrowIpcArrayStreamReaderPrivate {
struct ArrowIpcInputStream input;
struct ArrowIpcDecoder decoder;
int use_shared_buffers;
enum ArrowValidationLevel validation_level;
struct ArrowSchema out_schema;
int64_t field_index;
struct ArrowBuffer header;
Expand Down Expand Up @@ -459,7 +460,7 @@ static int ArrowIpcArrayStreamReaderProcessRecordBatch(
ArrowSharedBufferInit(&shared, &private_data->body), &private_data->error);
ArrowErrorCode result = ArrowIpcDecoderDecodeArrayFromSharedWithDictionaries(
&private_data->decoder, &shared, private_data->field_index,
&private_data->dictionaries, out, NANOARROW_VALIDATION_LEVEL_FULL,
&private_data->dictionaries, out, private_data->validation_level,
&private_data->error);
ArrowBufferReset(&shared);
NANOARROW_RETURN_NOT_OK(result);
Expand All @@ -470,7 +471,7 @@ static int ArrowIpcArrayStreamReaderProcessRecordBatch(

NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderDecodeArrayWithDictionaries(
&private_data->decoder, body_view, private_data->field_index,
&private_data->dictionaries, out, NANOARROW_VALIDATION_LEVEL_FULL,
&private_data->dictionaries, out, private_data->validation_level,
&private_data->error));
}

Expand All @@ -488,7 +489,7 @@ static int ArrowIpcArrayStreamReaderProcessDictionary(
NANOARROW_RETURN_NOT_OK_WITH_ERROR(
ArrowSharedBufferInit(&shared, &private_data->body), &private_data->error);
int result = ArrowIpcDecoderDecodeDictionaryFromShared(
&private_data->decoder, &shared, NANOARROW_VALIDATION_LEVEL_FULL,
&private_data->decoder, &shared, private_data->validation_level,
&private_data->dictionaries, &private_data->error);
ArrowBufferReset(&shared);
NANOARROW_RETURN_NOT_OK(result);
Expand All @@ -497,7 +498,7 @@ static int ArrowIpcArrayStreamReaderProcessDictionary(
body_view.data.data = private_data->body.data;
body_view.size_bytes = private_data->body.size_bytes;
NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderDecodeDictionary(
&private_data->decoder, body_view, NANOARROW_VALIDATION_LEVEL_FULL,
&private_data->decoder, body_view, private_data->validation_level,
&private_data->dictionaries, &private_data->error));
}

Expand Down Expand Up @@ -573,6 +574,18 @@ static const char* ArrowIpcArrayStreamReaderGetLastError(
ArrowErrorCode ArrowIpcArrayStreamReaderInit(
struct ArrowArrayStream* out, struct ArrowIpcInputStream* input_stream,
struct ArrowIpcArrayStreamReaderOptions* options) {
if (options != NULL) {
switch (options->validation_level) {
case NANOARROW_VALIDATION_LEVEL_NONE:
case NANOARROW_VALIDATION_LEVEL_MINIMAL:
case NANOARROW_VALIDATION_LEVEL_DEFAULT:
case NANOARROW_VALIDATION_LEVEL_FULL:
break;
default:
return EINVAL;
}
}

struct ArrowIpcArrayStreamReaderPrivate* private_data =
(struct ArrowIpcArrayStreamReaderPrivate*)ArrowMalloc(
sizeof(struct ArrowIpcArrayStreamReaderPrivate));
Expand All @@ -596,9 +609,11 @@ ArrowErrorCode ArrowIpcArrayStreamReaderInit(
if (options != NULL) {
private_data->field_index = options->field_index;
private_data->use_shared_buffers = options->use_shared_buffers;
private_data->validation_level = options->validation_level;
} else {
private_data->field_index = -1;
private_data->use_shared_buffers = ArrowSharedBufferIsThreadSafe();
private_data->validation_level = NANOARROW_VALIDATION_LEVEL_FULL;
}

out->private_data = private_data;
Expand Down
69 changes: 69 additions & 0 deletions src/nanoarrow/ipc/reader_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@

#include <stdio.h>

#include <string>

#include "nanoarrow/nanoarrow_ipc.h"

static uint8_t kSimpleSchema[] = {
Expand Down Expand Up @@ -272,6 +274,7 @@ TEST(NanoarrowIpcReader, StreamReaderBasicNoSharedBuffers) {
struct ArrowIpcArrayStreamReaderOptions options;
options.field_index = -1;
options.use_shared_buffers = 0;
options.validation_level = NANOARROW_VALIDATION_LEVEL_FULL;
ASSERT_EQ(ArrowIpcArrayStreamReaderInit(&stream, &input, &options), NANOARROW_OK);

struct ArrowSchema schema;
Expand Down Expand Up @@ -495,6 +498,7 @@ TEST(NanoarrowIpcReader, StreamReaderUnsupportedFieldIndex) {
struct ArrowIpcArrayStreamReaderOptions options;
options.field_index = 0;
options.use_shared_buffers = 0;
options.validation_level = NANOARROW_VALIDATION_LEVEL_FULL;
ASSERT_EQ(ArrowIpcArrayStreamReaderInit(&stream, &input, &options), NANOARROW_OK);

struct ArrowSchema schema;
Expand Down Expand Up @@ -586,6 +590,71 @@ TEST(NanoarrowIpcReader, StreamReaderDictionary) {
ArrowArrayStreamRelease(&stream);
}

TEST(NanoarrowIpcReader, StreamReaderValidationLevel) {
// Point the last dictionary index past the end of the three-value dictionary.
// Only full validation inspects index values, so a lower level accepts the batch.
uint8_t record_batch[sizeof(kDictionaryRecordBatch)];
memcpy(record_batch, kDictionaryRecordBatch, sizeof(kDictionaryRecordBatch));
ASSERT_EQ(record_batch[sizeof(record_batch) - 6], 0x02);
record_batch[sizeof(record_batch) - 6] = 0x07;

for (auto validation_level :
{NANOARROW_VALIDATION_LEVEL_FULL, NANOARROW_VALIDATION_LEVEL_DEFAULT}) {
SCOPED_TRACE(validation_level);
struct ArrowBuffer input_buffer;
ArrowBufferInit(&input_buffer);
ASSERT_EQ(
ArrowBufferAppend(&input_buffer, kDictionarySchema, sizeof(kDictionarySchema)),
NANOARROW_OK);
ASSERT_EQ(
ArrowBufferAppend(&input_buffer, kDictionaryBatch, sizeof(kDictionaryBatch)),
NANOARROW_OK);
ASSERT_EQ(ArrowBufferAppend(&input_buffer, record_batch, sizeof(record_batch)),
NANOARROW_OK);

struct ArrowIpcInputStream input;
ASSERT_EQ(ArrowIpcInputStreamInitBuffer(&input, &input_buffer), NANOARROW_OK);

struct ArrowArrayStream stream;
struct ArrowIpcArrayStreamReaderOptions options;
options.field_index = -1;
options.use_shared_buffers = ArrowSharedBufferIsThreadSafe();
options.validation_level = validation_level;
ASSERT_EQ(ArrowIpcArrayStreamReaderInit(&stream, &input, &options), NANOARROW_OK);

struct ArrowArray array;
if (validation_level == NANOARROW_VALIDATION_LEVEL_FULL) {
ASSERT_EQ(ArrowArrayStreamGetNext(&stream, &array, nullptr), EINVAL);
EXPECT_NE(std::string(ArrowArrayStreamGetLastError(&stream))
.find("Expected dictionary index >= 0 and < 3 but found value 7"),
std::string::npos);
} else {
ASSERT_EQ(ArrowArrayStreamGetNext(&stream, &array, nullptr), NANOARROW_OK);
EXPECT_EQ(array.length, 3);
ArrowArrayRelease(&array);
}

ArrowArrayStreamRelease(&stream);
}
}

TEST(NanoarrowIpcReader, StreamReaderInvalidValidationLevel) {
struct ArrowBuffer input_buffer;
ArrowBufferInit(&input_buffer);
struct ArrowIpcInputStream input;
ASSERT_EQ(ArrowIpcInputStreamInitBuffer(&input, &input_buffer), NANOARROW_OK);

struct ArrowArrayStream stream;
struct ArrowIpcArrayStreamReaderOptions options;
options.field_index = -1;
options.use_shared_buffers = 0;
// NOLINTNEXTLINE(clang-analyzer-optin.core.EnumCastOutOfRange): deliberately invalid
options.validation_level = static_cast<ArrowValidationLevel>(4);
ASSERT_EQ(ArrowIpcArrayStreamReaderInit(&stream, &input, &options), EINVAL);
Comment on lines +648 to +653

@paleolimbot paleolimbot Oct 16, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

clang-tidy objects to 4 here since it's an invalid value. I think you can add a clang-tidy ignore line, but if not, you can also remove this test.

@rustyconover rustyconover Oct 16, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added a NOLINTNEXTLINE(clang-analyzer-optin.core.EnumCastOutOfRange) so the test can keep exercising the EINVAL path. I checked with clang-tidy 19: the warning shows up without the comment and is gone with it.


input.release(&input);
}

TEST(NanoarrowIpcReader, StreamReaderDictionaryBatchWithoutDictionarySchema) {
// Send a dictionary batch when the schema has no dictionaries
struct ArrowBuffer input_buffer;
Expand Down
8 changes: 8 additions & 0 deletions src/nanoarrow/nanoarrow_ipc.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -936,6 +936,14 @@ struct ArrowIpcArrayStreamReaderOptions {
/// a single field there is probably no advantage to using shared buffers.
/// Defaults to the value of ArrowSharedBufferIsThreadSafe().
int use_shared_buffers;

/// \brief The level of validation to apply to each decoded array
///
/// Defaults to NANOARROW_VALIDATION_LEVEL_FULL, which is appropriate for untrusted
/// input. Use a lower level (e.g., NANOARROW_VALIDATION_LEVEL_DEFAULT) to skip checks
/// that are O(n) in the length of the decoded buffers when the input comes from a
/// trusted producer.
enum ArrowValidationLevel validation_level;
};

/// \brief Initialize an ArrowArrayStream from an input stream of bytes
Expand Down
Loading