From af4ef3076f5cbe4d9159fb54321be0b431da6633 Mon Sep 17 00:00:00 2001 From: Animesh Kumar Date: Tue, 6 Oct 2026 04:10:51 +0530 Subject: [PATCH] fix(ipc): Decode a DictionaryEncoding without indexType as int32 indices Schema.fbs says that a null DictionaryEncoding.indexType means signed int32 indices, but the decoder passed the missing table to ArrowIpcDecoderSetTypeInt() and crashed. Use int32 when the field is absent. Closes #956 --- src/nanoarrow/ipc/decoder.c | 10 +++++-- src/nanoarrow/ipc/decoder_test.cc | 44 +++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 9826b7591..8eb91fe52 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -1246,8 +1246,14 @@ static int ArrowIpcSetDictionaryEncoding( error); NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowSchemaSetName(schema->dictionary, ""), error); - NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderSetTypeInt( - schema, ns(DictionaryEncoding_indexType_get(dictionary_encoding)), error)); + // Schema.fbs: "If this field is null, the indices must be signed int32." + if (ns(DictionaryEncoding_indexType_is_present(dictionary_encoding))) { + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderSetTypeInt( + schema, ns(DictionaryEncoding_indexType_get(dictionary_encoding)), error)); + } else { + NANOARROW_RETURN_NOT_OK( + ArrowIpcDecoderSetTypeSimple(schema, NANOARROW_TYPE_INT32, error)); + } if (ns(DictionaryEncoding_isOrdered_get(dictionary_encoding))) { schema->flags |= ARROW_FLAG_DICTIONARY_ORDERED; diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index dcdde1761..b2cbaf7dc 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -195,6 +195,23 @@ alignas(8) static uint8_t kDictionarySchema[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +// Schema message with one field, some_col: dictionary, +// written by pyarrow 25.0.1 with DictionaryEncoding.indexType removed (null means +// signed int32 per Schema.fbs) +alignas(8) static uint8_t kDictionarySchemaNoIndexType[] = { + 0xff, 0xff, 0xff, 0xff, 0xa0, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0a, 0x00, 0x0c, 0x00, 0x06, 0x00, 0x05, 0x00, 0x08, 0x00, 0x0a, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00, 0xb4, 0xff, 0xff, 0xff, 0x04, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x10, 0x00, 0x18, 0x00, + 0x08, 0x00, 0x06, 0x00, 0x07, 0x00, 0x0c, 0x00, 0x10, 0x00, 0x14, 0x00, 0x10, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x05, 0x14, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, + 0x24, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, + 0x00, 0x00, 0x73, 0x6f, 0x6d, 0x65, 0x5f, 0x63, 0x6f, 0x6c, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0xd8, 0xff, 0xff, 0xff, 0x0c, 0x00, + 0x00, 0x00, 0x08, 0x00, 0x0c, 0x00, 0x08, 0x00, 0x07, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x20, 0x00, 0x00, 0x00, 0x04, 0x00, 0x04, 0x00, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00}; + alignas(8) static uint8_t kDictionaryBatch[] = { 0xff, 0xff, 0xff, 0xff, 0xa8, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x14, 0x00, 0x06, 0x00, 0x05, 0x00, 0x08, 0x00, 0x0c, 0x00, @@ -664,6 +681,33 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema) { ArrowIpcDecoderReset(&decoder); } +TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchemaWithoutIndexType) { + struct ArrowIpcDecoder decoder; + struct ArrowError error; + struct ArrowSchema schema; + + struct ArrowBufferView data; + data.data.as_uint8 = kDictionarySchemaNoIndexType; + data.size_bytes = sizeof(kDictionarySchemaNoIndexType); + + ASSERT_EQ(ArrowIpcDecoderInit(&decoder), NANOARROW_OK); + + ASSERT_EQ(ArrowIpcDecoderDecodeHeader(&decoder, data, &error), NANOARROW_OK) + << error.message; + ASSERT_EQ(decoder.message_type, NANOARROW_IPC_MESSAGE_TYPE_SCHEMA); + + ASSERT_EQ(ArrowIpcDecoderDecodeSchema(&decoder, &schema, &error), NANOARROW_OK) + << error.message; + ASSERT_EQ(schema.n_children, 1); + EXPECT_STREQ(schema.children[0]->name, "some_col"); + EXPECT_STREQ(schema.children[0]->format, "i"); + ASSERT_NE(schema.children[0]->dictionary, nullptr); + EXPECT_STREQ(schema.children[0]->dictionary->format, "u"); + + ArrowSchemaRelease(&schema); + ArrowIpcDecoderReset(&decoder); +} + TEST(NanoarrowIpcTest, NanoarrowIpcDictionaryEncodingsUniqueIds) { // Create dictionary encodings there is at least one duplicated identifier struct ArrowIpcDictionaryEncodings dictionary_encodings;