diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 9826b7591..8eb91fe52 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -1246,8 +1246,14 @@ static int ArrowIpcSetDictionaryEncoding( error); NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowSchemaSetName(schema->dictionary, ""), error); - NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderSetTypeInt( - schema, ns(DictionaryEncoding_indexType_get(dictionary_encoding)), error)); + // Schema.fbs: "If this field is null, the indices must be signed int32." + if (ns(DictionaryEncoding_indexType_is_present(dictionary_encoding))) { + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderSetTypeInt( + schema, ns(DictionaryEncoding_indexType_get(dictionary_encoding)), error)); + } else { + NANOARROW_RETURN_NOT_OK( + ArrowIpcDecoderSetTypeSimple(schema, NANOARROW_TYPE_INT32, error)); + } if (ns(DictionaryEncoding_isOrdered_get(dictionary_encoding))) { schema->flags |= ARROW_FLAG_DICTIONARY_ORDERED; diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index dcdde1761..b2cbaf7dc 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -195,6 +195,23 @@ alignas(8) static uint8_t kDictionarySchema[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +// Schema message with one field, some_col: dictionary, +// written by pyarrow 25.0.1 with DictionaryEncoding.indexType removed (null means +// signed int32 per Schema.fbs) +alignas(8) static uint8_t kDictionarySchemaNoIndexType[] = { + 0xff, 0xff, 0xff, 0xff, 0xa0, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0a, 0x00, 0x0c, 0x00, 0x06, 0x00, 0x05, 0x00, 0x08, 0x00, 0x0a, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00, 0xb4, 0xff, 0xff, 0xff, 0x04, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x10, 0x00, 0x18, 0x00, + 0x08, 0x00, 0x06, 0x00, 0x07, 0x00, 0x0c, 0x00, 0x10, 0x00, 0x14, 0x00, 0x10, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x05, 0x14, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, + 0x24, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, + 0x00, 0x00, 0x73, 0x6f, 0x6d, 0x65, 0x5f, 0x63, 0x6f, 0x6c, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0xd8, 0xff, 0xff, 0xff, 0x0c, 0x00, + 0x00, 0x00, 0x08, 0x00, 0x0c, 0x00, 0x08, 0x00, 0x07, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x20, 0x00, 0x00, 0x00, 0x04, 0x00, 0x04, 0x00, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00}; + alignas(8) static uint8_t kDictionaryBatch[] = { 0xff, 0xff, 0xff, 0xff, 0xa8, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x14, 0x00, 0x06, 0x00, 0x05, 0x00, 0x08, 0x00, 0x0c, 0x00, @@ -664,6 +681,33 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema) { ArrowIpcDecoderReset(&decoder); } +TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchemaWithoutIndexType) { + struct ArrowIpcDecoder decoder; + struct ArrowError error; + struct ArrowSchema schema; + + struct ArrowBufferView data; + data.data.as_uint8 = kDictionarySchemaNoIndexType; + data.size_bytes = sizeof(kDictionarySchemaNoIndexType); + + ASSERT_EQ(ArrowIpcDecoderInit(&decoder), NANOARROW_OK); + + ASSERT_EQ(ArrowIpcDecoderDecodeHeader(&decoder, data, &error), NANOARROW_OK) + << error.message; + ASSERT_EQ(decoder.message_type, NANOARROW_IPC_MESSAGE_TYPE_SCHEMA); + + ASSERT_EQ(ArrowIpcDecoderDecodeSchema(&decoder, &schema, &error), NANOARROW_OK) + << error.message; + ASSERT_EQ(schema.n_children, 1); + EXPECT_STREQ(schema.children[0]->name, "some_col"); + EXPECT_STREQ(schema.children[0]->format, "i"); + ASSERT_NE(schema.children[0]->dictionary, nullptr); + EXPECT_STREQ(schema.children[0]->dictionary->format, "u"); + + ArrowSchemaRelease(&schema); + ArrowIpcDecoderReset(&decoder); +} + TEST(NanoarrowIpcTest, NanoarrowIpcDictionaryEncodingsUniqueIds) { // Create dictionary encodings there is at least one duplicated identifier struct ArrowIpcDictionaryEncodings dictionary_encodings;