diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 80e53ec8d..c722c2d52 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -1275,9 +1275,11 @@ static int ArrowIpcDecoderSetType(struct ArrowSchema* schema, ns(Field_table_t) } // When decoding dictionaries, we move the value type to the schema->dictionary -// member, but we need to move the field metadata back because in IPC there -// is no such thing as dictionary metadata (even extension metadata) -// https://github.com/apache/arrow/issues/49704 +// member, but we need to move the field metadata back because Field.custom_metadata +// is the metadata of the field (i.e., the dictionary-encoded ArrowSchema). The +// metadata of the dictionary value type is DictionaryEncoding.custom_metadata +// (format version 1.6, https://github.com/apache/arrow/issues/49704), which is +// set on schema->dictionary separately. static int ArrowIpcMoveDictionaryMetadataBackToField(struct ArrowSchema* schema) { NANOARROW_DCHECK(schema->dictionary != NULL); NANOARROW_RETURN_NOT_OK(ArrowSchemaSetMetadata(schema, schema->dictionary->metadata)); @@ -1338,6 +1340,13 @@ static int ArrowIpcSetDictionaryEncoding( NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowIpcMoveDictionaryMetadataBackToField(schema), error); + // The metadata of the dictionary value type (e.g., the extension type of the values), + // which is absent if the writer predates format version 1.6 or the value type has no + // metadata. + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderSetMetadata( + schema->dictionary, ns(DictionaryEncoding_custom_metadata(dictionary_encoding)), + error)); + // Track the identifier if we have a dictionaries object in which to track it if (dictionaries != NULL) { int64_t id = ns(DictionaryEncoding_id(dictionary_encoding)); diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index c7ac78ce5..7a955c6b4 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -1314,6 +1314,48 @@ TEST(NanoarrowIpcTest, NanoarrowIpcFooterDecodingErrors) { } #if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) +static bool IsDictionaryOfExtension(const std::shared_ptr& data_type) { + return data_type->id() == arrow::Type::DICTIONARY && + std::static_pointer_cast(data_type)->value_type()->id() == + arrow::Type::EXTENSION; +} + +static bool HasDictionaryOfExtension(const arrow::FieldVector& fields) { + for (const auto& field : fields) { + if (IsDictionaryOfExtension(field->type()) || + HasDictionaryOfExtension(field->type()->fields())) { + return true; + } + } + + return false; +} + +static bool HasDictionaryOfExtension(const arrow::Schema& schema) { + return HasDictionaryOfExtension(schema.fields()); +} + +// Arrow C++ writes the metadata of a dictionary value type (e.g., the extension type +// of the values) to DictionaryEncoding.custom_metadata from format version 1.6 +// (https://github.com/apache/arrow/issues/49704). Earlier versions write it to +// Field.custom_metadata, where it can't be distinguished from the metadata of the +// field itself and Arrow C++ can't read it back. +static bool ArrowCppWritesDictionaryValueMetadata() { + auto dummy_schema = arrow::schema( + {arrow::field("", arrow::dictionary(arrow::int32(), arrow::extension::uuid()))}); + auto maybe_serialized = arrow::ipc::SerializeSchema(*dummy_schema); + if (!maybe_serialized.ok()) { + return false; + } + + // Skip the continuation token and the message size + const uint8_t* data = maybe_serialized.ValueUnsafe()->data() + 2 * sizeof(int32_t); + ns(Message_table_t) message = ns(Message_as_root(data)); + ns(Field_table_t) field = ns(Field_vec_at( + ns(Schema_fields((ns(Schema_table_t))ns(Message_header(message)))), 0)); + return ns(DictionaryEncoding_custom_metadata_is_present(ns(Field_dictionary(field)))); +} + class ArrowTypeParameterizedTestFixture : public ::testing::TestWithParam> { protected: @@ -1322,6 +1364,10 @@ class ArrowTypeParameterizedTestFixture TEST_P(ArrowTypeParameterizedTestFixture, NanoarrowIpcArrowTypeRoundtrip) { const std::shared_ptr& data_type = GetParam(); + if (IsDictionaryOfExtension(data_type) && !ArrowCppWritesDictionaryValueMetadata()) { + GTEST_SKIP() << "Arrow C++ does not write DictionaryEncoding.custom_metadata"; + } + std::shared_ptr dummy_schema = arrow::schema({arrow::field("dummy_name", data_type)}); auto maybe_serialized = arrow::ipc::SerializeSchema(*dummy_schema); @@ -1385,6 +1431,11 @@ std::string ArrowSchemaToString(const struct ArrowSchema* schema) { out += "{" + metadata + "}"; } + // Include the dictionary (value) schema, whose metadata is distinct from the field's + if (schema->dictionary != nullptr) { + out += "[dictionary: " + ArrowSchemaToString(schema->dictionary) + "]"; + } + bool comma = false; if (schema->format[0] == '+') { out += "<"; @@ -1450,9 +1501,7 @@ TEST_P(ArrowTypeParameterizedTestFixture, NanoarrowIpcNanoarrowTypeRoundtrip) { TEST_P(ArrowTypeParameterizedTestFixture, NanoarrowIpcArrowArrayRoundtrip) { const std::shared_ptr& data_type = GetParam(); - if (data_type->id() == arrow::Type::DICTIONARY && - std::static_pointer_cast(data_type)->value_type()->id() == - Type::EXTENSION) { + if (IsDictionaryOfExtension(data_type)) { GTEST_SKIP() << "Arrow C++ MakeEmpty() doesn't support dictionary with extension value types"; } @@ -1862,6 +1911,430 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeMetadataWithoutKey) { EXPECT_EQ(ArrowSchemaMetadataToString(schema->metadata), "=schema_value"); } +// Sets the metadata of schema from key/value pairs whose keys and values may contain +// embedded nulls or bytes that are not valid UTF-8 +static void SetMetadataPairs(struct ArrowSchema* schema, + const std::vector>& kv) { + nanoarrow::UniqueBuffer metadata; + ASSERT_EQ(ArrowMetadataBuilderInit(metadata.get(), nullptr), NANOARROW_OK); + for (const auto& item : kv) { + struct ArrowStringView key = {item.first.data(), + static_cast(item.first.size())}; + struct ArrowStringView value = {item.second.data(), + static_cast(item.second.size())}; + ASSERT_EQ(ArrowMetadataBuilderAppend(metadata.get(), key, value), NANOARROW_OK); + } + + if (kv.empty()) { + // Empty (but not NULL) metadata + ASSERT_EQ(ArrowBufferAppendInt32(metadata.get(), 0), NANOARROW_OK); + } + + ASSERT_EQ(ArrowSchemaSetMetadata(schema, reinterpret_cast(metadata->data)), + NANOARROW_OK); +} + +static void InitDictionaryField(struct ArrowSchema* schema, const char* name, + enum ArrowType index_type, enum ArrowType value_type) { + ASSERT_EQ(ArrowSchemaInitFromType(schema, index_type), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetName(schema, name), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaAllocateDictionary(schema), NANOARROW_OK); + if (value_type == NANOARROW_TYPE_FIXED_SIZE_BINARY) { + ArrowSchemaInit(schema->dictionary); + ASSERT_EQ(ArrowSchemaSetTypeFixedSize(schema->dictionary, + NANOARROW_TYPE_FIXED_SIZE_BINARY, 16), + NANOARROW_OK); + } else { + ASSERT_EQ(ArrowSchemaInitFromType(schema->dictionary, value_type), NANOARROW_OK); + } +} + +static const char kOpaqueMetadata[] = + R"({"type_name":"geometry","vendor_name":"nanoarrow"})"; + +// The metadata of a dictionary value type (i.e., schema->dictionary->metadata) +// alongside field metadata (i.e., schema->metadata), nested in a struct and a list. +static void MakeDictionaryValueMetadataSchema(struct ArrowSchema* schema) { + ASSERT_EQ(ArrowSchemaInitFromType(schema, NANOARROW_TYPE_STRUCT), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaAllocateChildren(schema, 5), NANOARROW_OK); + + // A dictionary of arrow.uuid values with field metadata + struct ArrowSchema* uuid_dict = schema->children[0]; + ASSERT_NO_FATAL_FAILURE(InitDictionaryField(uuid_dict, "uuid_dict", NANOARROW_TYPE_INT8, + NANOARROW_TYPE_FIXED_SIZE_BINARY)); + ASSERT_NO_FATAL_FAILURE(SetMetadataPairs( + uuid_dict->dictionary, + {{"ARROW:extension:name", "arrow.uuid"}, {"ARROW:extension:metadata", ""}})); + ASSERT_NO_FATAL_FAILURE(SetMetadataPairs(uuid_dict, {{"field_key", "field_value"}})); + + // A dictionary of arrow.opaque values plus a key whose value is not valid UTF-8, + // nested in a struct + struct ArrowSchema* struct_col = schema->children[1]; + ASSERT_EQ(ArrowSchemaInitFromType(struct_col, NANOARROW_TYPE_STRUCT), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetName(struct_col, "struct_col"), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaAllocateChildren(struct_col, 1), NANOARROW_OK); + struct ArrowSchema* opaque_dict = struct_col->children[0]; + ASSERT_NO_FATAL_FAILURE(InitDictionaryField( + opaque_dict, "opaque_dict", NANOARROW_TYPE_INT32, NANOARROW_TYPE_STRING)); + ASSERT_NO_FATAL_FAILURE(SetMetadataPairs( + opaque_dict->dictionary, {{"ARROW:extension:name", "arrow.opaque"}, + {"ARROW:extension:metadata", kOpaqueMetadata}, + {"some key", std::string("\xff\xfe\xc3\x28")}})); + + // A dictionary of values with arbitrary metadata, alongside field metadata, nested + // in a list + struct ArrowSchema* list_col = schema->children[2]; + ASSERT_EQ(ArrowSchemaInitFromType(list_col, NANOARROW_TYPE_LIST), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetName(list_col, "list_col"), NANOARROW_OK); + struct ArrowSchema* item = list_col->children[0]; + ArrowSchemaRelease(item); + ASSERT_NO_FATAL_FAILURE( + InitDictionaryField(item, "item", NANOARROW_TYPE_INT16, NANOARROW_TYPE_STRING)); + ASSERT_NO_FATAL_FAILURE(SetMetadataPairs(item->dictionary, {{"value_key", "a"}})); + ASSERT_NO_FATAL_FAILURE(SetMetadataPairs(item, {{"item_key", "b"}})); + + // A dictionary without value metadata + ASSERT_NO_FATAL_FAILURE(InitDictionaryField( + schema->children[3], "plain_dict", NANOARROW_TYPE_INT32, NANOARROW_TYPE_STRING)); + + // A dictionary with empty (but not NULL) value metadata + ASSERT_NO_FATAL_FAILURE(InitDictionaryField( + schema->children[4], "empty_dict", NANOARROW_TYPE_INT32, NANOARROW_TYPE_STRING)); + ASSERT_NO_FATAL_FAILURE(SetMetadataPairs(schema->children[4]->dictionary, {})); +} + +static std::string KeyValueToString(ns(KeyValue_vec_t) kv_vec) { + std::string out; + for (size_t i = 0; i < ns(KeyValue_vec_len(kv_vec)); i++) { + ns(KeyValue_table_t) kv = ns(KeyValue_vec_at(kv_vec, i)); + flatbuffers_string_t key = ns(KeyValue_key(kv)); + flatbuffers_string_t value = ns(KeyValue_value(kv)); + if (i > 0) { + out += ", "; + } + out.append(key, flatbuffers_string_len(key)); + out += "="; + out.append(value, flatbuffers_string_len(value)); + } + return out; +} + +// Checks the schema decoded from MakeDictionaryValueMetadataSchema() +static void CheckDictionaryValueMetadataSchema(const struct ArrowSchema* actual, + const struct ArrowSchema* expected) { + EXPECT_EQ(ArrowSchemaToString(actual), ArrowSchemaToString(expected)); + ASSERT_EQ(actual->n_children, 5); + + const struct ArrowSchema* uuid_dict = actual->children[0]; + ASSERT_NE(uuid_dict->dictionary, nullptr); + EXPECT_EQ(ArrowSchemaMetadataToString(uuid_dict->metadata), "field_key=field_value"); + EXPECT_EQ(ArrowSchemaMetadataToString(uuid_dict->dictionary->metadata), + "ARROW:extension:name=arrow.uuid, ARROW:extension:metadata="); + EXPECT_STREQ(uuid_dict->dictionary->format, "w:16"); + + const struct ArrowSchema* opaque_dict = actual->children[1]->children[0]; + ASSERT_NE(opaque_dict->dictionary, nullptr); + EXPECT_EQ(opaque_dict->metadata, nullptr); + EXPECT_EQ(ArrowSchemaMetadataToString(opaque_dict->dictionary->metadata), + std::string("ARROW:extension:name=arrow.opaque, ARROW:extension:metadata=") + + kOpaqueMetadata + ", some key=" + std::string("\xff\xfe\xc3\x28")); + + const struct ArrowSchema* item = actual->children[2]->children[0]; + ASSERT_NE(item->dictionary, nullptr); + EXPECT_EQ(ArrowSchemaMetadataToString(item->metadata), "item_key=b"); + EXPECT_EQ(ArrowSchemaMetadataToString(item->dictionary->metadata), "value_key=a"); + + ASSERT_NE(actual->children[3]->dictionary, nullptr); + EXPECT_EQ(actual->children[3]->metadata, nullptr); + EXPECT_EQ(actual->children[3]->dictionary->metadata, nullptr); + + // Empty value metadata is not written and so is decoded as NULL + ASSERT_NE(actual->children[4]->dictionary, nullptr); + EXPECT_EQ(actual->children[4]->dictionary->metadata, nullptr); +} + +TEST(NanoarrowIpcTest, NanoarrowIpcDictionaryValueMetadataRoundtrip) { + nanoarrow::UniqueSchema schema; + ASSERT_NO_FATAL_FAILURE(MakeDictionaryValueMetadataSchema(schema.get())); + + struct ArrowError error; + nanoarrow::ipc::UniqueEncoder encoder; + ASSERT_EQ(ArrowIpcEncoderInit(encoder.get()), NANOARROW_OK); + ASSERT_EQ(ArrowIpcEncoderEncodeSchema(encoder.get(), schema.get(), &error), + NANOARROW_OK) + << error.message; + + nanoarrow::UniqueBuffer buffer; + ASSERT_EQ( + ArrowIpcEncoderFinalizeBuffer(encoder.get(), /*encapsulate=*/true, buffer.get()), + NANOARROW_OK); + + // Check the flatbuffer: the value metadata is in DictionaryEncoding.custom_metadata + // and the field metadata is in Field.custom_metadata + ns(Message_table_t) message = ns(Message_as_root(buffer->data + 2 * sizeof(int32_t))); + ASSERT_NE(message, nullptr); + ASSERT_EQ(ns(Message_header_type(message)), ns(MessageHeader_Schema)); + ns(Field_vec_t) fields = + ns(Schema_fields((ns(Schema_table_t))ns(Message_header(message)))); + ASSERT_EQ(ns(Field_vec_len(fields)), 5); + + ns(Field_table_t) uuid_dict = ns(Field_vec_at(fields, 0)); + ASSERT_EQ(ns(Field_type_type(uuid_dict)), ns(Type_FixedSizeBinary)); + ns(DictionaryEncoding_table_t) encoding = ns(Field_dictionary(uuid_dict)); + ASSERT_NE(encoding, nullptr); + EXPECT_EQ(KeyValueToString(ns(DictionaryEncoding_custom_metadata(encoding))), + "ARROW:extension:name=arrow.uuid, ARROW:extension:metadata="); + EXPECT_EQ(KeyValueToString(ns(Field_custom_metadata(uuid_dict))), + "field_key=field_value"); + + ns(Field_table_t) opaque_dict = + ns(Field_vec_at(ns(Field_children(ns(Field_vec_at(fields, 1)))), 0)); + encoding = ns(Field_dictionary(opaque_dict)); + ASSERT_NE(encoding, nullptr); + EXPECT_EQ(KeyValueToString(ns(DictionaryEncoding_custom_metadata(encoding))), + std::string("ARROW:extension:name=arrow.opaque, ARROW:extension:metadata=") + + kOpaqueMetadata + ", some key=" + std::string("\xff\xfe\xc3\x28")); + EXPECT_FALSE(ns(Field_custom_metadata_is_present(opaque_dict))); + + ns(Field_table_t) item = + ns(Field_vec_at(ns(Field_children(ns(Field_vec_at(fields, 2)))), 0)); + encoding = ns(Field_dictionary(item)); + ASSERT_NE(encoding, nullptr); + EXPECT_EQ(KeyValueToString(ns(DictionaryEncoding_custom_metadata(encoding))), + "value_key=a"); + EXPECT_EQ(KeyValueToString(ns(Field_custom_metadata(item))), "item_key=b"); + + // Absent and empty value metadata are both omitted + for (size_t i : {3, 4}) { + ns(Field_table_t) field = ns(Field_vec_at(fields, i)); + encoding = ns(Field_dictionary(field)); + ASSERT_NE(encoding, nullptr); + EXPECT_FALSE(ns(DictionaryEncoding_custom_metadata_is_present(encoding))); + EXPECT_FALSE(ns(Field_custom_metadata_is_present(field))); + } + + // Decode and check that the schema roundtripped + struct ArrowBufferView buffer_view; + buffer_view.data.data = buffer->data; + buffer_view.size_bytes = buffer->size_bytes; + + nanoarrow::ipc::UniqueDecoder decoder; + ASSERT_EQ(ArrowIpcDecoderInit(decoder.get()), NANOARROW_OK); + ASSERT_EQ(ArrowIpcDecoderVerifyHeader(decoder.get(), buffer_view, &error), NANOARROW_OK) + << error.message; + ASSERT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), buffer_view, &error), NANOARROW_OK) + << error.message; + + nanoarrow::UniqueSchema roundtripped; + nanoarrow::ipc::UniqueDictionaryEncodings encodings; + ASSERT_EQ(ArrowIpcDecoderDecodeSchemaWithDictionaries(decoder.get(), roundtripped.get(), + encodings.get(), &error), + NANOARROW_OK) + << error.message; + ASSERT_NO_FATAL_FAILURE( + CheckDictionaryValueMetadataSchema(roundtripped.get(), schema.get())); + + // The dictionary encodings resolve to the dictionary-encoded fields + for (const struct ArrowSchema* field : + {roundtripped->children[0], roundtripped->children[1]->children[0], + roundtripped->children[2]->children[0], roundtripped->children[3], + roundtripped->children[4]}) { + const struct ArrowIpcDictionaryEncoding* found = + ArrowIpcDictionaryEncodingsFind(encodings.get(), field); + ASSERT_NE(found, nullptr); + EXPECT_EQ(found->schema, field); + } +} + +TEST(NanoarrowIpcTest, NanoarrowIpcDictionaryValueMetadataFooterRoundtrip) { + using namespace nanoarrow::literals; + + nanoarrow::ipc::UniqueFooter footer; + ASSERT_NO_FATAL_FAILURE(MakeDictionaryValueMetadataSchema(&footer->schema)); + ArrowIpcDictionaryEncodingsInit(&footer->dictionaries); + ASSERT_EQ( + ArrowIpcDictionaryEncodingsAppendSchema(&footer->dictionaries, &footer->schema), + NANOARROW_OK); + + nanoarrow::ipc::UniqueEncoder encoder; + ASSERT_EQ(ArrowIpcEncoderInit(encoder.get()), NANOARROW_OK); + + struct ArrowError error; + ASSERT_EQ(ArrowIpcEncoderEncodeFooter(encoder.get(), footer.get(), &error), + NANOARROW_OK) + << error.message; + + nanoarrow::UniqueBuffer buffer; + ASSERT_EQ( + ArrowIpcEncoderFinalizeBuffer(encoder.get(), /*encapsulate=*/false, buffer.get()), + NANOARROW_OK); + +#ifdef __BIG_ENDIAN__ + uint32_t footer_size_le = bswap32(static_cast(buffer->size_bytes)); + ASSERT_EQ(ArrowBufferAppendInt32(buffer.get(), footer_size_le), NANOARROW_OK); +#else + ASSERT_EQ( + ArrowBufferAppendInt32(buffer.get(), static_cast(buffer->size_bytes)), + NANOARROW_OK); +#endif + ASSERT_EQ(ArrowBufferAppendStringView(buffer.get(), "ARROW1"_asv), NANOARROW_OK); + + struct ArrowBufferView buffer_view; + buffer_view.data.data = buffer->data; + buffer_view.size_bytes = buffer->size_bytes; + + nanoarrow::ipc::UniqueDecoder decoder; + ASSERT_EQ(ArrowIpcDecoderInit(decoder.get()), NANOARROW_OK); + ASSERT_EQ(ArrowIpcDecoderVerifyFooter(decoder.get(), buffer_view, &error), NANOARROW_OK) + << error.message; + ASSERT_EQ(ArrowIpcDecoderDecodeFooter(decoder.get(), buffer_view, &error), NANOARROW_OK) + << error.message; + + ASSERT_NO_FATAL_FAILURE( + CheckDictionaryValueMetadataSchema(&decoder->footer->schema, &footer->schema)); + EXPECT_EQ(decoder->footer->dictionaries.encodings.size_bytes, + 5 * static_cast(sizeof(struct ArrowIpcDictionaryEncoding))); +} + +// Builds a Schema message with a single dictionary-encoded utf8 field "col" with +// int32 indices, whose Field.custom_metadata is field_metadata and whose +// DictionaryEncoding.custom_metadata is value_metadata (omitted if nullptr). +static void MakeDictionarySchemaMessage( + const std::vector>* field_metadata, + const std::vector>* value_metadata, + struct ArrowBuffer* out) { + flatcc_builder_t builder; + flatcc_builder_init(&builder); + + ASSERT_EQ(ns(Message_start_as_root(&builder)), 0); + ASSERT_EQ(ns(Message_version_add(&builder, ns(MetadataVersion_V5))), 0); + ASSERT_EQ(ns(Message_header_Schema_start(&builder)), 0); + ASSERT_EQ(ns(Schema_endianness_add(&builder, ns(Endianness_Little))), 0); + ASSERT_EQ(ns(Schema_fields_start(&builder)), 0); + ASSERT_EQ(ns(Schema_fields_push_start(&builder)), 0); + ASSERT_EQ(ns(Field_name_create_str(&builder, "col")), 0); + ASSERT_EQ(ns(Field_nullable_add(&builder, 1)), 0); + ASSERT_EQ(ns(Field_type_Utf8_create(&builder)), 0); + + ASSERT_EQ(ns(Field_dictionary_start(&builder)), 0); + ASSERT_EQ(ns(DictionaryEncoding_id_add(&builder, 0)), 0); + ASSERT_EQ(ns(DictionaryEncoding_indexType_create(&builder, 32, 1)), 0); + if (value_metadata != nullptr) { + ASSERT_EQ(ns(DictionaryEncoding_custom_metadata_start(&builder)), 0); + for (const auto& item : *value_metadata) { + ASSERT_EQ(ns(DictionaryEncoding_custom_metadata_push_start(&builder)), 0); + ASSERT_EQ(ns(KeyValue_key_create(&builder, item.first.data(), item.first.size())), + 0); + ASSERT_EQ( + ns(KeyValue_value_create(&builder, item.second.data(), item.second.size())), 0); + ASSERT_NE(ns(DictionaryEncoding_custom_metadata_push_end(&builder)), nullptr); + } + ASSERT_EQ(ns(DictionaryEncoding_custom_metadata_end(&builder)), 0); + } + ASSERT_EQ(ns(Field_dictionary_end(&builder)), 0); + + if (field_metadata != nullptr) { + ASSERT_EQ(ns(Field_custom_metadata_start(&builder)), 0); + for (const auto& item : *field_metadata) { + ASSERT_EQ(ns(Field_custom_metadata_push_start(&builder)), 0); + ASSERT_EQ(ns(KeyValue_key_create(&builder, item.first.data(), item.first.size())), + 0); + ASSERT_EQ( + ns(KeyValue_value_create(&builder, item.second.data(), item.second.size())), 0); + ASSERT_NE(ns(Field_custom_metadata_push_end(&builder)), nullptr); + } + ASSERT_EQ(ns(Field_custom_metadata_end(&builder)), 0); + } + + ASSERT_NE(ns(Schema_fields_push_end(&builder)), nullptr); + ASSERT_EQ(ns(Schema_fields_end(&builder)), 0); + ASSERT_EQ(ns(Message_header_Schema_end(&builder)), 0); + ASSERT_EQ(ns(Message_bodyLength_add(&builder, 0)), 0); + ASSERT_NE(ns(Message_end_as_root(&builder)), 0); + + ASSERT_NO_FATAL_FAILURE(FinishEncapsulatedMessage(&builder, out)); + flatcc_builder_clear(&builder); +} + +static void DecodeSchemaMessage(struct ArrowBuffer* message, struct ArrowSchema* out) { + struct ArrowBufferView data; + data.data.as_uint8 = message->data; + data.size_bytes = message->size_bytes; + + struct ArrowError error; + nanoarrow::ipc::UniqueDecoder decoder; + ASSERT_EQ(ArrowIpcDecoderInit(decoder.get()), NANOARROW_OK); + ASSERT_EQ(ArrowIpcDecoderVerifyHeader(decoder.get(), data, &error), NANOARROW_OK) + << error.message; + ASSERT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), data, &error), NANOARROW_OK) + << error.message; + ASSERT_EQ(ArrowIpcDecoderDecodeSchema(decoder.get(), out, &error), NANOARROW_OK) + << error.message; +} + +TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryValueMetadata) { + std::vector> field_metadata = { + {"ARROW:extension:name", "some_ext_with_dictionary_storage"}, + {"field_key", "field_value"}}; + std::vector> value_metadata = { + {"ARROW:extension:name", "arrow.opaque"}, + {"ARROW:extension:metadata", kOpaqueMetadata}, + {std::string("key\x00\xff", 5), std::string("\x00\xc3\x28", 3)}}; + + // Both Field.custom_metadata and DictionaryEncoding.custom_metadata + nanoarrow::UniqueBuffer message; + ASSERT_NO_FATAL_FAILURE( + MakeDictionarySchemaMessage(&field_metadata, &value_metadata, message.get())); + nanoarrow::UniqueSchema schema; + ASSERT_NO_FATAL_FAILURE(DecodeSchemaMessage(message.get(), schema.get())); + ASSERT_EQ(schema->n_children, 1); + ASSERT_NE(schema->children[0]->dictionary, nullptr); + EXPECT_STREQ(schema->children[0]->format, "i"); + EXPECT_STREQ(schema->children[0]->dictionary->format, "u"); + EXPECT_EQ(ArrowSchemaMetadataToString(schema->children[0]->metadata), + "ARROW:extension:name=some_ext_with_dictionary_storage, " + "field_key=field_value"); + EXPECT_EQ(ArrowSchemaMetadataToString(schema->children[0]->dictionary->metadata), + std::string("ARROW:extension:name=arrow.opaque, ARROW:extension:metadata=") + + kOpaqueMetadata + ", " + std::string("key\x00\xff", 5) + "=" + + std::string("\x00\xc3\x28", 3)); + + // Only DictionaryEncoding.custom_metadata + message.reset(); + ASSERT_NO_FATAL_FAILURE( + MakeDictionarySchemaMessage(nullptr, &value_metadata, message.get())); + schema.reset(); + ASSERT_NO_FATAL_FAILURE(DecodeSchemaMessage(message.get(), schema.get())); + EXPECT_EQ(schema->children[0]->metadata, nullptr); + EXPECT_EQ(ArrowSchemaMetadataToString(schema->children[0]->dictionary->metadata), + std::string("ARROW:extension:name=arrow.opaque, ARROW:extension:metadata=") + + kOpaqueMetadata + ", " + std::string("key\x00\xff", 5) + "=" + + std::string("\x00\xc3\x28", 3)); + + // Without DictionaryEncoding.custom_metadata (e.g., written before format version + // 1.6), Field.custom_metadata remains the metadata of the field + std::vector> legacy_field_metadata = { + {"ARROW:extension:name", "arrow.uuid"}, {"ARROW:extension:metadata", ""}}; + message.reset(); + ASSERT_NO_FATAL_FAILURE( + MakeDictionarySchemaMessage(&legacy_field_metadata, nullptr, message.get())); + schema.reset(); + ASSERT_NO_FATAL_FAILURE(DecodeSchemaMessage(message.get(), schema.get())); + EXPECT_EQ(ArrowSchemaMetadataToString(schema->children[0]->metadata), + "ARROW:extension:name=arrow.uuid, ARROW:extension:metadata="); + EXPECT_EQ(schema->children[0]->dictionary->metadata, nullptr); + + // An empty DictionaryEncoding.custom_metadata is equivalent to none + std::vector> empty_metadata; + message.reset(); + ASSERT_NO_FATAL_FAILURE( + MakeDictionarySchemaMessage(nullptr, &empty_metadata, message.get())); + schema.reset(); + ASSERT_NO_FATAL_FAILURE(DecodeSchemaMessage(message.get(), schema.get())); + EXPECT_EQ(schema->children[0]->metadata, nullptr); + EXPECT_EQ(schema->children[0]->dictionary->metadata, nullptr); +} + #if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) TEST_P(ArrowTypeParameterizedTestFixture, NanoarrowIpcNanoarrowArrayRoundtrip) { if (GetParam()->id() == arrow::Type::DICTIONARY) { @@ -2023,11 +2496,10 @@ INSTANTIATE_TEST_SUITE_P( // Extension type arrow::extension::uuid(), // Extension type with dictionary as the storage type - dict_extension() - // Dictionary-encoded extension is not supported in IPC + dict_extension(), + // Dictionary with extension value type // https://github.com/apache/arrow/issues/49704 - // arrow::dictionary(arrow::int32(), arrow::extension::uuid())) - )); + arrow::dictionary(arrow::int32(), arrow::extension::uuid()))); class ArrowSchemaParameterizedTestFixture : public ::testing::TestWithParam> { @@ -2037,6 +2509,11 @@ class ArrowSchemaParameterizedTestFixture TEST_P(ArrowSchemaParameterizedTestFixture, NanoarrowIpcArrowSchemaRoundtrip) { const std::shared_ptr& arrow_schema = GetParam(); + if (HasDictionaryOfExtension(*arrow_schema) && + !ArrowCppWritesDictionaryValueMetadata()) { + GTEST_SKIP() << "Arrow C++ does not write DictionaryEncoding.custom_metadata"; + } + auto maybe_serialized = arrow::ipc::SerializeSchema(*arrow_schema); ASSERT_TRUE(maybe_serialized.ok()); @@ -2101,6 +2578,41 @@ TEST_P(ArrowSchemaParameterizedTestFixture, NanoarrowIpcNanoarrowSchemaRoundtrip EXPECT_EQ(ArrowSchemaToString(roundtripped.get()), ArrowSchemaToString(schema.get())); } +TEST_P(ArrowSchemaParameterizedTestFixture, NanoarrowIpcArrowReadsNanoarrowSchema) { + const std::shared_ptr& arrow_schema = GetParam(); + if (HasDictionaryOfExtension(*arrow_schema) && + !ArrowCppWritesDictionaryValueMetadata()) { + GTEST_SKIP() << "Arrow C++ does not read DictionaryEncoding.custom_metadata"; + } + + nanoarrow::UniqueSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*arrow_schema, schema.get()).ok()); + + nanoarrow::ipc::UniqueEncoder encoder; + ASSERT_EQ(ArrowIpcEncoderInit(encoder.get()), NANOARROW_OK); + + struct ArrowError error; + ASSERT_EQ(ArrowIpcEncoderEncodeSchema(encoder.get(), schema.get(), &error), + NANOARROW_OK) + << error.message; + + nanoarrow::UniqueBuffer buffer; + ASSERT_EQ( + ArrowIpcEncoderFinalizeBuffer(encoder.get(), /*encapsulate=*/true, buffer.get()), + NANOARROW_OK); + + arrow::io::BufferReader reader( + std::make_shared(buffer->data, buffer->size_bytes)); + arrow::ipc::DictionaryMemo memo; + auto maybe_schema = arrow::ipc::ReadSchema(&reader, &memo); + ASSERT_TRUE(maybe_schema.ok()) << maybe_schema.status().ToString(); + + // Better failure message if we first check for string equality + EXPECT_EQ(maybe_schema.ValueUnsafe()->ToString(/*show_metadata=*/true), + arrow_schema->ToString(/*show_metadata=*/true)); + EXPECT_TRUE(maybe_schema.ValueUnsafe()->Equals(arrow_schema, true)); +} + TEST_P(ArrowSchemaParameterizedTestFixture, NanoarrowIpcNanoarrowFooterRoundtrip) { using namespace nanoarrow::literals; const std::shared_ptr& arrow_schema = GetParam(); @@ -2180,12 +2692,18 @@ INSTANTIATE_TEST_SUITE_P( // Dictionary with field metadata arrow::schema({arrow::field( "some_name", arrow::dictionary(arrow::int32(), arrow::utf8()), - arrow::KeyValueMetadata::Make({"key1", "key2"}, {"value1", "value2"}))}) - // Dictionary with extension storage and field metadata is not supported in IPC - // arrow::schema({arrow::field( - // "some_name", arrow::dictionary(arrow::int32(), arrow::extension::uuid()), - // arrow::KeyValueMetadata::Make({"key1", "key2"}, {"value1", "value2"}))}) - )); + arrow::KeyValueMetadata::Make({"key1", "key2"}, {"value1", "value2"}))}), + // Dictionary with extension value type and field metadata + // https://github.com/apache/arrow/issues/49704 + arrow::schema({arrow::field( + "some_name", arrow::dictionary(arrow::int32(), arrow::extension::uuid()), + arrow::KeyValueMetadata::Make({"key1", "key2"}, {"value1", "value2"}))}), + // Nested dictionary with extension value type and field metadata + arrow::schema({arrow::field( + "some_name", + arrow::list(arrow::field( + "item", arrow::dictionary(arrow::int16(), arrow::extension::uuid()), + arrow::KeyValueMetadata::Make({"key1"}, {"value1"}))))}))); // Advance data past the message whose header was just decoded static void AdvancePastMessage(struct ArrowIpcDecoder* decoder, diff --git a/src/nanoarrow/ipc/encoder.c b/src/nanoarrow/ipc/encoder.c index ab7f0b60d..dc6c23b33 100644 --- a/src/nanoarrow/ipc/encoder.c +++ b/src/nanoarrow/ipc/encoder.c @@ -608,20 +608,37 @@ static ArrowErrorCode ArrowIpcEncodeField( return EINVAL; } - // Create the Int type for the index type - ns(Int_ref_t) index_type_ref = - ns(Int_create(builder, index_bitwidth, index_is_signed)); - FLATCC_RETURN_IF_NULL(index_type_ref, error); - - // Create the DictionaryEncoding with id, indexType, isOrdered, and dictionaryKind + // Add the DictionaryEncoding with id, indexType, isOrdered, dictionaryKind, and + // custom_metadata to the field flatbuffers_bool_t is_ordered = (schema->flags & ARROW_FLAG_DICTIONARY_ORDERED) != 0; - ns(DictionaryEncoding_ref_t) dict_encoding_ref = - ns(DictionaryEncoding_create(builder, encoding->id, index_type_ref, is_ordered, - ns(DictionaryKind_DenseArray))); - FLATCC_RETURN_IF_NULL(dict_encoding_ref, error); + FLATCC_RETURN_UNLESS_0(Field_dictionary_start(builder), error); + FLATCC_RETURN_UNLESS_0(DictionaryEncoding_id_add(builder, encoding->id), error); + FLATCC_RETURN_UNLESS_0( + DictionaryEncoding_indexType_create(builder, index_bitwidth, index_is_signed), + error); + FLATCC_RETURN_UNLESS_0(DictionaryEncoding_isOrdered_add(builder, is_ordered), error); + FLATCC_RETURN_UNLESS_0( + DictionaryEncoding_dictionaryKind_add(builder, ns(DictionaryKind_DenseArray)), + error); + + // The metadata of the dictionary (value) schema is the metadata of the dictionary + // value type, whose IPC counterpart is DictionaryEncoding.custom_metadata (format + // version 1.6, https://github.com/apache/arrow/issues/49704). The metadata of the + // field itself (i.e., schema->metadata) is written to Field.custom_metadata below. + // Writers omit DictionaryEncoding.custom_metadata if there is no value metadata. + struct ArrowMetadataReader value_metadata; + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowMetadataReaderInit(&value_metadata, schema->dictionary->metadata), error); + if (value_metadata.remaining_keys > 0) { + FLATCC_RETURN_UNLESS_0(DictionaryEncoding_custom_metadata_start(builder), error); + NANOARROW_RETURN_NOT_OK(ArrowIpcEncodeMetadata( + builder, schema->dictionary->metadata, + &ns(DictionaryEncoding_custom_metadata_push_start), + &ns(DictionaryEncoding_custom_metadata_push_end), error)); + FLATCC_RETURN_UNLESS_0(DictionaryEncoding_custom_metadata_end(builder), error); + } - // Add the dictionary encoding to the field - FLATCC_RETURN_UNLESS_0(Field_dictionary_add(builder, dict_encoding_ref), error); + FLATCC_RETURN_UNLESS_0(Field_dictionary_end(builder), error); // Support dictionary values with children by encoding children from // schema->dictionary (and add a roundtrip test for a nested value type). diff --git a/src/nanoarrow/ipc/flatcc_generated.h b/src/nanoarrow/ipc/flatcc_generated.h index 82ba6b451..d4d37e05c 100644 --- a/src/nanoarrow/ipc/flatcc_generated.h +++ b/src/nanoarrow/ipc/flatcc_generated.h @@ -1958,8 +1958,14 @@ static inline int org_apache_arrow_flatbuf_TimeUnit_is_known_value(org_apache_ar } } +/** The unit of an Interval. + * + * All integers in the units below are stored in the endianness indicated + * by the schema. */ typedef int16_t org_apache_arrow_flatbuf_IntervalUnit_enum_t; __flatbuffers_define_integer_type(org_apache_arrow_flatbuf_IntervalUnit, org_apache_arrow_flatbuf_IntervalUnit_enum_t, 16) +/** Indicates the number of elapsed whole months, stored as + * 4-byte signed integers. */ #define org_apache_arrow_flatbuf_IntervalUnit_YEAR_MONTH ((org_apache_arrow_flatbuf_IntervalUnit_enum_t)INT16_C(0)) #define org_apache_arrow_flatbuf_IntervalUnit_DAY_TIME ((org_apache_arrow_flatbuf_IntervalUnit_enum_t)INT16_C(1)) #define org_apache_arrow_flatbuf_IntervalUnit_MONTH_DAY_NANO ((org_apache_arrow_flatbuf_IntervalUnit_enum_t)INT16_C(2)) @@ -2506,6 +2512,9 @@ __flatbuffers_define_scalar_field(0, org_apache_arrow_flatbuf_Timestamp, unit, o * the data (see above). */ __flatbuffers_define_string_field(1, org_apache_arrow_flatbuf_Timestamp, timezone, 0) +/** A "calendar" interval which models types that don't necessarily + * have a precise duration without the context of a base timestamp (e.g. + * days can differ in length during day light savings time transitions). */ struct org_apache_arrow_flatbuf_Interval_table { uint8_t unused__; }; static inline size_t org_apache_arrow_flatbuf_Interval_vec_len(org_apache_arrow_flatbuf_Interval_vec_t vec) @@ -2516,6 +2525,19 @@ __flatbuffers_table_as_root(org_apache_arrow_flatbuf_Interval) __flatbuffers_define_scalar_field(0, org_apache_arrow_flatbuf_Interval, unit, org_apache_arrow_flatbuf_IntervalUnit, org_apache_arrow_flatbuf_IntervalUnit_enum_t, INT16_C(0)) +/** An absolute length of time unrelated to any calendar artifacts. + * + * For the purposes of Arrow Implementations, adding this value to a Timestamp + * ("t1") naively (i.e. simply summing the two numbers) is acceptable even + * though in some cases the resulting Timestamp (t2) would not account for + * leap-seconds during the elapsed time between "t1" and "t2". Similarly, + * representing the difference between two Unix timestamps is acceptable, but + * would yield a value that is possibly a few seconds off from the true elapsed + * time. + * + * The resolution defaults to millisecond, but can be any of the other + * supported TimeUnit values as with Timestamp and Time types. This type is + * always represented as an 8-byte integer. */ struct org_apache_arrow_flatbuf_Duration_table { uint8_t unused__; }; static inline size_t org_apache_arrow_flatbuf_Duration_vec_len(org_apache_arrow_flatbuf_Duration_vec_t vec) @@ -2532,7 +2554,7 @@ typedef uint8_t org_apache_arrow_flatbuf_Type_union_type_t; __flatbuffers_define_integer_type(org_apache_arrow_flatbuf_Type, org_apache_arrow_flatbuf_Type_union_type_t, 8) __flatbuffers_define_union(flatbuffers_, org_apache_arrow_flatbuf_Type) /** ---------------------------------------------------------------------- - * user defined key value pairs to add custom metadata to arrow + * user defined key value pairs to add custom metadata to Arrow * key namespacing is the responsibility of the user */ #define org_apache_arrow_flatbuf_Type_NONE ((org_apache_arrow_flatbuf_Type_union_type_t)UINT8_C(0)) #define org_apache_arrow_flatbuf_Type_Null ((org_apache_arrow_flatbuf_Type_union_type_t)UINT8_C(1)) @@ -2666,6 +2688,8 @@ __flatbuffers_define_table_field(1, org_apache_arrow_flatbuf_DictionaryEncoding, * preserve that metadata here */ __flatbuffers_define_scalar_field(2, org_apache_arrow_flatbuf_DictionaryEncoding, isOrdered, flatbuffers_bool, flatbuffers_bool_t, UINT8_C(0)) __flatbuffers_define_scalar_field(3, org_apache_arrow_flatbuf_DictionaryEncoding, dictionaryKind, org_apache_arrow_flatbuf_DictionaryKind, org_apache_arrow_flatbuf_DictionaryKind_enum_t, INT16_C(0)) +/** Metadata for the dictionary value type, e.g. the extension type of the values */ +__flatbuffers_define_vector_field(4, org_apache_arrow_flatbuf_DictionaryEncoding, custom_metadata, org_apache_arrow_flatbuf_KeyValue_vec_t, 0) /** ---------------------------------------------------------------------- * A field represents a named column in a record / row batch or child of a @@ -2707,6 +2731,7 @@ __flatbuffers_table_as_root(org_apache_arrow_flatbuf_Schema) * if endianness doesn't match the underlying system then the vectors need to be converted */ __flatbuffers_define_scalar_field(0, org_apache_arrow_flatbuf_Schema, endianness, org_apache_arrow_flatbuf_Endianness, org_apache_arrow_flatbuf_Endianness_enum_t, INT16_C(0)) __flatbuffers_define_vector_field(1, org_apache_arrow_flatbuf_Schema, fields, org_apache_arrow_flatbuf_Field_vec_t, 0) +/** User-defined metadata */ __flatbuffers_define_vector_field(2, org_apache_arrow_flatbuf_Schema, custom_metadata, org_apache_arrow_flatbuf_KeyValue_vec_t, 0) /** Features used in the stream/file. */ __flatbuffers_define_vector_field(3, org_apache_arrow_flatbuf_Schema, features, org_apache_arrow_flatbuf_Feature_vec_t, 0) @@ -2926,7 +2951,7 @@ __flatbuffers_build_table(flatbuffers_, org_apache_arrow_flatbuf_KeyValue, 2) static const flatbuffers_voffset_t __org_apache_arrow_flatbuf_DictionaryEncoding_required[] = { 0 }; typedef flatbuffers_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_ref_t; static org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_clone(flatbuffers_builder_t *B, org_apache_arrow_flatbuf_DictionaryEncoding_table_t t); -__flatbuffers_build_table(flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding, 4) +__flatbuffers_build_table(flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding, 5) static const flatbuffers_voffset_t __org_apache_arrow_flatbuf_Field_required[] = { 0 }; typedef flatbuffers_ref_t org_apache_arrow_flatbuf_Field_ref_t; @@ -3073,8 +3098,10 @@ __flatbuffers_build_table_prolog(flatbuffers_, org_apache_arrow_flatbuf_Duration static inline org_apache_arrow_flatbuf_KeyValue_ref_t org_apache_arrow_flatbuf_KeyValue_create(flatbuffers_builder_t *B __org_apache_arrow_flatbuf_KeyValue_formal_args); __flatbuffers_build_table_prolog(flatbuffers_, org_apache_arrow_flatbuf_KeyValue, org_apache_arrow_flatbuf_KeyValue_file_identifier, org_apache_arrow_flatbuf_KeyValue_type_identifier) -#define __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args , int64_t v0, org_apache_arrow_flatbuf_Int_ref_t v1, flatbuffers_bool_t v2, org_apache_arrow_flatbuf_DictionaryKind_enum_t v3 -#define __org_apache_arrow_flatbuf_DictionaryEncoding_call_args , v0, v1, v2, v3 +#define __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args ,\ + int64_t v0, org_apache_arrow_flatbuf_Int_ref_t v1, flatbuffers_bool_t v2, org_apache_arrow_flatbuf_DictionaryKind_enum_t v3, org_apache_arrow_flatbuf_KeyValue_vec_ref_t v4 +#define __org_apache_arrow_flatbuf_DictionaryEncoding_call_args ,\ + v0, v1, v2, v3, v4 static inline org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_create(flatbuffers_builder_t *B __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args); __flatbuffers_build_table_prolog(flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding, org_apache_arrow_flatbuf_DictionaryEncoding_file_identifier, org_apache_arrow_flatbuf_DictionaryEncoding_type_identifier) @@ -3729,12 +3756,14 @@ __flatbuffers_build_scalar_field(0, flatbuffers_, org_apache_arrow_flatbuf_Dicti __flatbuffers_build_table_field(1, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_indexType, org_apache_arrow_flatbuf_Int, org_apache_arrow_flatbuf_DictionaryEncoding) __flatbuffers_build_scalar_field(2, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_isOrdered, flatbuffers_bool, flatbuffers_bool_t, 1, 1, UINT8_C(0), org_apache_arrow_flatbuf_DictionaryEncoding) __flatbuffers_build_scalar_field(3, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind, org_apache_arrow_flatbuf_DictionaryKind, org_apache_arrow_flatbuf_DictionaryKind_enum_t, 2, 2, INT16_C(0), org_apache_arrow_flatbuf_DictionaryEncoding) +__flatbuffers_build_table_vector_field(4, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_custom_metadata, org_apache_arrow_flatbuf_KeyValue, org_apache_arrow_flatbuf_DictionaryEncoding) static inline org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_create(flatbuffers_builder_t *B __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args) { if (org_apache_arrow_flatbuf_DictionaryEncoding_start(B) || org_apache_arrow_flatbuf_DictionaryEncoding_id_add(B, v0) || org_apache_arrow_flatbuf_DictionaryEncoding_indexType_add(B, v1) + || org_apache_arrow_flatbuf_DictionaryEncoding_custom_metadata_add(B, v4) || org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind_add(B, v3) || org_apache_arrow_flatbuf_DictionaryEncoding_isOrdered_add(B, v2)) { return 0; @@ -3748,6 +3777,7 @@ static org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbu if (org_apache_arrow_flatbuf_DictionaryEncoding_start(B) || org_apache_arrow_flatbuf_DictionaryEncoding_id_pick(B, t) || org_apache_arrow_flatbuf_DictionaryEncoding_indexType_pick(B, t) + || org_apache_arrow_flatbuf_DictionaryEncoding_custom_metadata_pick(B, t) || org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind_pick(B, t) || org_apache_arrow_flatbuf_DictionaryEncoding_isOrdered_pick(B, t)) { return 0; @@ -5220,6 +5250,7 @@ static int org_apache_arrow_flatbuf_DictionaryEncoding_verify_table(flatcc_table if ((ret = flatcc_verify_table_field(td, 1, 0, &org_apache_arrow_flatbuf_Int_verify_table) /* indexType */)) return ret; if ((ret = flatcc_verify_field(td, 2, 1, 1) /* isOrdered */)) return ret; if ((ret = flatcc_verify_field(td, 3, 2, 2) /* dictionaryKind */)) return ret; + if ((ret = flatcc_verify_table_vector_field(td, 4, 0, &org_apache_arrow_flatbuf_KeyValue_verify_table) /* custom_metadata */)) return ret; return flatcc_verify_ok; } @@ -6370,8 +6401,14 @@ static inline int org_apache_arrow_flatbuf_TimeUnit_is_known_value(org_apache_ar } } +/** The unit of an Interval. + * + * All integers in the units below are stored in the endianness indicated + * by the schema. */ typedef int16_t org_apache_arrow_flatbuf_IntervalUnit_enum_t; __flatbuffers_define_integer_type(org_apache_arrow_flatbuf_IntervalUnit, org_apache_arrow_flatbuf_IntervalUnit_enum_t, 16) +/** Indicates the number of elapsed whole months, stored as + * 4-byte signed integers. */ #define org_apache_arrow_flatbuf_IntervalUnit_YEAR_MONTH ((org_apache_arrow_flatbuf_IntervalUnit_enum_t)INT16_C(0)) #define org_apache_arrow_flatbuf_IntervalUnit_DAY_TIME ((org_apache_arrow_flatbuf_IntervalUnit_enum_t)INT16_C(1)) #define org_apache_arrow_flatbuf_IntervalUnit_MONTH_DAY_NANO ((org_apache_arrow_flatbuf_IntervalUnit_enum_t)INT16_C(2)) @@ -6918,6 +6955,9 @@ __flatbuffers_define_scalar_field(0, org_apache_arrow_flatbuf_Timestamp, unit, o * the data (see above). */ __flatbuffers_define_string_field(1, org_apache_arrow_flatbuf_Timestamp, timezone, 0) +/** A "calendar" interval which models types that don't necessarily + * have a precise duration without the context of a base timestamp (e.g. + * days can differ in length during day light savings time transitions). */ struct org_apache_arrow_flatbuf_Interval_table { uint8_t unused__; }; static inline size_t org_apache_arrow_flatbuf_Interval_vec_len(org_apache_arrow_flatbuf_Interval_vec_t vec) @@ -6928,6 +6968,19 @@ __flatbuffers_table_as_root(org_apache_arrow_flatbuf_Interval) __flatbuffers_define_scalar_field(0, org_apache_arrow_flatbuf_Interval, unit, org_apache_arrow_flatbuf_IntervalUnit, org_apache_arrow_flatbuf_IntervalUnit_enum_t, INT16_C(0)) +/** An absolute length of time unrelated to any calendar artifacts. + * + * For the purposes of Arrow Implementations, adding this value to a Timestamp + * ("t1") naively (i.e. simply summing the two numbers) is acceptable even + * though in some cases the resulting Timestamp (t2) would not account for + * leap-seconds during the elapsed time between "t1" and "t2". Similarly, + * representing the difference between two Unix timestamps is acceptable, but + * would yield a value that is possibly a few seconds off from the true elapsed + * time. + * + * The resolution defaults to millisecond, but can be any of the other + * supported TimeUnit values as with Timestamp and Time types. This type is + * always represented as an 8-byte integer. */ struct org_apache_arrow_flatbuf_Duration_table { uint8_t unused__; }; static inline size_t org_apache_arrow_flatbuf_Duration_vec_len(org_apache_arrow_flatbuf_Duration_vec_t vec) @@ -6944,7 +6997,7 @@ typedef uint8_t org_apache_arrow_flatbuf_Type_union_type_t; __flatbuffers_define_integer_type(org_apache_arrow_flatbuf_Type, org_apache_arrow_flatbuf_Type_union_type_t, 8) __flatbuffers_define_union(flatbuffers_, org_apache_arrow_flatbuf_Type) /** ---------------------------------------------------------------------- - * user defined key value pairs to add custom metadata to arrow + * user defined key value pairs to add custom metadata to Arrow * key namespacing is the responsibility of the user */ #define org_apache_arrow_flatbuf_Type_NONE ((org_apache_arrow_flatbuf_Type_union_type_t)UINT8_C(0)) #define org_apache_arrow_flatbuf_Type_Null ((org_apache_arrow_flatbuf_Type_union_type_t)UINT8_C(1)) @@ -7078,6 +7131,8 @@ __flatbuffers_define_table_field(1, org_apache_arrow_flatbuf_DictionaryEncoding, * preserve that metadata here */ __flatbuffers_define_scalar_field(2, org_apache_arrow_flatbuf_DictionaryEncoding, isOrdered, flatbuffers_bool, flatbuffers_bool_t, UINT8_C(0)) __flatbuffers_define_scalar_field(3, org_apache_arrow_flatbuf_DictionaryEncoding, dictionaryKind, org_apache_arrow_flatbuf_DictionaryKind, org_apache_arrow_flatbuf_DictionaryKind_enum_t, INT16_C(0)) +/** Metadata for the dictionary value type, e.g. the extension type of the values */ +__flatbuffers_define_vector_field(4, org_apache_arrow_flatbuf_DictionaryEncoding, custom_metadata, org_apache_arrow_flatbuf_KeyValue_vec_t, 0) /** ---------------------------------------------------------------------- * A field represents a named column in a record / row batch or child of a @@ -7119,6 +7174,7 @@ __flatbuffers_table_as_root(org_apache_arrow_flatbuf_Schema) * if endianness doesn't match the underlying system then the vectors need to be converted */ __flatbuffers_define_scalar_field(0, org_apache_arrow_flatbuf_Schema, endianness, org_apache_arrow_flatbuf_Endianness, org_apache_arrow_flatbuf_Endianness_enum_t, INT16_C(0)) __flatbuffers_define_vector_field(1, org_apache_arrow_flatbuf_Schema, fields, org_apache_arrow_flatbuf_Field_vec_t, 0) +/** User-defined metadata */ __flatbuffers_define_vector_field(2, org_apache_arrow_flatbuf_Schema, custom_metadata, org_apache_arrow_flatbuf_KeyValue_vec_t, 0) /** Features used in the stream/file. */ __flatbuffers_define_vector_field(3, org_apache_arrow_flatbuf_Schema, features, org_apache_arrow_flatbuf_Feature_vec_t, 0) @@ -7338,7 +7394,7 @@ __flatbuffers_build_table(flatbuffers_, org_apache_arrow_flatbuf_KeyValue, 2) static const flatbuffers_voffset_t __org_apache_arrow_flatbuf_DictionaryEncoding_required[] = { 0 }; typedef flatbuffers_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_ref_t; static org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_clone(flatbuffers_builder_t *B, org_apache_arrow_flatbuf_DictionaryEncoding_table_t t); -__flatbuffers_build_table(flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding, 4) +__flatbuffers_build_table(flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding, 5) static const flatbuffers_voffset_t __org_apache_arrow_flatbuf_Field_required[] = { 0 }; typedef flatbuffers_ref_t org_apache_arrow_flatbuf_Field_ref_t; @@ -7485,8 +7541,10 @@ __flatbuffers_build_table_prolog(flatbuffers_, org_apache_arrow_flatbuf_Duration static inline org_apache_arrow_flatbuf_KeyValue_ref_t org_apache_arrow_flatbuf_KeyValue_create(flatbuffers_builder_t *B __org_apache_arrow_flatbuf_KeyValue_formal_args); __flatbuffers_build_table_prolog(flatbuffers_, org_apache_arrow_flatbuf_KeyValue, org_apache_arrow_flatbuf_KeyValue_file_identifier, org_apache_arrow_flatbuf_KeyValue_type_identifier) -#define __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args , int64_t v0, org_apache_arrow_flatbuf_Int_ref_t v1, flatbuffers_bool_t v2, org_apache_arrow_flatbuf_DictionaryKind_enum_t v3 -#define __org_apache_arrow_flatbuf_DictionaryEncoding_call_args , v0, v1, v2, v3 +#define __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args ,\ + int64_t v0, org_apache_arrow_flatbuf_Int_ref_t v1, flatbuffers_bool_t v2, org_apache_arrow_flatbuf_DictionaryKind_enum_t v3, org_apache_arrow_flatbuf_KeyValue_vec_ref_t v4 +#define __org_apache_arrow_flatbuf_DictionaryEncoding_call_args ,\ + v0, v1, v2, v3, v4 static inline org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_create(flatbuffers_builder_t *B __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args); __flatbuffers_build_table_prolog(flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding, org_apache_arrow_flatbuf_DictionaryEncoding_file_identifier, org_apache_arrow_flatbuf_DictionaryEncoding_type_identifier) @@ -8141,12 +8199,14 @@ __flatbuffers_build_scalar_field(0, flatbuffers_, org_apache_arrow_flatbuf_Dicti __flatbuffers_build_table_field(1, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_indexType, org_apache_arrow_flatbuf_Int, org_apache_arrow_flatbuf_DictionaryEncoding) __flatbuffers_build_scalar_field(2, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_isOrdered, flatbuffers_bool, flatbuffers_bool_t, 1, 1, UINT8_C(0), org_apache_arrow_flatbuf_DictionaryEncoding) __flatbuffers_build_scalar_field(3, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind, org_apache_arrow_flatbuf_DictionaryKind, org_apache_arrow_flatbuf_DictionaryKind_enum_t, 2, 2, INT16_C(0), org_apache_arrow_flatbuf_DictionaryEncoding) +__flatbuffers_build_table_vector_field(4, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_custom_metadata, org_apache_arrow_flatbuf_KeyValue, org_apache_arrow_flatbuf_DictionaryEncoding) static inline org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_create(flatbuffers_builder_t *B __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args) { if (org_apache_arrow_flatbuf_DictionaryEncoding_start(B) || org_apache_arrow_flatbuf_DictionaryEncoding_id_add(B, v0) || org_apache_arrow_flatbuf_DictionaryEncoding_indexType_add(B, v1) + || org_apache_arrow_flatbuf_DictionaryEncoding_custom_metadata_add(B, v4) || org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind_add(B, v3) || org_apache_arrow_flatbuf_DictionaryEncoding_isOrdered_add(B, v2)) { return 0; @@ -8160,6 +8220,7 @@ static org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbu if (org_apache_arrow_flatbuf_DictionaryEncoding_start(B) || org_apache_arrow_flatbuf_DictionaryEncoding_id_pick(B, t) || org_apache_arrow_flatbuf_DictionaryEncoding_indexType_pick(B, t) + || org_apache_arrow_flatbuf_DictionaryEncoding_custom_metadata_pick(B, t) || org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind_pick(B, t) || org_apache_arrow_flatbuf_DictionaryEncoding_isOrdered_pick(B, t)) { return 0; @@ -9632,6 +9693,7 @@ static int org_apache_arrow_flatbuf_DictionaryEncoding_verify_table(flatcc_table if ((ret = flatcc_verify_table_field(td, 1, 0, &org_apache_arrow_flatbuf_Int_verify_table) /* indexType */)) return ret; if ((ret = flatcc_verify_field(td, 2, 1, 1) /* isOrdered */)) return ret; if ((ret = flatcc_verify_field(td, 3, 2, 2) /* dictionaryKind */)) return ret; + if ((ret = flatcc_verify_table_vector_field(td, 4, 0, &org_apache_arrow_flatbuf_KeyValue_verify_table) /* custom_metadata */)) return ret; return flatcc_verify_ok; } @@ -9854,6 +9916,8 @@ __flatbuffers_define_scalar_field(0, org_apache_arrow_flatbuf_TensorDim, size, f /** Name of the dimension, optional */ __flatbuffers_define_string_field(1, org_apache_arrow_flatbuf_TensorDim, name, 0) +/** DEPRECATED: The recommended way to pass tensors over Arrow IPC is + * using columns with the `arrow.fixed_shape_tensor` canonical type. */ struct org_apache_arrow_flatbuf_Tensor_table { uint8_t unused__; }; static inline size_t org_apache_arrow_flatbuf_Tensor_vec_len(org_apache_arrow_flatbuf_Tensor_vec_t vec) @@ -10416,6 +10480,11 @@ __flatbuffers_define_vector_field(4, org_apache_arrow_flatbuf_SparseTensorIndexC typedef uint8_t org_apache_arrow_flatbuf_SparseTensorIndex_union_type_t; __flatbuffers_define_integer_type(org_apache_arrow_flatbuf_SparseTensorIndex, org_apache_arrow_flatbuf_SparseTensorIndex_union_type_t, 8) __flatbuffers_define_union(flatbuffers_, org_apache_arrow_flatbuf_SparseTensorIndex) +/** DEPRECATED. Unlike dense tensors (see Tensor.fbs), the SparseTensor message + * currently doesn't have a recommended replacement as a RecordBatch column type. + * Feel free to discuss this on the development mailing-list (*) if this use case + * is important to you. + * (*) see https://arrow.apache.org/community/ */ #define org_apache_arrow_flatbuf_SparseTensorIndex_NONE ((org_apache_arrow_flatbuf_SparseTensorIndex_union_type_t)UINT8_C(0)) #define org_apache_arrow_flatbuf_SparseTensorIndex_SparseTensorIndexCOO ((org_apache_arrow_flatbuf_SparseTensorIndex_union_type_t)UINT8_C(1)) #define org_apache_arrow_flatbuf_SparseTensorIndex_SparseMatrixIndexCSX ((org_apache_arrow_flatbuf_SparseTensorIndex_union_type_t)UINT8_C(2)) @@ -11074,6 +11143,9 @@ typedef flatbuffers_uoffset_t *org_apache_arrow_flatbuf_Message_mutable_vec_t; typedef int8_t org_apache_arrow_flatbuf_CompressionType_enum_t; __flatbuffers_define_integer_type(org_apache_arrow_flatbuf_CompressionType, org_apache_arrow_flatbuf_CompressionType_enum_t, 8) +/** LZ4 frame format, for portability, as provided by lz4frame.h or wrappers + * thereof. Not to be confused with "raw" (also called "block") format + * provided by lz4.h */ #define org_apache_arrow_flatbuf_CompressionType_LZ4_FRAME ((org_apache_arrow_flatbuf_CompressionType_enum_t)INT8_C(0)) #define org_apache_arrow_flatbuf_CompressionType_ZSTD ((org_apache_arrow_flatbuf_CompressionType_enum_t)INT8_C(1)) @@ -11246,7 +11318,11 @@ __flatbuffers_define_scalar_field(2, org_apache_arrow_flatbuf_DictionaryBatch, i * * Arrow implementations do not need to implement all of the message types, * which may include experimental metadata types. For maximum compatibility, - * it is best to send data using RecordBatch */ + * it is best to send data using RecordBatch. + * + * Tensor and SparseTensor are DEPRECATED. The recommended way to pass dense + * tensors over Arrow IPC is a RecordBatch field with the `arrow.fixed_shape_tensor` + * canonical type. Sparse tensors currently do not have a recommended replacement. */ typedef uint8_t org_apache_arrow_flatbuf_MessageHeader_union_type_t; __flatbuffers_define_integer_type(org_apache_arrow_flatbuf_MessageHeader, org_apache_arrow_flatbuf_MessageHeader_union_type_t, 8) __flatbuffers_define_union(flatbuffers_, org_apache_arrow_flatbuf_MessageHeader) @@ -12526,8 +12602,14 @@ static inline int org_apache_arrow_flatbuf_TimeUnit_is_known_value(org_apache_ar } } +/** The unit of an Interval. + * + * All integers in the units below are stored in the endianness indicated + * by the schema. */ typedef int16_t org_apache_arrow_flatbuf_IntervalUnit_enum_t; __flatbuffers_define_integer_type(org_apache_arrow_flatbuf_IntervalUnit, org_apache_arrow_flatbuf_IntervalUnit_enum_t, 16) +/** Indicates the number of elapsed whole months, stored as + * 4-byte signed integers. */ #define org_apache_arrow_flatbuf_IntervalUnit_YEAR_MONTH ((org_apache_arrow_flatbuf_IntervalUnit_enum_t)INT16_C(0)) #define org_apache_arrow_flatbuf_IntervalUnit_DAY_TIME ((org_apache_arrow_flatbuf_IntervalUnit_enum_t)INT16_C(1)) #define org_apache_arrow_flatbuf_IntervalUnit_MONTH_DAY_NANO ((org_apache_arrow_flatbuf_IntervalUnit_enum_t)INT16_C(2)) @@ -13074,6 +13156,9 @@ __flatbuffers_define_scalar_field(0, org_apache_arrow_flatbuf_Timestamp, unit, o * the data (see above). */ __flatbuffers_define_string_field(1, org_apache_arrow_flatbuf_Timestamp, timezone, 0) +/** A "calendar" interval which models types that don't necessarily + * have a precise duration without the context of a base timestamp (e.g. + * days can differ in length during day light savings time transitions). */ struct org_apache_arrow_flatbuf_Interval_table { uint8_t unused__; }; static inline size_t org_apache_arrow_flatbuf_Interval_vec_len(org_apache_arrow_flatbuf_Interval_vec_t vec) @@ -13084,6 +13169,19 @@ __flatbuffers_table_as_root(org_apache_arrow_flatbuf_Interval) __flatbuffers_define_scalar_field(0, org_apache_arrow_flatbuf_Interval, unit, org_apache_arrow_flatbuf_IntervalUnit, org_apache_arrow_flatbuf_IntervalUnit_enum_t, INT16_C(0)) +/** An absolute length of time unrelated to any calendar artifacts. + * + * For the purposes of Arrow Implementations, adding this value to a Timestamp + * ("t1") naively (i.e. simply summing the two numbers) is acceptable even + * though in some cases the resulting Timestamp (t2) would not account for + * leap-seconds during the elapsed time between "t1" and "t2". Similarly, + * representing the difference between two Unix timestamps is acceptable, but + * would yield a value that is possibly a few seconds off from the true elapsed + * time. + * + * The resolution defaults to millisecond, but can be any of the other + * supported TimeUnit values as with Timestamp and Time types. This type is + * always represented as an 8-byte integer. */ struct org_apache_arrow_flatbuf_Duration_table { uint8_t unused__; }; static inline size_t org_apache_arrow_flatbuf_Duration_vec_len(org_apache_arrow_flatbuf_Duration_vec_t vec) @@ -13100,7 +13198,7 @@ typedef uint8_t org_apache_arrow_flatbuf_Type_union_type_t; __flatbuffers_define_integer_type(org_apache_arrow_flatbuf_Type, org_apache_arrow_flatbuf_Type_union_type_t, 8) __flatbuffers_define_union(flatbuffers_, org_apache_arrow_flatbuf_Type) /** ---------------------------------------------------------------------- - * user defined key value pairs to add custom metadata to arrow + * user defined key value pairs to add custom metadata to Arrow * key namespacing is the responsibility of the user */ #define org_apache_arrow_flatbuf_Type_NONE ((org_apache_arrow_flatbuf_Type_union_type_t)UINT8_C(0)) #define org_apache_arrow_flatbuf_Type_Null ((org_apache_arrow_flatbuf_Type_union_type_t)UINT8_C(1)) @@ -13234,6 +13332,8 @@ __flatbuffers_define_table_field(1, org_apache_arrow_flatbuf_DictionaryEncoding, * preserve that metadata here */ __flatbuffers_define_scalar_field(2, org_apache_arrow_flatbuf_DictionaryEncoding, isOrdered, flatbuffers_bool, flatbuffers_bool_t, UINT8_C(0)) __flatbuffers_define_scalar_field(3, org_apache_arrow_flatbuf_DictionaryEncoding, dictionaryKind, org_apache_arrow_flatbuf_DictionaryKind, org_apache_arrow_flatbuf_DictionaryKind_enum_t, INT16_C(0)) +/** Metadata for the dictionary value type, e.g. the extension type of the values */ +__flatbuffers_define_vector_field(4, org_apache_arrow_flatbuf_DictionaryEncoding, custom_metadata, org_apache_arrow_flatbuf_KeyValue_vec_t, 0) /** ---------------------------------------------------------------------- * A field represents a named column in a record / row batch or child of a @@ -13275,6 +13375,7 @@ __flatbuffers_table_as_root(org_apache_arrow_flatbuf_Schema) * if endianness doesn't match the underlying system then the vectors need to be converted */ __flatbuffers_define_scalar_field(0, org_apache_arrow_flatbuf_Schema, endianness, org_apache_arrow_flatbuf_Endianness, org_apache_arrow_flatbuf_Endianness_enum_t, INT16_C(0)) __flatbuffers_define_vector_field(1, org_apache_arrow_flatbuf_Schema, fields, org_apache_arrow_flatbuf_Field_vec_t, 0) +/** User-defined metadata */ __flatbuffers_define_vector_field(2, org_apache_arrow_flatbuf_Schema, custom_metadata, org_apache_arrow_flatbuf_KeyValue_vec_t, 0) /** Features used in the stream/file. */ __flatbuffers_define_vector_field(3, org_apache_arrow_flatbuf_Schema, features, org_apache_arrow_flatbuf_Feature_vec_t, 0) @@ -13494,7 +13595,7 @@ __flatbuffers_build_table(flatbuffers_, org_apache_arrow_flatbuf_KeyValue, 2) static const flatbuffers_voffset_t __org_apache_arrow_flatbuf_DictionaryEncoding_required[] = { 0 }; typedef flatbuffers_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_ref_t; static org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_clone(flatbuffers_builder_t *B, org_apache_arrow_flatbuf_DictionaryEncoding_table_t t); -__flatbuffers_build_table(flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding, 4) +__flatbuffers_build_table(flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding, 5) static const flatbuffers_voffset_t __org_apache_arrow_flatbuf_Field_required[] = { 0 }; typedef flatbuffers_ref_t org_apache_arrow_flatbuf_Field_ref_t; @@ -13641,8 +13742,10 @@ __flatbuffers_build_table_prolog(flatbuffers_, org_apache_arrow_flatbuf_Duration static inline org_apache_arrow_flatbuf_KeyValue_ref_t org_apache_arrow_flatbuf_KeyValue_create(flatbuffers_builder_t *B __org_apache_arrow_flatbuf_KeyValue_formal_args); __flatbuffers_build_table_prolog(flatbuffers_, org_apache_arrow_flatbuf_KeyValue, org_apache_arrow_flatbuf_KeyValue_file_identifier, org_apache_arrow_flatbuf_KeyValue_type_identifier) -#define __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args , int64_t v0, org_apache_arrow_flatbuf_Int_ref_t v1, flatbuffers_bool_t v2, org_apache_arrow_flatbuf_DictionaryKind_enum_t v3 -#define __org_apache_arrow_flatbuf_DictionaryEncoding_call_args , v0, v1, v2, v3 +#define __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args ,\ + int64_t v0, org_apache_arrow_flatbuf_Int_ref_t v1, flatbuffers_bool_t v2, org_apache_arrow_flatbuf_DictionaryKind_enum_t v3, org_apache_arrow_flatbuf_KeyValue_vec_ref_t v4 +#define __org_apache_arrow_flatbuf_DictionaryEncoding_call_args ,\ + v0, v1, v2, v3, v4 static inline org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_create(flatbuffers_builder_t *B __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args); __flatbuffers_build_table_prolog(flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding, org_apache_arrow_flatbuf_DictionaryEncoding_file_identifier, org_apache_arrow_flatbuf_DictionaryEncoding_type_identifier) @@ -14297,12 +14400,14 @@ __flatbuffers_build_scalar_field(0, flatbuffers_, org_apache_arrow_flatbuf_Dicti __flatbuffers_build_table_field(1, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_indexType, org_apache_arrow_flatbuf_Int, org_apache_arrow_flatbuf_DictionaryEncoding) __flatbuffers_build_scalar_field(2, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_isOrdered, flatbuffers_bool, flatbuffers_bool_t, 1, 1, UINT8_C(0), org_apache_arrow_flatbuf_DictionaryEncoding) __flatbuffers_build_scalar_field(3, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind, org_apache_arrow_flatbuf_DictionaryKind, org_apache_arrow_flatbuf_DictionaryKind_enum_t, 2, 2, INT16_C(0), org_apache_arrow_flatbuf_DictionaryEncoding) +__flatbuffers_build_table_vector_field(4, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_custom_metadata, org_apache_arrow_flatbuf_KeyValue, org_apache_arrow_flatbuf_DictionaryEncoding) static inline org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_create(flatbuffers_builder_t *B __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args) { if (org_apache_arrow_flatbuf_DictionaryEncoding_start(B) || org_apache_arrow_flatbuf_DictionaryEncoding_id_add(B, v0) || org_apache_arrow_flatbuf_DictionaryEncoding_indexType_add(B, v1) + || org_apache_arrow_flatbuf_DictionaryEncoding_custom_metadata_add(B, v4) || org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind_add(B, v3) || org_apache_arrow_flatbuf_DictionaryEncoding_isOrdered_add(B, v2)) { return 0; @@ -14316,6 +14421,7 @@ static org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbu if (org_apache_arrow_flatbuf_DictionaryEncoding_start(B) || org_apache_arrow_flatbuf_DictionaryEncoding_id_pick(B, t) || org_apache_arrow_flatbuf_DictionaryEncoding_indexType_pick(B, t) + || org_apache_arrow_flatbuf_DictionaryEncoding_custom_metadata_pick(B, t) || org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind_pick(B, t) || org_apache_arrow_flatbuf_DictionaryEncoding_isOrdered_pick(B, t)) { return 0; @@ -15788,6 +15894,7 @@ static int org_apache_arrow_flatbuf_DictionaryEncoding_verify_table(flatcc_table if ((ret = flatcc_verify_table_field(td, 1, 0, &org_apache_arrow_flatbuf_Int_verify_table) /* indexType */)) return ret; if ((ret = flatcc_verify_field(td, 2, 1, 1) /* isOrdered */)) return ret; if ((ret = flatcc_verify_field(td, 3, 2, 2) /* dictionaryKind */)) return ret; + if ((ret = flatcc_verify_table_vector_field(td, 4, 0, &org_apache_arrow_flatbuf_KeyValue_verify_table) /* custom_metadata */)) return ret; return flatcc_verify_ok; } @@ -16632,8 +16739,14 @@ static inline int org_apache_arrow_flatbuf_TimeUnit_is_known_value(org_apache_ar } } +/** The unit of an Interval. + * + * All integers in the units below are stored in the endianness indicated + * by the schema. */ typedef int16_t org_apache_arrow_flatbuf_IntervalUnit_enum_t; __flatbuffers_define_integer_type(org_apache_arrow_flatbuf_IntervalUnit, org_apache_arrow_flatbuf_IntervalUnit_enum_t, 16) +/** Indicates the number of elapsed whole months, stored as + * 4-byte signed integers. */ #define org_apache_arrow_flatbuf_IntervalUnit_YEAR_MONTH ((org_apache_arrow_flatbuf_IntervalUnit_enum_t)INT16_C(0)) #define org_apache_arrow_flatbuf_IntervalUnit_DAY_TIME ((org_apache_arrow_flatbuf_IntervalUnit_enum_t)INT16_C(1)) #define org_apache_arrow_flatbuf_IntervalUnit_MONTH_DAY_NANO ((org_apache_arrow_flatbuf_IntervalUnit_enum_t)INT16_C(2)) @@ -17180,6 +17293,9 @@ __flatbuffers_define_scalar_field(0, org_apache_arrow_flatbuf_Timestamp, unit, o * the data (see above). */ __flatbuffers_define_string_field(1, org_apache_arrow_flatbuf_Timestamp, timezone, 0) +/** A "calendar" interval which models types that don't necessarily + * have a precise duration without the context of a base timestamp (e.g. + * days can differ in length during day light savings time transitions). */ struct org_apache_arrow_flatbuf_Interval_table { uint8_t unused__; }; static inline size_t org_apache_arrow_flatbuf_Interval_vec_len(org_apache_arrow_flatbuf_Interval_vec_t vec) @@ -17190,6 +17306,19 @@ __flatbuffers_table_as_root(org_apache_arrow_flatbuf_Interval) __flatbuffers_define_scalar_field(0, org_apache_arrow_flatbuf_Interval, unit, org_apache_arrow_flatbuf_IntervalUnit, org_apache_arrow_flatbuf_IntervalUnit_enum_t, INT16_C(0)) +/** An absolute length of time unrelated to any calendar artifacts. + * + * For the purposes of Arrow Implementations, adding this value to a Timestamp + * ("t1") naively (i.e. simply summing the two numbers) is acceptable even + * though in some cases the resulting Timestamp (t2) would not account for + * leap-seconds during the elapsed time between "t1" and "t2". Similarly, + * representing the difference between two Unix timestamps is acceptable, but + * would yield a value that is possibly a few seconds off from the true elapsed + * time. + * + * The resolution defaults to millisecond, but can be any of the other + * supported TimeUnit values as with Timestamp and Time types. This type is + * always represented as an 8-byte integer. */ struct org_apache_arrow_flatbuf_Duration_table { uint8_t unused__; }; static inline size_t org_apache_arrow_flatbuf_Duration_vec_len(org_apache_arrow_flatbuf_Duration_vec_t vec) @@ -17206,7 +17335,7 @@ typedef uint8_t org_apache_arrow_flatbuf_Type_union_type_t; __flatbuffers_define_integer_type(org_apache_arrow_flatbuf_Type, org_apache_arrow_flatbuf_Type_union_type_t, 8) __flatbuffers_define_union(flatbuffers_, org_apache_arrow_flatbuf_Type) /** ---------------------------------------------------------------------- - * user defined key value pairs to add custom metadata to arrow + * user defined key value pairs to add custom metadata to Arrow * key namespacing is the responsibility of the user */ #define org_apache_arrow_flatbuf_Type_NONE ((org_apache_arrow_flatbuf_Type_union_type_t)UINT8_C(0)) #define org_apache_arrow_flatbuf_Type_Null ((org_apache_arrow_flatbuf_Type_union_type_t)UINT8_C(1)) @@ -17340,6 +17469,8 @@ __flatbuffers_define_table_field(1, org_apache_arrow_flatbuf_DictionaryEncoding, * preserve that metadata here */ __flatbuffers_define_scalar_field(2, org_apache_arrow_flatbuf_DictionaryEncoding, isOrdered, flatbuffers_bool, flatbuffers_bool_t, UINT8_C(0)) __flatbuffers_define_scalar_field(3, org_apache_arrow_flatbuf_DictionaryEncoding, dictionaryKind, org_apache_arrow_flatbuf_DictionaryKind, org_apache_arrow_flatbuf_DictionaryKind_enum_t, INT16_C(0)) +/** Metadata for the dictionary value type, e.g. the extension type of the values */ +__flatbuffers_define_vector_field(4, org_apache_arrow_flatbuf_DictionaryEncoding, custom_metadata, org_apache_arrow_flatbuf_KeyValue_vec_t, 0) /** ---------------------------------------------------------------------- * A field represents a named column in a record / row batch or child of a @@ -17381,6 +17512,7 @@ __flatbuffers_table_as_root(org_apache_arrow_flatbuf_Schema) * if endianness doesn't match the underlying system then the vectors need to be converted */ __flatbuffers_define_scalar_field(0, org_apache_arrow_flatbuf_Schema, endianness, org_apache_arrow_flatbuf_Endianness, org_apache_arrow_flatbuf_Endianness_enum_t, INT16_C(0)) __flatbuffers_define_vector_field(1, org_apache_arrow_flatbuf_Schema, fields, org_apache_arrow_flatbuf_Field_vec_t, 0) +/** User-defined metadata */ __flatbuffers_define_vector_field(2, org_apache_arrow_flatbuf_Schema, custom_metadata, org_apache_arrow_flatbuf_KeyValue_vec_t, 0) /** Features used in the stream/file. */ __flatbuffers_define_vector_field(3, org_apache_arrow_flatbuf_Schema, features, org_apache_arrow_flatbuf_Feature_vec_t, 0) @@ -17600,7 +17732,7 @@ __flatbuffers_build_table(flatbuffers_, org_apache_arrow_flatbuf_KeyValue, 2) static const flatbuffers_voffset_t __org_apache_arrow_flatbuf_DictionaryEncoding_required[] = { 0 }; typedef flatbuffers_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_ref_t; static org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_clone(flatbuffers_builder_t *B, org_apache_arrow_flatbuf_DictionaryEncoding_table_t t); -__flatbuffers_build_table(flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding, 4) +__flatbuffers_build_table(flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding, 5) static const flatbuffers_voffset_t __org_apache_arrow_flatbuf_Field_required[] = { 0 }; typedef flatbuffers_ref_t org_apache_arrow_flatbuf_Field_ref_t; @@ -17747,8 +17879,10 @@ __flatbuffers_build_table_prolog(flatbuffers_, org_apache_arrow_flatbuf_Duration static inline org_apache_arrow_flatbuf_KeyValue_ref_t org_apache_arrow_flatbuf_KeyValue_create(flatbuffers_builder_t *B __org_apache_arrow_flatbuf_KeyValue_formal_args); __flatbuffers_build_table_prolog(flatbuffers_, org_apache_arrow_flatbuf_KeyValue, org_apache_arrow_flatbuf_KeyValue_file_identifier, org_apache_arrow_flatbuf_KeyValue_type_identifier) -#define __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args , int64_t v0, org_apache_arrow_flatbuf_Int_ref_t v1, flatbuffers_bool_t v2, org_apache_arrow_flatbuf_DictionaryKind_enum_t v3 -#define __org_apache_arrow_flatbuf_DictionaryEncoding_call_args , v0, v1, v2, v3 +#define __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args ,\ + int64_t v0, org_apache_arrow_flatbuf_Int_ref_t v1, flatbuffers_bool_t v2, org_apache_arrow_flatbuf_DictionaryKind_enum_t v3, org_apache_arrow_flatbuf_KeyValue_vec_ref_t v4 +#define __org_apache_arrow_flatbuf_DictionaryEncoding_call_args ,\ + v0, v1, v2, v3, v4 static inline org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_create(flatbuffers_builder_t *B __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args); __flatbuffers_build_table_prolog(flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding, org_apache_arrow_flatbuf_DictionaryEncoding_file_identifier, org_apache_arrow_flatbuf_DictionaryEncoding_type_identifier) @@ -18403,12 +18537,14 @@ __flatbuffers_build_scalar_field(0, flatbuffers_, org_apache_arrow_flatbuf_Dicti __flatbuffers_build_table_field(1, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_indexType, org_apache_arrow_flatbuf_Int, org_apache_arrow_flatbuf_DictionaryEncoding) __flatbuffers_build_scalar_field(2, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_isOrdered, flatbuffers_bool, flatbuffers_bool_t, 1, 1, UINT8_C(0), org_apache_arrow_flatbuf_DictionaryEncoding) __flatbuffers_build_scalar_field(3, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind, org_apache_arrow_flatbuf_DictionaryKind, org_apache_arrow_flatbuf_DictionaryKind_enum_t, 2, 2, INT16_C(0), org_apache_arrow_flatbuf_DictionaryEncoding) +__flatbuffers_build_table_vector_field(4, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_custom_metadata, org_apache_arrow_flatbuf_KeyValue, org_apache_arrow_flatbuf_DictionaryEncoding) static inline org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_create(flatbuffers_builder_t *B __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args) { if (org_apache_arrow_flatbuf_DictionaryEncoding_start(B) || org_apache_arrow_flatbuf_DictionaryEncoding_id_add(B, v0) || org_apache_arrow_flatbuf_DictionaryEncoding_indexType_add(B, v1) + || org_apache_arrow_flatbuf_DictionaryEncoding_custom_metadata_add(B, v4) || org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind_add(B, v3) || org_apache_arrow_flatbuf_DictionaryEncoding_isOrdered_add(B, v2)) { return 0; @@ -18422,6 +18558,7 @@ static org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbu if (org_apache_arrow_flatbuf_DictionaryEncoding_start(B) || org_apache_arrow_flatbuf_DictionaryEncoding_id_pick(B, t) || org_apache_arrow_flatbuf_DictionaryEncoding_indexType_pick(B, t) + || org_apache_arrow_flatbuf_DictionaryEncoding_custom_metadata_pick(B, t) || org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind_pick(B, t) || org_apache_arrow_flatbuf_DictionaryEncoding_isOrdered_pick(B, t)) { return 0; @@ -19894,6 +20031,7 @@ static int org_apache_arrow_flatbuf_DictionaryEncoding_verify_table(flatcc_table if ((ret = flatcc_verify_table_field(td, 1, 0, &org_apache_arrow_flatbuf_Int_verify_table) /* indexType */)) return ret; if ((ret = flatcc_verify_field(td, 2, 1, 1) /* isOrdered */)) return ret; if ((ret = flatcc_verify_field(td, 3, 2, 2) /* dictionaryKind */)) return ret; + if ((ret = flatcc_verify_table_vector_field(td, 4, 0, &org_apache_arrow_flatbuf_KeyValue_verify_table) /* custom_metadata */)) return ret; return flatcc_verify_ok; } @@ -20116,6 +20254,8 @@ __flatbuffers_define_scalar_field(0, org_apache_arrow_flatbuf_TensorDim, size, f /** Name of the dimension, optional */ __flatbuffers_define_string_field(1, org_apache_arrow_flatbuf_TensorDim, name, 0) +/** DEPRECATED: The recommended way to pass tensors over Arrow IPC is + * using columns with the `arrow.fixed_shape_tensor` canonical type. */ struct org_apache_arrow_flatbuf_Tensor_table { uint8_t unused__; }; static inline size_t org_apache_arrow_flatbuf_Tensor_vec_len(org_apache_arrow_flatbuf_Tensor_vec_t vec) @@ -20678,6 +20818,11 @@ __flatbuffers_define_vector_field(4, org_apache_arrow_flatbuf_SparseTensorIndexC typedef uint8_t org_apache_arrow_flatbuf_SparseTensorIndex_union_type_t; __flatbuffers_define_integer_type(org_apache_arrow_flatbuf_SparseTensorIndex, org_apache_arrow_flatbuf_SparseTensorIndex_union_type_t, 8) __flatbuffers_define_union(flatbuffers_, org_apache_arrow_flatbuf_SparseTensorIndex) +/** DEPRECATED. Unlike dense tensors (see Tensor.fbs), the SparseTensor message + * currently doesn't have a recommended replacement as a RecordBatch column type. + * Feel free to discuss this on the development mailing-list (*) if this use case + * is important to you. + * (*) see https://arrow.apache.org/community/ */ #define org_apache_arrow_flatbuf_SparseTensorIndex_NONE ((org_apache_arrow_flatbuf_SparseTensorIndex_union_type_t)UINT8_C(0)) #define org_apache_arrow_flatbuf_SparseTensorIndex_SparseTensorIndexCOO ((org_apache_arrow_flatbuf_SparseTensorIndex_union_type_t)UINT8_C(1)) #define org_apache_arrow_flatbuf_SparseTensorIndex_SparseMatrixIndexCSX ((org_apache_arrow_flatbuf_SparseTensorIndex_union_type_t)UINT8_C(2)) @@ -21919,8 +22064,14 @@ static inline int org_apache_arrow_flatbuf_TimeUnit_is_known_value(org_apache_ar } } +/** The unit of an Interval. + * + * All integers in the units below are stored in the endianness indicated + * by the schema. */ typedef int16_t org_apache_arrow_flatbuf_IntervalUnit_enum_t; __flatbuffers_define_integer_type(org_apache_arrow_flatbuf_IntervalUnit, org_apache_arrow_flatbuf_IntervalUnit_enum_t, 16) +/** Indicates the number of elapsed whole months, stored as + * 4-byte signed integers. */ #define org_apache_arrow_flatbuf_IntervalUnit_YEAR_MONTH ((org_apache_arrow_flatbuf_IntervalUnit_enum_t)INT16_C(0)) #define org_apache_arrow_flatbuf_IntervalUnit_DAY_TIME ((org_apache_arrow_flatbuf_IntervalUnit_enum_t)INT16_C(1)) #define org_apache_arrow_flatbuf_IntervalUnit_MONTH_DAY_NANO ((org_apache_arrow_flatbuf_IntervalUnit_enum_t)INT16_C(2)) @@ -22467,6 +22618,9 @@ __flatbuffers_define_scalar_field(0, org_apache_arrow_flatbuf_Timestamp, unit, o * the data (see above). */ __flatbuffers_define_string_field(1, org_apache_arrow_flatbuf_Timestamp, timezone, 0) +/** A "calendar" interval which models types that don't necessarily + * have a precise duration without the context of a base timestamp (e.g. + * days can differ in length during day light savings time transitions). */ struct org_apache_arrow_flatbuf_Interval_table { uint8_t unused__; }; static inline size_t org_apache_arrow_flatbuf_Interval_vec_len(org_apache_arrow_flatbuf_Interval_vec_t vec) @@ -22477,6 +22631,19 @@ __flatbuffers_table_as_root(org_apache_arrow_flatbuf_Interval) __flatbuffers_define_scalar_field(0, org_apache_arrow_flatbuf_Interval, unit, org_apache_arrow_flatbuf_IntervalUnit, org_apache_arrow_flatbuf_IntervalUnit_enum_t, INT16_C(0)) +/** An absolute length of time unrelated to any calendar artifacts. + * + * For the purposes of Arrow Implementations, adding this value to a Timestamp + * ("t1") naively (i.e. simply summing the two numbers) is acceptable even + * though in some cases the resulting Timestamp (t2) would not account for + * leap-seconds during the elapsed time between "t1" and "t2". Similarly, + * representing the difference between two Unix timestamps is acceptable, but + * would yield a value that is possibly a few seconds off from the true elapsed + * time. + * + * The resolution defaults to millisecond, but can be any of the other + * supported TimeUnit values as with Timestamp and Time types. This type is + * always represented as an 8-byte integer. */ struct org_apache_arrow_flatbuf_Duration_table { uint8_t unused__; }; static inline size_t org_apache_arrow_flatbuf_Duration_vec_len(org_apache_arrow_flatbuf_Duration_vec_t vec) @@ -22493,7 +22660,7 @@ typedef uint8_t org_apache_arrow_flatbuf_Type_union_type_t; __flatbuffers_define_integer_type(org_apache_arrow_flatbuf_Type, org_apache_arrow_flatbuf_Type_union_type_t, 8) __flatbuffers_define_union(flatbuffers_, org_apache_arrow_flatbuf_Type) /** ---------------------------------------------------------------------- - * user defined key value pairs to add custom metadata to arrow + * user defined key value pairs to add custom metadata to Arrow * key namespacing is the responsibility of the user */ #define org_apache_arrow_flatbuf_Type_NONE ((org_apache_arrow_flatbuf_Type_union_type_t)UINT8_C(0)) #define org_apache_arrow_flatbuf_Type_Null ((org_apache_arrow_flatbuf_Type_union_type_t)UINT8_C(1)) @@ -22627,6 +22794,8 @@ __flatbuffers_define_table_field(1, org_apache_arrow_flatbuf_DictionaryEncoding, * preserve that metadata here */ __flatbuffers_define_scalar_field(2, org_apache_arrow_flatbuf_DictionaryEncoding, isOrdered, flatbuffers_bool, flatbuffers_bool_t, UINT8_C(0)) __flatbuffers_define_scalar_field(3, org_apache_arrow_flatbuf_DictionaryEncoding, dictionaryKind, org_apache_arrow_flatbuf_DictionaryKind, org_apache_arrow_flatbuf_DictionaryKind_enum_t, INT16_C(0)) +/** Metadata for the dictionary value type, e.g. the extension type of the values */ +__flatbuffers_define_vector_field(4, org_apache_arrow_flatbuf_DictionaryEncoding, custom_metadata, org_apache_arrow_flatbuf_KeyValue_vec_t, 0) /** ---------------------------------------------------------------------- * A field represents a named column in a record / row batch or child of a @@ -22668,6 +22837,7 @@ __flatbuffers_table_as_root(org_apache_arrow_flatbuf_Schema) * if endianness doesn't match the underlying system then the vectors need to be converted */ __flatbuffers_define_scalar_field(0, org_apache_arrow_flatbuf_Schema, endianness, org_apache_arrow_flatbuf_Endianness, org_apache_arrow_flatbuf_Endianness_enum_t, INT16_C(0)) __flatbuffers_define_vector_field(1, org_apache_arrow_flatbuf_Schema, fields, org_apache_arrow_flatbuf_Field_vec_t, 0) +/** User-defined metadata */ __flatbuffers_define_vector_field(2, org_apache_arrow_flatbuf_Schema, custom_metadata, org_apache_arrow_flatbuf_KeyValue_vec_t, 0) /** Features used in the stream/file. */ __flatbuffers_define_vector_field(3, org_apache_arrow_flatbuf_Schema, features, org_apache_arrow_flatbuf_Feature_vec_t, 0) @@ -22887,7 +23057,7 @@ __flatbuffers_build_table(flatbuffers_, org_apache_arrow_flatbuf_KeyValue, 2) static const flatbuffers_voffset_t __org_apache_arrow_flatbuf_DictionaryEncoding_required[] = { 0 }; typedef flatbuffers_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_ref_t; static org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_clone(flatbuffers_builder_t *B, org_apache_arrow_flatbuf_DictionaryEncoding_table_t t); -__flatbuffers_build_table(flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding, 4) +__flatbuffers_build_table(flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding, 5) static const flatbuffers_voffset_t __org_apache_arrow_flatbuf_Field_required[] = { 0 }; typedef flatbuffers_ref_t org_apache_arrow_flatbuf_Field_ref_t; @@ -23034,8 +23204,10 @@ __flatbuffers_build_table_prolog(flatbuffers_, org_apache_arrow_flatbuf_Duration static inline org_apache_arrow_flatbuf_KeyValue_ref_t org_apache_arrow_flatbuf_KeyValue_create(flatbuffers_builder_t *B __org_apache_arrow_flatbuf_KeyValue_formal_args); __flatbuffers_build_table_prolog(flatbuffers_, org_apache_arrow_flatbuf_KeyValue, org_apache_arrow_flatbuf_KeyValue_file_identifier, org_apache_arrow_flatbuf_KeyValue_type_identifier) -#define __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args , int64_t v0, org_apache_arrow_flatbuf_Int_ref_t v1, flatbuffers_bool_t v2, org_apache_arrow_flatbuf_DictionaryKind_enum_t v3 -#define __org_apache_arrow_flatbuf_DictionaryEncoding_call_args , v0, v1, v2, v3 +#define __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args ,\ + int64_t v0, org_apache_arrow_flatbuf_Int_ref_t v1, flatbuffers_bool_t v2, org_apache_arrow_flatbuf_DictionaryKind_enum_t v3, org_apache_arrow_flatbuf_KeyValue_vec_ref_t v4 +#define __org_apache_arrow_flatbuf_DictionaryEncoding_call_args ,\ + v0, v1, v2, v3, v4 static inline org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_create(flatbuffers_builder_t *B __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args); __flatbuffers_build_table_prolog(flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding, org_apache_arrow_flatbuf_DictionaryEncoding_file_identifier, org_apache_arrow_flatbuf_DictionaryEncoding_type_identifier) @@ -23690,12 +23862,14 @@ __flatbuffers_build_scalar_field(0, flatbuffers_, org_apache_arrow_flatbuf_Dicti __flatbuffers_build_table_field(1, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_indexType, org_apache_arrow_flatbuf_Int, org_apache_arrow_flatbuf_DictionaryEncoding) __flatbuffers_build_scalar_field(2, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_isOrdered, flatbuffers_bool, flatbuffers_bool_t, 1, 1, UINT8_C(0), org_apache_arrow_flatbuf_DictionaryEncoding) __flatbuffers_build_scalar_field(3, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind, org_apache_arrow_flatbuf_DictionaryKind, org_apache_arrow_flatbuf_DictionaryKind_enum_t, 2, 2, INT16_C(0), org_apache_arrow_flatbuf_DictionaryEncoding) +__flatbuffers_build_table_vector_field(4, flatbuffers_, org_apache_arrow_flatbuf_DictionaryEncoding_custom_metadata, org_apache_arrow_flatbuf_KeyValue, org_apache_arrow_flatbuf_DictionaryEncoding) static inline org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbuf_DictionaryEncoding_create(flatbuffers_builder_t *B __org_apache_arrow_flatbuf_DictionaryEncoding_formal_args) { if (org_apache_arrow_flatbuf_DictionaryEncoding_start(B) || org_apache_arrow_flatbuf_DictionaryEncoding_id_add(B, v0) || org_apache_arrow_flatbuf_DictionaryEncoding_indexType_add(B, v1) + || org_apache_arrow_flatbuf_DictionaryEncoding_custom_metadata_add(B, v4) || org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind_add(B, v3) || org_apache_arrow_flatbuf_DictionaryEncoding_isOrdered_add(B, v2)) { return 0; @@ -23709,6 +23883,7 @@ static org_apache_arrow_flatbuf_DictionaryEncoding_ref_t org_apache_arrow_flatbu if (org_apache_arrow_flatbuf_DictionaryEncoding_start(B) || org_apache_arrow_flatbuf_DictionaryEncoding_id_pick(B, t) || org_apache_arrow_flatbuf_DictionaryEncoding_indexType_pick(B, t) + || org_apache_arrow_flatbuf_DictionaryEncoding_custom_metadata_pick(B, t) || org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind_pick(B, t) || org_apache_arrow_flatbuf_DictionaryEncoding_isOrdered_pick(B, t)) { return 0; @@ -25181,6 +25356,7 @@ static int org_apache_arrow_flatbuf_DictionaryEncoding_verify_table(flatcc_table if ((ret = flatcc_verify_table_field(td, 1, 0, &org_apache_arrow_flatbuf_Int_verify_table) /* indexType */)) return ret; if ((ret = flatcc_verify_field(td, 2, 1, 1) /* isOrdered */)) return ret; if ((ret = flatcc_verify_field(td, 3, 2, 2) /* dictionaryKind */)) return ret; + if ((ret = flatcc_verify_table_vector_field(td, 4, 0, &org_apache_arrow_flatbuf_KeyValue_verify_table) /* custom_metadata */)) return ret; return flatcc_verify_ok; } @@ -25403,6 +25579,8 @@ __flatbuffers_define_scalar_field(0, org_apache_arrow_flatbuf_TensorDim, size, f /** Name of the dimension, optional */ __flatbuffers_define_string_field(1, org_apache_arrow_flatbuf_TensorDim, name, 0) +/** DEPRECATED: The recommended way to pass tensors over Arrow IPC is + * using columns with the `arrow.fixed_shape_tensor` canonical type. */ struct org_apache_arrow_flatbuf_Tensor_table { uint8_t unused__; }; static inline size_t org_apache_arrow_flatbuf_Tensor_vec_len(org_apache_arrow_flatbuf_Tensor_vec_t vec) diff --git a/src/nanoarrow/ipc/writer_test.cc b/src/nanoarrow/ipc/writer_test.cc index edcc8839c..04bdc9737 100644 --- a/src/nanoarrow/ipc/writer_test.cc +++ b/src/nanoarrow/ipc/writer_test.cc @@ -409,6 +409,95 @@ TEST(NanoarrowIpcWriter, RoundtripDictionaryStream) { EXPECT_EQ(roundtrip_array->release, nullptr); } +// The metadata of the dictionary value schema (e.g., the extension type of the values) +// is written to DictionaryEncoding.custom_metadata and read back separately from the +// field metadata (https://github.com/apache/arrow/issues/49704) +TEST(NanoarrowIpcWriter, RoundtripDictionaryValueMetadataStream) { + struct ArrowError error; + + nanoarrow::UniqueSchema schema; + nanoarrow::UniqueArray array; + MakeDictionaryStructArray(array.get(), schema.get()); + + nanoarrow::UniqueBuffer value_metadata; + ASSERT_EQ(ArrowMetadataBuilderInit(value_metadata.get(), nullptr), NANOARROW_OK); + ASSERT_EQ(ArrowMetadataBuilderAppend(value_metadata.get(), + ArrowCharView("ARROW:extension:name"), + ArrowCharView("arrow.opaque")), + NANOARROW_OK); + ASSERT_EQ(ArrowMetadataBuilderAppend( + value_metadata.get(), ArrowCharView("ARROW:extension:metadata"), + ArrowCharView(R"({"type_name":"geometry","vendor_name":"nanoarrow"})")), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetMetadata(schema->children[0]->dictionary, + reinterpret_cast(value_metadata->data)), + NANOARROW_OK); + + nanoarrow::UniqueBuffer field_metadata; + ASSERT_EQ(ArrowMetadataBuilderInit(field_metadata.get(), nullptr), NANOARROW_OK); + ASSERT_EQ(ArrowMetadataBuilderAppend(field_metadata.get(), ArrowCharView("field_key"), + ArrowCharView("field_value")), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetMetadata(schema->children[0], + reinterpret_cast(field_metadata->data)), + NANOARROW_OK); + + nanoarrow::UniqueArrayStream array_stream; + ASSERT_EQ(ArrowBasicArrayStreamInit(array_stream.get(), schema.get(), 1), NANOARROW_OK); + ArrowBasicArrayStreamSetArray(array_stream.get(), 0, array.get()); + + nanoarrow::UniqueBuffer output; + nanoarrow::ipc::UniqueOutputStream out_stream; + ASSERT_EQ(ArrowIpcOutputStreamInitBuffer(out_stream.get(), output.get()), NANOARROW_OK); + + nanoarrow::ipc::UniqueWriter writer; + ASSERT_EQ(ArrowIpcWriterInit(writer.get(), out_stream.get()), NANOARROW_OK); + ASSERT_EQ(ArrowIpcWriterWriteArrayStream(writer.get(), array_stream.get(), &error), + NANOARROW_OK) + << error.message; + + struct ArrowIpcInputStream input; + ASSERT_EQ(ArrowIpcInputStreamInitBuffer(&input, output.get()), NANOARROW_OK); + + nanoarrow::UniqueArrayStream reader; + ASSERT_EQ(ArrowIpcArrayStreamReaderInit(reader.get(), &input, nullptr), NANOARROW_OK); + + nanoarrow::UniqueSchema roundtrip_schema; + ASSERT_EQ(ArrowArrayStreamGetSchema(reader.get(), roundtrip_schema.get(), &error), + NANOARROW_OK) + << error.message; + ASSERT_EQ(roundtrip_schema->n_children, 1); + ASSERT_NE(roundtrip_schema->children[0]->dictionary, nullptr); + + struct ArrowStringView value; + ASSERT_EQ(ArrowMetadataGetValue(roundtrip_schema->children[0]->metadata, + ArrowCharView("field_key"), &value), + NANOARROW_OK); + EXPECT_EQ(std::string(value.data, value.size_bytes), "field_value"); + EXPECT_EQ(ArrowMetadataHasKey(roundtrip_schema->children[0]->metadata, + ArrowCharView("ARROW:extension:name")), + 0); + + const char* roundtrip_value_metadata = + roundtrip_schema->children[0]->dictionary->metadata; + ASSERT_NE(roundtrip_value_metadata, nullptr); + EXPECT_EQ(ArrowMetadataSizeOf(roundtrip_value_metadata), + ArrowMetadataSizeOf(reinterpret_cast(value_metadata->data))); + EXPECT_EQ(memcmp(roundtrip_value_metadata, value_metadata->data, + ArrowMetadataSizeOf(roundtrip_value_metadata)), + 0); + EXPECT_EQ(ArrowMetadataHasKey(roundtrip_value_metadata, ArrowCharView("field_key")), 0); + + // The dictionary-encoded values decode as before + nanoarrow::UniqueArray roundtrip_array; + ASSERT_EQ(ArrowArrayStreamGetNext(reader.get(), roundtrip_array.get(), &error), + NANOARROW_OK) + << error.message; + ASSERT_EQ(roundtrip_array->length, 3); + ASSERT_NE(roundtrip_array->children[0]->dictionary, nullptr); + EXPECT_EQ(roundtrip_array->children[0]->dictionary->length, 2); +} + // A struct array with a single int32 column of repeating values (i.e., compressible) static constexpr int64_t kCompressibleBatchLength = 1024; diff --git a/src/nanoarrow/nanoarrow_testing.hpp b/src/nanoarrow/nanoarrow_testing.hpp index 945e19ea3..416d830d3 100644 --- a/src/nanoarrow/nanoarrow_testing.hpp +++ b/src/nanoarrow/nanoarrow_testing.hpp @@ -120,7 +120,8 @@ class NANOARROW_DLL TestingJSONWriter { ArrowErrorCode WriteFieldDictionary(std::ostream& out, int32_t dictionary_id, bool is_ordered, - const ArrowSchemaView* indices_field); + const ArrowSchemaView* indices_field, + const char* value_metadata); ArrowErrorCode WriteChildren(std::ostream& out, const ArrowSchema* field, const ArrowArrayView* value); @@ -278,6 +279,10 @@ class NANOARROW_DLL TestingJSONComparison { ArrowErrorCode CompareMetadata(const char* actual, const char* expected, ArrowError* error, const std::string& path = ""); + ArrowErrorCode CompareDictionaryMetadata(const ArrowSchema* actual, + const ArrowSchema* expected, ArrowError* error, + const std::string& path = ""); + ArrowErrorCode MetadataEqualKeyValue(const char* actual, const char* expected, bool* out, ArrowError* error); diff --git a/src/nanoarrow/testing/testing.cc b/src/nanoarrow/testing/testing.cc index df69cebe5..146f5a1e5 100644 --- a/src/nanoarrow/testing/testing.cc +++ b/src/nanoarrow/testing/testing.cc @@ -749,7 +749,8 @@ ArrowErrorCode TestingJSONWriter::WriteField(std::ostream& out, out << R"(, "dictionary": )"; view.type = view.storage_type; NANOARROW_RETURN_NOT_OK(WriteFieldDictionary( - out, dictionary_id, field->flags & ARROW_FLAG_DICTIONARY_ORDERED, &view)); + out, dictionary_id, field->flags & ARROW_FLAG_DICTIONARY_ORDERED, &view, + field->dictionary->metadata)); // Write dictionary children out << R"(, "children": )"; @@ -980,7 +981,7 @@ ArrowErrorCode TestingJSONWriter::WriteFieldChildren(std::ostream& out, } ArrowErrorCode TestingJSONWriter::WriteFieldDictionary( std::ostream& out, int32_t dictionary_id, bool is_ordered, - const ArrowSchemaView* indices_field) { + const ArrowSchemaView* indices_field, const char* value_metadata) { out << "{"; out << R"("id": )" << dictionary_id; @@ -994,6 +995,17 @@ ArrowErrorCode TestingJSONWriter::WriteFieldDictionary( out << R"(, "isOrdered": false)"; } + // The metadata of the dictionary value type (e.g., the extension type of the + // values), which is omitted if empty + if (ShouldWriteMetadata(value_metadata)) { + ArrowMetadataReader reader; + NANOARROW_RETURN_NOT_OK(ArrowMetadataReaderInit(&reader, value_metadata)); + if (reader.remaining_keys > 0) { + out << R"(, "metadata": )"; + NANOARROW_RETURN_NOT_OK(WriteMetadata(out, value_metadata)); + } + } + out << "}"; return NANOARROW_OK; } @@ -1635,10 +1647,15 @@ ArrowErrorCode SetField(ArrowSchema* schema, const json& value, NANOARROW_RETURN_NOT_OK( SetDictionary(schema, value["dictionary"], &dictionary_id, error)); - // Allocate a dictionary and put this value (minus dictionary, metadata, and name) + // Allocate a dictionary and put this value (minus dictionary, metadata, and name). + // The metadata of the dictionary value type (e.g., the extension type of the + // values) is the "metadata" of the "dictionary" object, if present. json value_copy = value; value_copy.erase("dictionary"); value_copy.erase("metadata"); + if (value["dictionary"].contains("metadata")) { + value_copy["metadata"] = value["dictionary"]["metadata"]; + } value_copy["name"] = nullptr; NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowSchemaAllocateDictionary(schema), error); NANOARROW_RETURN_NOT_OK( @@ -2778,6 +2795,51 @@ ArrowErrorCode TestingJSONComparison::CompareFieldBase(ArrowSchema* actual, NANOARROW_RETURN_NOT_OK(CompareMetadata(actual->metadata, expected->metadata, error, path + std::string(".metadata"))); + NANOARROW_RETURN_NOT_OK(CompareDictionaryMetadata(actual, expected, error, path)); + return NANOARROW_OK; +} + +namespace { + +// Empty dictionary value metadata is equivalent to no dictionary value metadata +// (e.g., it is omitted when written to IPC or JSON) +ArrowErrorCode MetadataOrNullIfEmpty(const char* metadata, const char** out) { + ArrowMetadataReader reader; + NANOARROW_RETURN_NOT_OK(ArrowMetadataReaderInit(&reader, metadata)); + *out = reader.remaining_keys > 0 ? metadata : nullptr; + return NANOARROW_OK; +} + +} // namespace + +ArrowErrorCode TestingJSONComparison::CompareDictionaryMetadata( + const ArrowSchema* actual, const ArrowSchema* expected, ArrowError* error, + const std::string& path) { + // Differences in the structure of the field were already recorded by the caller + if ((actual->dictionary == nullptr) != (expected->dictionary == nullptr) || + actual->n_children != expected->n_children) { + return NANOARROW_OK; + } + + for (int64_t i = 0; i < expected->n_children; i++) { + NANOARROW_RETURN_NOT_OK(CompareDictionaryMetadata( + actual->children[i], expected->children[i], error, + path + std::string(".children[") + std::to_string(i) + std::string("]"))); + } + + if (expected->dictionary != nullptr) { + const char* actual_metadata; + const char* expected_metadata; + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + MetadataOrNullIfEmpty(actual->dictionary->metadata, &actual_metadata), error); + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + MetadataOrNullIfEmpty(expected->dictionary->metadata, &expected_metadata), error); + NANOARROW_RETURN_NOT_OK(CompareMetadata(actual_metadata, expected_metadata, error, + path + std::string(".dictionary.metadata"))); + NANOARROW_RETURN_NOT_OK(CompareDictionaryMetadata( + actual->dictionary, expected->dictionary, error, path + ".dictionary")); + } + return NANOARROW_OK; } diff --git a/src/nanoarrow/testing/testing_test.cc b/src/nanoarrow/testing/testing_test.cc index 8c3cb4c36..7cdf34591 100644 --- a/src/nanoarrow/testing/testing_test.cc +++ b/src/nanoarrow/testing/testing_test.cc @@ -425,6 +425,72 @@ TEST(NanoarrowTestingTest, NanoarrowTestingTestFieldDict) { R"("isOrdered": false}, "children": []})"); } +TEST(NanoarrowTestingTest, NanoarrowTestingTestFieldDictValueMetadata) { + // The metadata of the dictionary value type is written in the "dictionary" object and + // the field metadata is written in the field + // (https://github.com/apache/arrow/issues/49704) + TestWriteJSON( + [](ArrowSchema* schema) { + nanoarrow::UniqueBuffer buffer; + NANOARROW_RETURN_NOT_OK(ArrowMetadataBuilderInit(buffer.get(), nullptr)); + NANOARROW_RETURN_NOT_OK(ArrowMetadataBuilderAppend( + buffer.get(), "ARROW:extension:name"_asv, "arrow.uuid"_asv)); + NANOARROW_RETURN_NOT_OK(ArrowMetadataBuilderAppend( + buffer.get(), "ARROW:extension:metadata"_asv, ""_asv)); + + NANOARROW_RETURN_NOT_OK(ArrowSchemaInitFromType(schema, NANOARROW_TYPE_INT8)); + NANOARROW_RETURN_NOT_OK(ArrowSchemaSetMetadata(schema, "\0\0\0\0")); + NANOARROW_RETURN_NOT_OK(ArrowSchemaAllocateDictionary(schema)); + ArrowSchemaInit(schema->dictionary); + NANOARROW_RETURN_NOT_OK(ArrowSchemaSetTypeFixedSize( + schema->dictionary, NANOARROW_TYPE_FIXED_SIZE_BINARY, 16)); + NANOARROW_RETURN_NOT_OK(ArrowSchemaSetMetadata( + schema->dictionary, reinterpret_cast(buffer->data))); + return NANOARROW_OK; + }, + /*append_expr*/ nullptr, &WriteFieldJSON, + R"({"name": null, "nullable": true, "type": {"name": "fixedsizebinary", "byteWidth": 16}, )" + R"("dictionary": {"id": 0, "indexType": {"name": "int", "bitWidth": 8, "isSigned": true}, )" + R"("isOrdered": false, "metadata": [{"key": "ARROW:extension:name", "value": "arrow.uuid"}, )" + R"({"key": "ARROW:extension:metadata", "value": ""}]}, "children": [], "metadata": []})"); + + // Empty value metadata is omitted + TestWriteJSON( + [](ArrowSchema* schema) { + NANOARROW_RETURN_NOT_OK(ArrowSchemaInitFromType(schema, NANOARROW_TYPE_INT16)); + NANOARROW_RETURN_NOT_OK(ArrowSchemaAllocateDictionary(schema)); + NANOARROW_RETURN_NOT_OK( + ArrowSchemaInitFromType(schema->dictionary, NANOARROW_TYPE_STRING)); + NANOARROW_RETURN_NOT_OK(ArrowSchemaSetMetadata(schema->dictionary, "\0\0\0\0")); + return NANOARROW_OK; + }, + /*append_expr*/ nullptr, &WriteFieldJSON, + R"({"name": null, "nullable": true, "type": {"name": "utf8"}, )" + R"("dictionary": {"id": 0, "indexType": {"name": "int", "bitWidth": 16, "isSigned": true}, )" + R"("isOrdered": false}, "children": []})"); + + // Value metadata is omitted when metadata is turned off + TestWriteJSON( + [](ArrowSchema* schema) { + nanoarrow::UniqueBuffer buffer; + NANOARROW_RETURN_NOT_OK(ArrowMetadataBuilderInit(buffer.get(), nullptr)); + NANOARROW_RETURN_NOT_OK( + ArrowMetadataBuilderAppend(buffer.get(), "k1"_asv, "v1"_asv)); + NANOARROW_RETURN_NOT_OK(ArrowSchemaInitFromType(schema, NANOARROW_TYPE_INT16)); + NANOARROW_RETURN_NOT_OK(ArrowSchemaAllocateDictionary(schema)); + NANOARROW_RETURN_NOT_OK( + ArrowSchemaInitFromType(schema->dictionary, NANOARROW_TYPE_STRING)); + NANOARROW_RETURN_NOT_OK(ArrowSchemaSetMetadata( + schema->dictionary, reinterpret_cast(buffer->data))); + return NANOARROW_OK; + }, + [](ArrowArray*) { return NANOARROW_OK; }, &WriteFieldJSON, + R"({"name": null, "nullable": true, "type": {"name": "utf8"}, )" + R"("dictionary": {"id": 0, "indexType": {"name": "int", "bitWidth": 16, "isSigned": true}, )" + R"("isOrdered": false}, "children": []})", + [](TestingJSONWriter& writer) { writer.set_include_metadata(false); }); +} + TEST(NanoarrowTestingTest, NanoarrowTestingTestFieldMetadata) { // Missing metadata TestWriteJSON( @@ -855,6 +921,68 @@ TEST(NanoarrowTestingTest, NanoarrowTestingTestReadFieldDictionary) { EXPECT_TRUE(schema->flags & ARROW_FLAG_DICTIONARY_ORDERED); } +TEST(NanoarrowTestingTest, NanoarrowTestingTestReadFieldDictionaryValueMetadata) { + nanoarrow::UniqueSchema schema; + TestingJSONReader reader; + ArrowError error; + error.message[0] = '\0'; + + // The "metadata" of the "dictionary" object is the metadata of the dictionary value + // type and the "metadata" of the field is the metadata of the field + ASSERT_EQ( + reader.ReadField( + R"({"name": "col1", "nullable": true, "type": {"name": "utf8"}, "children": [], )" + R"("dictionary": {"id": 0, "indexType": {"name": "int", "bitWidth": 32, "isSigned": true}, )" + R"("isOrdered": false, "metadata": [{"key": "ARROW:extension:name", "value": "arrow.opaque"}, )" + R"({"key": "ARROW:extension:metadata", "value": "{}"}]}, )" + R"("metadata": [{"key": "k1", "value": "v1"}]})", + schema.get(), &error), + NANOARROW_OK) + << error.message; + EXPECT_STREQ(schema->format, "i"); + ASSERT_NE(schema->dictionary, nullptr); + EXPECT_STREQ(schema->dictionary->format, "u"); + + ArrowStringView value; + ArrowMetadataReader metadata; + ASSERT_EQ(ArrowMetadataReaderInit(&metadata, schema->metadata), NANOARROW_OK); + EXPECT_EQ(metadata.remaining_keys, 1); + ASSERT_EQ(ArrowMetadataGetValue(schema->metadata, "k1"_asv, &value), NANOARROW_OK); + EXPECT_EQ(value, "v1"_asv); + + ASSERT_EQ(ArrowMetadataReaderInit(&metadata, schema->dictionary->metadata), + NANOARROW_OK); + EXPECT_EQ(metadata.remaining_keys, 2); + ASSERT_EQ(ArrowMetadataGetValue(schema->dictionary->metadata, + "ARROW:extension:name"_asv, &value), + NANOARROW_OK); + EXPECT_EQ(value, "arrow.opaque"_asv); + + // Without "metadata" in the "dictionary" object, the value type has no metadata + schema.reset(); + ASSERT_EQ( + reader.ReadField( + R"({"name": "col1", "nullable": true, "type": {"name": "utf8"}, "children": [], )" + R"("dictionary": {"id": 0, "indexType": {"name": "int", "bitWidth": 32, "isSigned": true}, )" + R"("isOrdered": false}, "metadata": [{"key": "k1", "value": "v1"}]})", + schema.get(), &error), + NANOARROW_OK) + << error.message; + EXPECT_NE(schema->metadata, nullptr); + EXPECT_EQ(schema->dictionary->metadata, nullptr); + + // Invalid value metadata + schema.reset(); + EXPECT_EQ( + reader.ReadField( + R"({"name": "col1", "nullable": true, "type": {"name": "utf8"}, "children": [], )" + R"("dictionary": {"id": 0, "indexType": {"name": "int", "bitWidth": 32, "isSigned": true}, )" + R"("isOrdered": false, "metadata": {}}})", + schema.get(), &error), + EINVAL); + EXPECT_STREQ(error.message, "Field or Schema metadata must be null or array"); +} + TEST(NanoarrowTestingTest, NanoarrowTestingTestRoundtripDataFile) { nanoarrow::UniqueArrayStream stream; ArrowError error; @@ -1344,6 +1472,21 @@ TEST(NanoarrowTestingTest, NanoarrowTestingTestFieldDictionaryRoundtrip) { R"({"name": null, "nullable": true, "type": {"name": "utf8"}, )" R"("dictionary": {"id": 0, "indexType": {"name": "int", "bitWidth": 16, "isSigned": true}, )" R"("isOrdered": true}, "children": []})"); + + // With value metadata and field metadata + TestFieldRoundtrip( + R"({"name": "col", "nullable": true, "type": {"name": "utf8"}, )" + R"("dictionary": {"id": 0, "indexType": {"name": "int", "bitWidth": 16, "isSigned": true}, )" + R"("isOrdered": false, "metadata": [{"key": "ARROW:extension:name", "value": "arrow.opaque"}, )" + R"({"key": "ARROW:extension:metadata", "value": "{}"}]}, "children": [], )" + R"("metadata": [{"key": "k1", "value": "v1"}]})"); + + // Nested + TestFieldRoundtrip( + R"({"name": "col", "nullable": true, "type": {"name": "list"}, "children": [)" + R"({"name": "item", "nullable": true, "type": {"name": "utf8"}, )" + R"("dictionary": {"id": 0, "indexType": {"name": "int", "bitWidth": 16, "isSigned": true}, )" + R"("isOrdered": false, "metadata": [{"key": "k2", "value": "v2"}]}, "children": []}]})"); } void AssertSchemasCompareEqual( @@ -1655,6 +1798,116 @@ TEST(NanoarrowTestingTest, NanoarrowTestingTestMetadataComparison) { ASSERT_EQ(comparison.CompareSchema(actual.get(), expected.get()), ENOTSUP); } +TEST(NanoarrowTestingTest, NanoarrowTestingTestDictionaryValueMetadataComparison) { + nanoarrow::UniqueSchema actual; + nanoarrow::UniqueSchema expected; + nanoarrow::UniqueBuffer buf; + + // A dictionary-encoded field and a list of a dictionary-encoded field + ArrowSchemaInit(actual.get()); + ASSERT_EQ(ArrowSchemaSetTypeStruct(actual.get(), 2), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(actual->children[0], NANOARROW_TYPE_INT32), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaAllocateDictionary(actual->children[0]), NANOARROW_OK); + ASSERT_EQ( + ArrowSchemaInitFromType(actual->children[0]->dictionary, NANOARROW_TYPE_STRING), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(actual->children[1], NANOARROW_TYPE_LIST), NANOARROW_OK); + ArrowSchema* item = actual->children[1]->children[0]; + ASSERT_EQ(ArrowSchemaSetType(item, NANOARROW_TYPE_INT32), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaAllocateDictionary(item), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaInitFromType(item->dictionary, NANOARROW_TYPE_STRING), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaDeepCopy(actual.get(), expected.get()), NANOARROW_OK); + AssertSchemasCompareEqual(actual.get(), expected.get()); + + // Empty value metadata is equivalent to no value metadata + ASSERT_EQ(ArrowSchemaSetMetadata(actual->children[0]->dictionary, "\0\0\0\0"), + NANOARROW_OK); + AssertSchemasCompareEqual(actual.get(), expected.get()); + + // With different value metadata + ASSERT_EQ(ArrowMetadataBuilderInit(buf.get(), nullptr), NANOARROW_OK); + ASSERT_EQ(ArrowMetadataBuilderAppend(buf.get(), "key1"_asv, "value1"_asv), + NANOARROW_OK); + ASSERT_EQ(ArrowMetadataBuilderAppend(buf.get(), "key2"_asv, "value2"_asv), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetMetadata(actual->children[0]->dictionary, + reinterpret_cast(buf->data)), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetMetadata(item->dictionary, reinterpret_cast(buf->data)), + NANOARROW_OK); + AssertSchemasCompareUnequal(actual.get(), expected.get(), /*num_differences*/ 2, + /*differences*/ + "Path: .children[0].dictionary.metadata" + R"( +- [{"key": "key1", "value": "value1"}, {"key": "key2", "value": "value2"}] ++ null + +)" + "Path: .children[1].children[0].dictionary.metadata" + R"( +- [{"key": "key1", "value": "value1"}, {"key": "key2", "value": "value2"}] ++ null + +)"); + + // With value metadata that differs only in order + buf.reset(); + ASSERT_EQ(ArrowMetadataBuilderInit(buf.get(), nullptr), NANOARROW_OK); + ASSERT_EQ(ArrowMetadataBuilderAppend(buf.get(), "key2"_asv, "value2"_asv), + NANOARROW_OK); + ASSERT_EQ(ArrowMetadataBuilderAppend(buf.get(), "key1"_asv, "value1"_asv), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetMetadata(expected->children[0]->dictionary, + reinterpret_cast(buf->data)), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetMetadata(expected->children[1]->children[0]->dictionary, + reinterpret_cast(buf->data)), + NANOARROW_OK); + AssertSchemasCompareUnequal(actual.get(), expected.get(), /*num_differences*/ 2, + /*differences*/ + "Path: .children[0].dictionary.metadata" + R"( +- [{"key": "key1", "value": "value1"}, {"key": "key2", "value": "value2"}] ++ [{"key": "key2", "value": "value2"}, {"key": "key1", "value": "value1"}] + +)" + "Path: .children[1].children[0].dictionary.metadata" + R"( +- [{"key": "key1", "value": "value1"}, {"key": "key2", "value": "value2"}] ++ [{"key": "key2", "value": "value2"}, {"key": "key1", "value": "value1"}] + +)"); + AssertSchemasCompareEqual(actual.get(), expected.get(), + [](TestingJSONComparison& comparison) { + comparison.set_compare_metadata_order(false); + }); + + // Value metadata is not the field metadata + ASSERT_EQ(ArrowSchemaSetMetadata(expected->children[0]->dictionary, nullptr), + NANOARROW_OK); + ASSERT_EQ( + ArrowSchemaSetMetadata(expected->children[0], reinterpret_cast(buf->data)), + NANOARROW_OK); + AssertSchemasCompareUnequal(actual.get(), expected.get(), /*num_differences*/ 2, + /*differences*/ + "Path: .children[0].metadata" + R"( +- null ++ [{"key": "key2", "value": "value2"}, {"key": "key1", "value": "value1"}] + +)" + "Path: .children[0].dictionary.metadata" + R"( +- [{"key": "key1", "value": "value1"}, {"key": "key2", "value": "value2"}] ++ null + +)", + [](TestingJSONComparison& comparison) { + comparison.set_compare_metadata_order(false); + }); +} + TEST(NanoarrowTestingTest, NanoarrowTestingTestArrayComparison) { nanoarrow::UniqueSchema schema; nanoarrow::UniqueArray actual;