From 7354c73502b347c03fdf360142d05a9c3f0d262d Mon Sep 17 00:00:00 2001 From: Asthenia0412 Date: Thu, 15 Oct 2026 05:28:07 +0000 Subject: [PATCH] [C++][CMake][IPC][JSON] Support NaN/Infinity in JSON parsing and accept DictionaryEncoding without indexType in IPC reader Two related Arrow format compatibility fixes: 1. Bump embedded simdjson from v4.6.4 to v5.0.5 and enable SIMDJSON_ENABLE_NAN_INF so JSON parsing accepts NaN, Infinity, -Infinity, Inf and -Inf (case-insensitive). The Arrow JSON reader already had NonFiniteDoubleFromRawToken as a partial workaround, but simdjson v5.0 gives first-class, case-insensitive support. 2. In the IPC reader, treat DictionaryEncoding.indexType being null as signed int32 indices per the Arrow format spec. Previously metadata_internal.cc used CHECK_FLATBUFFERS_NOT_NULL and rejected well-formed IPC streams written by e.g. apache-arrow (JavaScript) when the JS writer omitted indexType. Tests: - New C++ unit test IPC DictionaryEncodingWithoutIndexType round-trips through metadata_internal::GetSchema with a hand-built flatbuffer. - All existing JSON from_string tests (including NaN/Inf coverage) continue to pass on simdjson v5.0.5. --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 5 +++ cpp/src/arrow/ipc/metadata_internal.cc | 9 +++- cpp/src/arrow/ipc/read_write_test.cc | 50 +++++++++++++++++++++ cpp/subprojects/simdjson.wrap | 8 ++-- cpp/thirdparty/versions.txt | 4 +- 5 files changed, 68 insertions(+), 8 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 01e1ab147af7..4b7927bc8d6d 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2852,6 +2852,11 @@ function(build_simdjson) # which is required for Emscripten where Arrow threading is disabled. set(SIMDJSON_ENABLE_THREADS ${ARROW_ENABLE_THREADING}) + # Enable parsing of NaN and Infinity literals in JSON (permitted by + # Arrow's JSON reader even though not in strict JSON). Requires + # simdjson >= 5.0. + set(SIMDJSON_ENABLE_NAN_INF ON) + # simdjson enables precompiled headers unconditionally. # Recompiling simdjson.cpp against it produces differing artifacts # Disable precompiled headers to avoid reproducible build failures. diff --git a/cpp/src/arrow/ipc/metadata_internal.cc b/cpp/src/arrow/ipc/metadata_internal.cc index 02134aa805ab..5dd7c18542d5 100644 --- a/cpp/src/arrow/ipc/metadata_internal.cc +++ b/cpp/src/arrow/ipc/metadata_internal.cc @@ -898,8 +898,13 @@ Status FieldFromFlatbuffer(const flatbuf::Field* field, FieldPosition field_pos, // dictionary_memo std::shared_ptr index_type; auto int_data = encoding->indexType(); - CHECK_FLATBUFFERS_NOT_NULL(int_data, "DictionaryEncoding.indexType"); - RETURN_NOT_OK(IntFromFlatbuffer(int_data, &index_type)); + if (int_data == nullptr) { + // Per the Arrow format spec, if indexType is null, the index type is + // implicitly signed int32. + index_type = int32(); + } else { + RETURN_NOT_OK(IntFromFlatbuffer(int_data, &index_type)); + } dict_value_type = type; ARROW_ASSIGN_OR_RAISE(type, DictionaryType::Make(index_type, type, encoding->isOrdered())); diff --git a/cpp/src/arrow/ipc/read_write_test.cc b/cpp/src/arrow/ipc/read_write_test.cc index cfa6a541d5cc..58fb40e568c6 100644 --- a/cpp/src/arrow/ipc/read_write_test.cc +++ b/cpp/src/arrow/ipc/read_write_test.cc @@ -348,6 +348,56 @@ TEST_F(TestSchemaMetadata, NestedDictionaryFields) { } } +// ARROW-XXXXX: DictionaryEncoding.indexType may be null per the Arrow format +// spec, in which case indices must be signed int32. JavaScript arrow already +// relies on this when writing dictionary-encoded fields. +TEST_F(TestSchemaMetadata, DictionaryEncodingWithoutIndexType) { + // Manually construct a flatbuffer Schema containing a DictionaryEncoding + // whose indexType field is absent (null offset), then verify that the C++ + // reader defaults the index type to signed int32. + + flatbuffers::FlatBufferBuilder fbb; + + // value type: utf8 + auto utf8_type_offset = flatbuf::CreateUtf8(fbb); + + // dictionary encoding with no indexType (offset 0) + auto dict_encoding_offset = + flatbuf::CreateDictionaryEncoding(fbb, /*id=*/42, + /*indexType=*/0, + /*isOrdered=*/false); + + // field: nullable utf8 with dictionary encoding (no indexType) + auto field_name = fbb.CreateString("dict_field"); + auto field_offset = flatbuf::CreateField(fbb, field_name, /*nullable=*/true, + flatbuf::Type_Utf8, + utf8_type_offset.Union(), + dict_encoding_offset); + + auto fields = fbb.CreateVector>({field_offset}); + auto schema_offset = flatbuf::CreateSchema(fbb, flatbuf::Endianness_Little, fields); + fbb.Finish(schema_offset); + + // Extract the flatbuffer Schema root and go through metadata_internal's + // GetSchema, which is the function used by the IPC reader + const auto* fb_root_schema = flatbuf::GetSchema(fbb.GetBufferPointer()); + DictionaryMemo memo; + std::shared_ptr schema; + ASSERT_OK(internal::GetSchema(fb_root_schema, &memo, &schema)); + + ASSERT_EQ(schema->num_fields(), 1); + const auto& dict_field = schema->field(0); + const auto& dict_type = checked_cast(*dict_field->type()); + + // The format-spec default must be signed int32 + EXPECT_TRUE(dict_type.index_type()->Equals(*int32())); + // The value type must be preserved + EXPECT_TRUE(dict_type.value_type()->Equals(*utf8())); + // Nullability and ordered flag should round-trip + EXPECT_TRUE(dict_field->nullable()); + EXPECT_FALSE(dict_type.ordered()); +} + TEST_F(TestSchemaMetadata, KeyValueMetadata) { auto field_metadata = key_value_metadata({{"key", "value"}}); auto schema_metadata = key_value_metadata({{"foo", "bar"}, {"bizz", "buzz"}}); diff --git a/cpp/subprojects/simdjson.wrap b/cpp/subprojects/simdjson.wrap index 31d9de3c2750..7ee75371b233 100644 --- a/cpp/subprojects/simdjson.wrap +++ b/cpp/subprojects/simdjson.wrap @@ -16,8 +16,8 @@ # under the License. [wrap-file] -source_url = https://github.com/simdjson/simdjson/archive/refs/tags/v4.6.4.tar.gz -source_filename = simdjson-v4.6.4.tar.gz -source_hash = b091107844fe928158c5c2265c20360fff312889ddf7ebc4528a0f0f8f2ff9cd -directory = simdjson-v4.6.4 +source_url = https://github.com/simdjson/simdjson/archive/refs/tags/v5.0.5.tar.gz +source_filename = simdjson-v5.0.5.tar.gz +source_hash = 50bb17769e1054ff1bfcd5dbbfcf33544c77f6ebdb31a318436ea0e47f78f571 +directory = simdjson-v5.0.5 method = cmake diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 050275c43708..a50abcc7c908 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -102,8 +102,8 @@ ARROW_RAPIDJSON_BUILD_SHA256_CHECKSUM=b9290a9a6d444c8e049bd589ab804e0ccf2b05dc59 # for CRAN builds. This version includes musl libc support (GH-48010). ARROW_RE2_BUILD_VERSION=2023-03-01 ARROW_RE2_BUILD_SHA256_CHECKSUM=7a9a4824958586980926a300b4717202485c4b4115ac031822e29aa4ef207e48 -ARROW_SIMDJSON_BUILD_VERSION=v4.6.4 -ARROW_SIMDJSON_BUILD_SHA256_CHECKSUM=b091107844fe928158c5c2265c20360fff312889ddf7ebc4528a0f0f8f2ff9cd +ARROW_SIMDJSON_BUILD_VERSION=v5.0.5 +ARROW_SIMDJSON_BUILD_SHA256_CHECKSUM=50bb17769e1054ff1bfcd5dbbfcf33544c77f6ebdb31a318436ea0e47f78f571 ARROW_SNAPPY_BUILD_VERSION=1.2.2 ARROW_SNAPPY_BUILD_SHA256_CHECKSUM=90f74bc1fbf78a6c56b3c4a082a05103b3a56bb17bca1a27e052ea11723292dc ARROW_SUBSTRAIT_BUILD_VERSION=v0.44.0