[C++][Python] Support NaN/Infinity in JSON parsing and accept DictionaryEncoding without indexType in IPC reader

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -2852,6 +2852,11 @@ function(build_simdjson)
   # which is required for Emscripten where Arrow threading is disabled.
   set(SIMDJSON_ENABLE_THREADS ${ARROW_ENABLE_THREADING})
 
+  # Enable parsing of NaN and Infinity literals in JSON (permitted by
+  # Arrow's JSON reader even though not in strict JSON). Requires
+  # simdjson >= 5.0.
+  set(SIMDJSON_ENABLE_NAN_INF ON)
+
   # simdjson enables precompiled headers unconditionally.
   # Recompiling simdjson.cpp against it produces differing artifacts
   # Disable precompiled headers to avoid reproducible build failures.

cpp/src/arrow/ipc/metadata_internal.cc

@@ -898,8 +898,13 @@ Status FieldFromFlatbuffer(const flatbuf::Field* field, FieldPosition field_pos,
     // dictionary_memo
     std::shared_ptr<DataType> index_type;
     auto int_data = encoding->indexType();
-    CHECK_FLATBUFFERS_NOT_NULL(int_data, "DictionaryEncoding.indexType");
-    RETURN_NOT_OK(IntFromFlatbuffer(int_data, &index_type));
+    if (int_data == nullptr) {
+      // Per the Arrow format spec, if indexType is null, the index type is
+      // implicitly signed int32.
+      index_type = int32();
+    } else {
+      RETURN_NOT_OK(IntFromFlatbuffer(int_data, &index_type));
+    }
     dict_value_type = type;
     ARROW_ASSIGN_OR_RAISE(type,
                           DictionaryType::Make(index_type, type, encoding->isOrdered()));

cpp/src/arrow/ipc/read_write_test.cc

@@ -348,6 +348,56 @@ TEST_F(TestSchemaMetadata, NestedDictionaryFields) {
   }
 }
 
+// ARROW-XXXXX: DictionaryEncoding.indexType may be null per the Arrow format
+// spec, in which case indices must be signed int32. JavaScript arrow already
+// relies on this when writing dictionary-encoded fields.
+TEST_F(TestSchemaMetadata, DictionaryEncodingWithoutIndexType) {
+  // Manually construct a flatbuffer Schema containing a DictionaryEncoding
+  // whose indexType field is absent (null offset), then verify that the C++
+  // reader defaults the index type to signed int32.
+
+  flatbuffers::FlatBufferBuilder fbb;
+
+  // value type: utf8
+  auto utf8_type_offset = flatbuf::CreateUtf8(fbb);
+
+  // dictionary encoding with no indexType (offset 0)
+  auto dict_encoding_offset =
+      flatbuf::CreateDictionaryEncoding(fbb, /*id=*/42,
+                                        /*indexType=*/0,
+                                        /*isOrdered=*/false);
+
+  // field: nullable utf8 with dictionary encoding (no indexType)
+  auto field_name = fbb.CreateString("dict_field");
+  auto field_offset = flatbuf::CreateField(fbb, field_name, /*nullable=*/true,
+                                           flatbuf::Type_Utf8,
+                                           utf8_type_offset.Union(),
+                                           dict_encoding_offset);
+
+  auto fields = fbb.CreateVector<flatbuffers::Offset<flatbuf::Field>>({field_offset});
+  auto schema_offset = flatbuf::CreateSchema(fbb, flatbuf::Endianness_Little, fields);
+  fbb.Finish(schema_offset);
+
+  // Extract the flatbuffer Schema root and go through metadata_internal's
+  // GetSchema, which is the function used by the IPC reader
+  const auto* fb_root_schema = flatbuf::GetSchema(fbb.GetBufferPointer());
+  DictionaryMemo memo;
+  std::shared_ptr<Schema> schema;
+  ASSERT_OK(internal::GetSchema(fb_root_schema, &memo, &schema));
+
+  ASSERT_EQ(schema->num_fields(), 1);
+  const auto& dict_field = schema->field(0);
+  const auto& dict_type = checked_cast<const DictionaryType&>(*dict_field->type());
+
+  // The format-spec default must be signed int32
+  EXPECT_TRUE(dict_type.index_type()->Equals(*int32()));
+  // The value type must be preserved
+  EXPECT_TRUE(dict_type.value_type()->Equals(*utf8()));
+  // Nullability and ordered flag should round-trip
+  EXPECT_TRUE(dict_field->nullable());
+  EXPECT_FALSE(dict_type.ordered());
+}
+
 TEST_F(TestSchemaMetadata, KeyValueMetadata) {
   auto field_metadata = key_value_metadata({{"key", "value"}});
   auto schema_metadata = key_value_metadata({{"foo", "bar"}, {"bizz", "buzz"}});

cpp/subprojects/simdjson.wrap

@@ -16,8 +16,8 @@
 # under the License.
 
 [wrap-file]
-source_url = https://github.com/simdjson/simdjson/archive/refs/tags/v4.6.4.tar.gz
-source_filename = simdjson-v4.6.4.tar.gz
-source_hash = b091107844fe928158c5c2265c20360fff312889ddf7ebc4528a0f0f8f2ff9cd
-directory = simdjson-v4.6.4
+source_url = https://github.com/simdjson/simdjson/archive/refs/tags/v5.0.5.tar.gz
+source_filename = simdjson-v5.0.5.tar.gz
+source_hash = 50bb17769e1054ff1bfcd5dbbfcf33544c77f6ebdb31a318436ea0e47f78f571
+directory = simdjson-v5.0.5
 method = cmake

cpp/thirdparty/versions.txt

@@ -102,8 +102,8 @@ ARROW_RAPIDJSON_BUILD_SHA256_CHECKSUM=b9290a9a6d444c8e049bd589ab804e0ccf2b05dc59
 # for CRAN builds. This version includes musl libc support (GH-48010).
 ARROW_RE2_BUILD_VERSION=2023-03-01
 ARROW_RE2_BUILD_SHA256_CHECKSUM=7a9a4824958586980926a300b4717202485c4b4115ac031822e29aa4ef207e48
-ARROW_SIMDJSON_BUILD_VERSION=v4.6.4
-ARROW_SIMDJSON_BUILD_SHA256_CHECKSUM=b091107844fe928158c5c2265c20360fff312889ddf7ebc4528a0f0f8f2ff9cd
+ARROW_SIMDJSON_BUILD_VERSION=v5.0.5
+ARROW_SIMDJSON_BUILD_SHA256_CHECKSUM=50bb17769e1054ff1bfcd5dbbfcf33544c77f6ebdb31a318436ea0e47f78f571
 ARROW_SNAPPY_BUILD_VERSION=1.2.2
 ARROW_SNAPPY_BUILD_SHA256_CHECKSUM=90f74bc1fbf78a6c56b3c4a082a05103b3a56bb17bca1a27e052ea11723292dc
 ARROW_SUBSTRAIT_BUILD_VERSION=v0.44.0