From 5a1f424fd724e277255168e76fcc2d0622c4055a Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Thu, 27 Aug 2026 16:47:26 +0200 Subject: [PATCH 1/4] GH-50859: [C++][Parquet] Move JsonWriter to simdjson utilities Parquet uses JsonWriter independently of the Arrow JSON module. Move the writer into the simdjson utilities so it is available whenever simdjson is enabled, including ARROW_JSON=OFF builds. Preserve the writer files as renames and update CMake, Meson, callers, and tests. --- cpp/src/arrow/CMakeLists.txt | 10 +- cpp/src/arrow/dataset/file_json_test.cc | 2 +- cpp/src/arrow/extension/fixed_shape_tensor.cc | 2 +- cpp/src/arrow/extension/opaque.cc | 2 +- .../arrow/extension/variable_shape_tensor.cc | 2 +- .../sql/odbc/odbc_impl/json_converter.cc | 2 +- cpp/src/arrow/integration/json_integration.cc | 2 +- .../integration/json_integration_test.cc | 2 +- cpp/src/arrow/integration/json_internal.cc | 2 +- cpp/src/arrow/json/CMakeLists.txt | 1 - cpp/src/arrow/json/meson.build | 1 - cpp/src/arrow/meson.build | 41 ++-- cpp/src/arrow/util/CMakeLists.txt | 8 + .../{json => util}/json_writer_internal.cc | 2 +- .../{json => util}/json_writer_internal.h | 0 .../json_writer_internal_test.cc | 2 +- cpp/src/arrow/util/meson.build | 8 +- cpp/src/arrow/util/simdjson_internal.cc | 222 ++++++++++++++++++ cpp/src/arrow/util/simdjson_internal.h | 219 ++--------------- .../file_system_key_material_store.cc | 2 +- cpp/src/parquet/encryption/key_material.cc | 2 +- cpp/src/parquet/encryption/key_metadata.cc | 2 +- .../encryption/local_wrap_kms_client.cc | 2 +- .../parquet/geospatial/util_json_internal.cc | 2 +- cpp/src/parquet/printer.cc | 2 +- cpp/src/parquet/types.cc | 2 +- 26 files changed, 313 insertions(+), 231 deletions(-) rename cpp/src/arrow/{json => util}/json_writer_internal.cc (99%) rename cpp/src/arrow/{json => util}/json_writer_internal.h (100%) rename cpp/src/arrow/{json => util}/json_writer_internal_test.cc (99%) create mode 100644 cpp/src/arrow/util/simdjson_internal.cc diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index b9ff2ffa2c2d..ed8174f7176a 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -644,6 +644,9 @@ endif() if(ARROW_WITH_OPENTELEMETRY) list(APPEND ARROW_UTIL_SRCS util/tracing_internal.cc) endif() +if(ARROW_WITH_SIMDJSON) + list(APPEND ARROW_UTIL_SRCS util/json_writer_internal.cc util/simdjson_internal.cc) +endif() if(ARROW_WITH_SNAPPY) list(APPEND ARROW_UTIL_SRCS util/compression_snappy.cc) endif() @@ -664,6 +667,12 @@ foreach(ARROW_UTIL_TARGET ${ARROW_UTIL_TARGETS}) target_link_libraries(${ARROW_UTIL_TARGET} PRIVATE ${ARROW_XSIMD}) endforeach() +if(ARROW_WITH_SIMDJSON) + foreach(ARROW_UTIL_TARGET ${ARROW_UTIL_TARGETS}) + target_link_libraries(${ARROW_UTIL_TARGET} PRIVATE arrow::simdjson) + endforeach() +endif() + if(ARROW_USE_BOOST) foreach(ARROW_UTIL_TARGET ${ARROW_UTIL_TARGETS}) target_link_libraries(${ARROW_UTIL_TARGET} PRIVATE Boost::headers) @@ -1088,7 +1097,6 @@ if(ARROW_JSON) json/chunker.cc json/converter.cc json/from_string.cc - json/json_writer_internal.cc json/object_parser.cc json/parser.cc json/reader.cc) diff --git a/cpp/src/arrow/dataset/file_json_test.cc b/cpp/src/arrow/dataset/file_json_test.cc index b96d4ffed99c..e3d0c15776ca 100644 --- a/cpp/src/arrow/dataset/file_json_test.cc +++ b/cpp/src/arrow/dataset/file_json_test.cc @@ -20,10 +20,10 @@ #include "arrow/dataset/plan.h" #include "arrow/dataset/test_util_internal.h" #include "arrow/filesystem/mockfs.h" -#include "arrow/json/json_writer_internal.h" #include "arrow/json/parser.h" #include "arrow/testing/gtest_util.h" #include "arrow/testing/util.h" +#include "arrow/util/json_writer_internal.h" #include "arrow/util/logging_internal.h" namespace arrow { diff --git a/cpp/src/arrow/extension/fixed_shape_tensor.cc b/cpp/src/arrow/extension/fixed_shape_tensor.cc index 6a86d6a7a66f..48799899c7e2 100644 --- a/cpp/src/arrow/extension/fixed_shape_tensor.cc +++ b/cpp/src/arrow/extension/fixed_shape_tensor.cc @@ -27,8 +27,8 @@ #include "arrow/array/array_nested.h" #include "arrow/array/array_primitive.h" -#include "arrow/json/json_writer_internal.h" #include "arrow/tensor.h" +#include "arrow/util/json_writer_internal.h" #include "arrow/util/logging_internal.h" #include "arrow/util/print_internal.h" #include "arrow/util/simdjson_internal.h" diff --git a/cpp/src/arrow/extension/opaque.cc b/cpp/src/arrow/extension/opaque.cc index c6068babdb00..6e5a07405f5a 100644 --- a/cpp/src/arrow/extension/opaque.cc +++ b/cpp/src/arrow/extension/opaque.cc @@ -19,7 +19,7 @@ #include -#include "arrow/json/json_writer_internal.h" +#include "arrow/util/json_writer_internal.h" #include "arrow/util/logging_internal.h" #include "arrow/util/simdjson_internal.h" diff --git a/cpp/src/arrow/extension/variable_shape_tensor.cc b/cpp/src/arrow/extension/variable_shape_tensor.cc index 784cd334b1d6..0ecd9f18c80b 100644 --- a/cpp/src/arrow/extension/variable_shape_tensor.cc +++ b/cpp/src/arrow/extension/variable_shape_tensor.cc @@ -23,9 +23,9 @@ #include "arrow/extension/variable_shape_tensor.h" #include "arrow/array/array_primitive.h" -#include "arrow/json/json_writer_internal.h" #include "arrow/scalar.h" #include "arrow/tensor.h" +#include "arrow/util/json_writer_internal.h" #include "arrow/util/logging_internal.h" #include "arrow/util/print_internal.h" #include "arrow/util/simdjson_internal.h" diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/json_converter.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/json_converter.cc index 64e91ecd9bff..5ced882545db 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/json_converter.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/json_converter.cc @@ -20,8 +20,8 @@ #include #include "arrow/builder.h" #include "arrow/flight/sql/odbc/odbc_impl/util.h" -#include "arrow/json/json_writer_internal.h" #include "arrow/scalar.h" +#include "arrow/util/json_writer_internal.h" #include "arrow/visitor.h" using boost::beast::detail::base64::encode; diff --git a/cpp/src/arrow/integration/json_integration.cc b/cpp/src/arrow/integration/json_integration.cc index f978e1da5418..5a1def6d0b40 100644 --- a/cpp/src/arrow/integration/json_integration.cc +++ b/cpp/src/arrow/integration/json_integration.cc @@ -29,11 +29,11 @@ #include "arrow/integration/json_internal.h" #include "arrow/io/file.h" #include "arrow/ipc/dictionary.h" -#include "arrow/json/json_writer_internal.h" #include "arrow/record_batch.h" #include "arrow/result.h" #include "arrow/status.h" #include "arrow/type.h" +#include "arrow/util/json_writer_internal.h" #include "arrow/util/logging_internal.h" #include "arrow/util/simdjson_internal.h" diff --git a/cpp/src/arrow/integration/json_integration_test.cc b/cpp/src/arrow/integration/json_integration_test.cc index 700551c23b79..6687f26224c6 100644 --- a/cpp/src/arrow/integration/json_integration_test.cc +++ b/cpp/src/arrow/integration/json_integration_test.cc @@ -38,7 +38,6 @@ #include "arrow/ipc/reader.h" #include "arrow/ipc/test_common.h" #include "arrow/ipc/writer.h" -#include "arrow/json/json_writer_internal.h" #include "arrow/pretty_print.h" #include "arrow/status.h" #include "arrow/testing/builder.h" @@ -49,6 +48,7 @@ #include "arrow/type.h" #include "arrow/type_fwd.h" #include "arrow/util/io_util.h" +#include "arrow/util/json_writer_internal.h" #include "arrow/util/simdjson_internal.h" DEFINE_string(arrow, "", "Arrow file name"); diff --git a/cpp/src/arrow/integration/json_internal.cc b/cpp/src/arrow/integration/json_internal.cc index abf48b9df767..4ed852b121bf 100644 --- a/cpp/src/arrow/integration/json_internal.cc +++ b/cpp/src/arrow/integration/json_internal.cc @@ -36,7 +36,6 @@ #include "arrow/array/builder_time.h" #include "arrow/extension_type.h" #include "arrow/ipc/dictionary.h" -#include "arrow/json/json_writer_internal.h" #include "arrow/record_batch.h" #include "arrow/result.h" #include "arrow/scalar.h" @@ -48,6 +47,7 @@ #include "arrow/util/checked_cast.h" #include "arrow/util/decimal.h" #include "arrow/util/formatting.h" +#include "arrow/util/json_writer_internal.h" #include "arrow/util/key_value_metadata.h" #include "arrow/util/logging_internal.h" #include "arrow/util/range.h" diff --git a/cpp/src/arrow/json/CMakeLists.txt b/cpp/src/arrow/json/CMakeLists.txt index e7719c771c3d..cad6f0fbdf3f 100644 --- a/cpp/src/arrow/json/CMakeLists.txt +++ b/cpp/src/arrow/json/CMakeLists.txt @@ -21,7 +21,6 @@ add_arrow_test(test chunker_test.cc converter_test.cc from_string_test.cc - json_writer_internal_test.cc object_parser_test.cc parser_test.cc reader_test.cc diff --git a/cpp/src/arrow/json/meson.build b/cpp/src/arrow/json/meson.build index ee2a26b2ccde..8c5813feec41 100644 --- a/cpp/src/arrow/json/meson.build +++ b/cpp/src/arrow/json/meson.build @@ -22,7 +22,6 @@ exc = executable( 'chunker_test.cc', 'converter_test.cc', 'from_string_test.cc', - 'json_writer_internal_test.cc', 'parser_test.cc', 'reader_test.cc', ], diff --git a/cpp/src/arrow/meson.build b/cpp/src/arrow/meson.build index b5443c28cf84..298c7a5e02c1 100644 --- a/cpp/src/arrow/meson.build +++ b/cpp/src/arrow/meson.build @@ -18,6 +18,22 @@ dl_dep = dependency('dl') threads_dep = dependency('threads') +if needs_json or needs_integration or needs_parquet + simdjson_dep = dependency('simdjson', allow_fallback: false, required: false) + + if not simdjson_dep.found() + cmake = import('cmake') + simdjson_opts = cmake.subproject_options() + + simdjson_opts.add_cmake_defines({'SIMDJSON_EXCEPTIONS': 'OFF'}) + + simdjson_proj = cmake.subproject('simdjson', options: simdjson_opts) + simdjson_dep = simdjson_proj.dependency('simdjson') + endif +else + simdjson_dep = disabler() +endif + arrow_components = { 'arrow_array': { 'sources': [ @@ -224,6 +240,14 @@ arrow_util_srcs = [ arrow_util_deps = [threads_dep] +if needs_json or needs_integration or needs_parquet + arrow_util_srcs += [ + 'util/json_writer_internal.cc', + 'util/simdjson_internal.cc', + ] + arrow_util_deps += [simdjson_dep] +endif + if needs_brotli arrow_util_srcs += ['util/compression_brotli.cc'] arrow_util_deps += [dependency('libbrotlidec'), dependency('libbrotlienc')] @@ -318,22 +342,6 @@ else rapidjson_dep = disabler() endif -if needs_json or needs_integration - simdjson_dep = dependency('simdjson', allow_fallback: false, required: false) - - if not simdjson_dep.found() - cmake = import('cmake') - simdjson_opts = cmake.subproject_options() - - simdjson_opts.add_cmake_defines({'SIMDJSON_EXCEPTIONS': 'OFF'}) - - simdjson_proj = cmake.subproject('simdjson', options: simdjson_opts) - simdjson_dep = simdjson_proj.dependency('simdjson') - endif -else - simdjson_dep = disabler() -endif - azure_dep = disabler() gcs_dep = disabler() s3_dep = disabler() @@ -530,7 +538,6 @@ if needs_json 'json/chunker.cc', 'json/converter.cc', 'json/from_string.cc', - 'json/json_writer_internal.cc', 'json/object_parser.cc', 'json/options.cc', 'json/parser.cc', diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt index 628e9a4d1c7e..f77c8faa0caf 100644 --- a/cpp/src/arrow/util/CMakeLists.txt +++ b/cpp/src/arrow/util/CMakeLists.txt @@ -87,6 +87,14 @@ add_arrow_test(utility-test EXTRA_LINK_LIBS ${ARROW_UTILITY_TEST_LINK_LIBS}) +if(ARROW_WITH_SIMDJSON) + add_arrow_test(json-writer-internal-test + SOURCES + json_writer_internal_test.cc + EXTRA_LINK_LIBS + simdjson::simdjson) +endif() + add_arrow_test(async-utility-test SOURCES async_generator_test.cc diff --git a/cpp/src/arrow/json/json_writer_internal.cc b/cpp/src/arrow/util/json_writer_internal.cc similarity index 99% rename from cpp/src/arrow/json/json_writer_internal.cc rename to cpp/src/arrow/util/json_writer_internal.cc index e49841fda658..15175f0d313f 100644 --- a/cpp/src/arrow/json/json_writer_internal.cc +++ b/cpp/src/arrow/util/json_writer_internal.cc @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/json/json_writer_internal.h" +#include "arrow/util/json_writer_internal.h" #include "arrow/util/simdjson_internal.h" namespace arrow::json { diff --git a/cpp/src/arrow/json/json_writer_internal.h b/cpp/src/arrow/util/json_writer_internal.h similarity index 100% rename from cpp/src/arrow/json/json_writer_internal.h rename to cpp/src/arrow/util/json_writer_internal.h diff --git a/cpp/src/arrow/json/json_writer_internal_test.cc b/cpp/src/arrow/util/json_writer_internal_test.cc similarity index 99% rename from cpp/src/arrow/json/json_writer_internal_test.cc rename to cpp/src/arrow/util/json_writer_internal_test.cc index a0123e808362..8d8e64528225 100644 --- a/cpp/src/arrow/json/json_writer_internal_test.cc +++ b/cpp/src/arrow/util/json_writer_internal_test.cc @@ -17,8 +17,8 @@ #include -#include "arrow/json/json_writer_internal.h" #include "arrow/testing/gtest_util.h" +#include "arrow/util/json_writer_internal.h" namespace sj = simdjson::ondemand; diff --git a/cpp/src/arrow/util/meson.build b/cpp/src/arrow/util/meson.build index c39e09c2d030..48a39072e3a6 100644 --- a/cpp/src/arrow/util/meson.build +++ b/cpp/src/arrow/util/meson.build @@ -222,6 +222,12 @@ utility_test_srcs = [ 'value_parsing_test.cc', ] +utility_test_deps = [arrow_test_dep_no_main] +if needs_json or needs_integration or needs_parquet + utility_test_srcs += ['json_writer_internal_test.cc'] + utility_test_deps += [simdjson_dep] +endif + if host_machine.system() == 'windows' # This manifest enables long file paths on Windows 10+ # See https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#enable-long-paths-in-windows-10-version-1607-and-later @@ -235,7 +241,7 @@ endif exc = executable( 'arrow-utility-test', sources: utility_test_srcs, - dependencies: arrow_test_dep_no_main, + dependencies: utility_test_deps, implicit_include_directories: false, ) test('arrow-utility-test', exc) diff --git a/cpp/src/arrow/util/simdjson_internal.cc b/cpp/src/arrow/util/simdjson_internal.cc new file mode 100644 index 000000000000..165b617768f3 --- /dev/null +++ b/cpp/src/arrow/util/simdjson_internal.cc @@ -0,0 +1,222 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/util/simdjson_internal.h" + +namespace arrow::internal { + +const char* JsonTypeName(simdjson::dom::element_type type) { + switch (type) { + case simdjson::dom::element_type::ARRAY: + return "array"; + case simdjson::dom::element_type::OBJECT: + return "object"; + case simdjson::dom::element_type::INT64: + case simdjson::dom::element_type::UINT64: + case simdjson::dom::element_type::DOUBLE: + return "number"; + case simdjson::dom::element_type::STRING: + return "string"; + case simdjson::dom::element_type::BOOL: + return "boolean"; + case simdjson::dom::element_type::NULL_VALUE: + return "null"; + default: + return "unknown"; + } +} + +Result GetJsonArray(simdjson::dom::element value, + std::string_view name) { + if (!value.is_array()) { + return Status::Invalid(name, " must be an array, got ", JsonTypeName(value.type())); + } + return ResolveSimdjsonResult(value.get_array(), "Failed to get JSON array"); +} + +Result GetJsonInt(simdjson::dom::element value, std::string_view name, + std::string_view expected) { + if (!value.is_int64()) { + return Status::Invalid(name, " must contain ", expected, ", got ", + JsonTypeName(value.type())); + } + return ResolveSimdjsonResult(value.get_int64(), "Failed to get JSON integer"); +} + +Result ParseJsonObject(simdjson::dom::parser& parser, + const std::string& json) { + return ResolveSimdjsonResult(parser.parse(json).get_object(), + "Invalid serialized JSON data"); +} + +Result> GetOptionalJsonField( + const simdjson::dom::object& object, std::string_view key) { + auto field = object.at_key(key); + if (field.error() == simdjson::NO_SUCH_FIELD) { + return std::nullopt; + } + + ARROW_ASSIGN_OR_RAISE( + auto value, + ResolveSimdjsonResult(std::move(field), "Failed to get JSON object field")); + + return std::optional(std::move(value)); +} + +Result> GetJsonIntArray(simdjson::dom::element value, + std::string_view name) { + ARROW_ASSIGN_OR_RAISE(auto array, GetJsonArray(value, name)); + + std::vector result; + result.reserve(array.size()); + + for (auto element : array) { + ARROW_ASSIGN_OR_RAISE(auto number, GetJsonInt(element, name, "integers")); + result.push_back(number); + } + + return result; +} + +Result>> GetJsonNullableIntArray( + simdjson::dom::element value, std::string_view name) { + ARROW_ASSIGN_OR_RAISE(auto array, GetJsonArray(value, name)); + + std::vector> result; + result.reserve(array.size()); + + for (auto element : array) { + if (element.is_null()) { + result.emplace_back(std::nullopt); + } else { + ARROW_ASSIGN_OR_RAISE(auto number, GetJsonInt(element, name, "integers or nulls")); + result.emplace_back(number); + } + } + + return result; +} + +Result> GetJsonStringArray(simdjson::dom::element value, + std::string_view name) { + ARROW_ASSIGN_OR_RAISE(auto array, GetJsonArray(value, name)); + + std::vector result; + result.reserve(array.size()); + + for (auto element : array) { + if (!element.is_string()) { + return Status::Invalid(name, " must contain strings, got ", + JsonTypeName(element.type())); + } + + ARROW_ASSIGN_OR_RAISE( + auto string, + ResolveSimdjsonResult(element.get_string(), "Failed to get JSON string")); + result.emplace_back(string); + } + + return result; +} + +const char* JsonTypeName(simdjson::ondemand::json_type type) { + switch (type) { + case simdjson::ondemand::json_type::array: + return "array"; + case simdjson::ondemand::json_type::object: + return "object"; + case simdjson::ondemand::json_type::number: + return "number"; + case simdjson::ondemand::json_type::string: + return "string"; + case simdjson::ondemand::json_type::boolean: + return "boolean"; + case simdjson::ondemand::json_type::null: + return "null"; + default: + return "unknown"; + } +} + +Result IsJsonNull(simdjson::ondemand::value& value) { + bool is_null; + auto error_code = value.is_null().get(is_null); + if (error_code != simdjson::SUCCESS) { + return Status::Invalid("Error checking for JSON null: ", + simdjson::error_message(error_code)); + } + return is_null; +} + +Result MinifyJson(std::string_view json) { + std::string minified(json.size(), '\0'); + size_t minified_len = 0; + + if (auto error = + simdjson::minify(json.data(), json.size(), minified.data(), minified_len); + error != simdjson::SUCCESS) { + return Status::Invalid("Failed to minify JSON: ", simdjson::error_message(error)); + } + + minified.resize(minified_len); + return minified; +} + +Status ConsumeJsonValue(simdjson::ondemand::value value) { + return VisitJsonValue( + value, ValidateJsonObject, ValidateJsonArray, + [](std::string_view) { return Status::OK(); }, [](bool) { return Status::OK(); }, + []() { return Status::OK(); }, [](int64_t) { return Status::OK(); }, + [](uint64_t) { return Status::OK(); }, [](double) { return Status::OK(); }, + [](simdjson::ondemand::value) { return Status::OK(); }); +} + +Status ValidateJsonObject(simdjson::ondemand::object object) { + for (auto field_result : object) { + ARROW_ASSIGN_OR_RAISE( + auto field, ResolveSimdjsonResult(field_result, "Failed to iterate JSON object")); + + RETURN_NOT_OK(ConsumeJsonValue(field.value())); + } + + return Status::OK(); +} + +Status ValidateJsonArray(simdjson::ondemand::array array) { + for (auto element_result : array) { + ARROW_ASSIGN_OR_RAISE( + auto value, + ResolveSimdjsonResult(element_result, "Failed to iterate JSON array")); + + RETURN_NOT_OK(ConsumeJsonValue(value)); + } + + return Status::OK(); +} + +Status ValidateJsonDocument(simdjson::ondemand::parser& parser, + simdjson::padded_string& json) { + ARROW_ASSIGN_OR_RAISE( + auto document, ResolveSimdjsonResult(parser.iterate(json), "Failed to parse JSON")); + + ARROW_ASSIGN_OR_RAISE(auto value, ResolveSimdjsonResult(document.get_value(), + "Failed to get JSON value")); + + return ConsumeJsonValue(value); +} + +} // namespace arrow::internal diff --git a/cpp/src/arrow/util/simdjson_internal.h b/cpp/src/arrow/util/simdjson_internal.h index 799ebfc5ea62..c507cd4be789 100644 --- a/cpp/src/arrow/util/simdjson_internal.h +++ b/cpp/src/arrow/util/simdjson_internal.h @@ -29,9 +29,9 @@ #include "arrow/result.h" #include "arrow/status.h" +#include "arrow/util/visibility.h" -namespace arrow { -namespace internal { +namespace arrow::internal { // Empty struct to represent the type of a simdjson null value struct SimdjsonNull {}; @@ -94,121 +94,30 @@ Result ResolveSimdjsonResult(simdjson::simdjson_result result, return value; } -inline const char* JsonTypeName(simdjson::dom::element_type type) { - switch (type) { - case simdjson::dom::element_type::ARRAY: - return "array"; - case simdjson::dom::element_type::OBJECT: - return "object"; - case simdjson::dom::element_type::INT64: - case simdjson::dom::element_type::UINT64: - case simdjson::dom::element_type::DOUBLE: - return "number"; - case simdjson::dom::element_type::STRING: - return "string"; - case simdjson::dom::element_type::BOOL: - return "boolean"; - case simdjson::dom::element_type::NULL_VALUE: - return "null"; - default: - return "unknown"; - } -} +ARROW_EXPORT const char* JsonTypeName(simdjson::dom::element_type type); -inline Result GetJsonArray(simdjson::dom::element value, - std::string_view name) { - if (!value.is_array()) { - return Status::Invalid(name, " must be an array, got ", JsonTypeName(value.type())); - } - return ResolveSimdjsonResult(value.get_array(), "Failed to get JSON array"); -} +ARROW_EXPORT Result GetJsonArray(simdjson::dom::element value, + std::string_view name); -inline Result GetJsonInt(simdjson::dom::element value, std::string_view name, - std::string_view expected) { - if (!value.is_int64()) { - return Status::Invalid(name, " must contain ", expected, ", got ", - JsonTypeName(value.type())); - } - return ResolveSimdjsonResult(value.get_int64(), "Failed to get JSON integer"); -} +ARROW_EXPORT Result GetJsonInt(simdjson::dom::element value, + std::string_view name, std::string_view expected); -inline Result ParseJsonObject(simdjson::dom::parser& parser, - const std::string& json) { - return ResolveSimdjsonResult(parser.parse(json).get_object(), - "Invalid serialized JSON data"); -} +ARROW_EXPORT Result ParseJsonObject(simdjson::dom::parser& parser, + const std::string& json); // object.at_key() performs a linear search. This is acceptable here // since these objects are expected to contain only a small number of fields. -inline Result> GetOptionalJsonField( - const simdjson::dom::object& object, std::string_view key) { - auto field = object.at_key(key); - if (field.error() == simdjson::NO_SUCH_FIELD) { - return std::nullopt; - } - - ARROW_ASSIGN_OR_RAISE( - auto value, - ResolveSimdjsonResult(std::move(field), "Failed to get JSON object field")); - - return std::optional(std::move(value)); -} - -inline Result> GetJsonIntArray(simdjson::dom::element value, - std::string_view name) { - ARROW_ASSIGN_OR_RAISE(auto array, GetJsonArray(value, name)); - - std::vector result; - result.reserve(array.size()); - - for (auto element : array) { - ARROW_ASSIGN_OR_RAISE(auto number, GetJsonInt(element, name, "integers")); - result.push_back(number); - } - - return result; -} - -inline Result>> GetJsonNullableIntArray( - simdjson::dom::element value, std::string_view name) { - ARROW_ASSIGN_OR_RAISE(auto array, GetJsonArray(value, name)); - - std::vector> result; - result.reserve(array.size()); - - for (auto element : array) { - if (element.is_null()) { - result.emplace_back(std::nullopt); - } else { - ARROW_ASSIGN_OR_RAISE(auto number, GetJsonInt(element, name, "integers or nulls")); - result.emplace_back(number); - } - } - - return result; -} - -inline Result> GetJsonStringArray(simdjson::dom::element value, - std::string_view name) { - ARROW_ASSIGN_OR_RAISE(auto array, GetJsonArray(value, name)); +ARROW_EXPORT Result> GetOptionalJsonField( + const simdjson::dom::object& object, std::string_view key); - std::vector result; - result.reserve(array.size()); +ARROW_EXPORT Result> GetJsonIntArray(simdjson::dom::element value, + std::string_view name); - for (auto element : array) { - if (!element.is_string()) { - return Status::Invalid(name, " must contain strings, got ", - JsonTypeName(element.type())); - } +ARROW_EXPORT Result>> GetJsonNullableIntArray( + simdjson::dom::element value, std::string_view name); - ARROW_ASSIGN_OR_RAISE( - auto string, - ResolveSimdjsonResult(element.get_string(), "Failed to get JSON string")); - result.emplace_back(string); - } - - return result; -} +ARROW_EXPORT Result> GetJsonStringArray( + simdjson::dom::element value, std::string_view name); template because peeking the nonRootScalar can fail (parsed lazily) -inline Result IsJsonNull(simdjson::ondemand::value& value) { - bool is_null; - auto error_code = value.is_null().get(is_null); - if (error_code != simdjson::SUCCESS) { - return Status::Invalid("Error checking for JSON null: ", - simdjson::error_message(error_code)); - } - return is_null; -} +ARROW_EXPORT Result IsJsonNull(simdjson::ondemand::value& value); template Result GetJsonAs(simdjson::ondemand::value& value) { @@ -386,66 +270,15 @@ Result GetJsonField(simdjson::ondemand::object& object, std::string_view key) return Status::KeyError("Missing JSON field: ", key); } -inline Result MinifyJson(std::string_view json) { - std::string minified(json.size(), '\0'); - size_t minified_len = 0; +ARROW_EXPORT Result MinifyJson(std::string_view json); - if (auto error = - simdjson::minify(json.data(), json.size(), minified.data(), minified_len); - error != simdjson::SUCCESS) { - return Status::Invalid("Failed to minify JSON: ", simdjson::error_message(error)); - } - - minified.resize(minified_len); - return minified; -} - -inline Status ValidateJsonObject(simdjson::ondemand::object object); +ARROW_EXPORT Status ValidateJsonObject(simdjson::ondemand::object object); -inline Status ValidateJsonArray(simdjson::ondemand::array array); +ARROW_EXPORT Status ValidateJsonArray(simdjson::ondemand::array array); -inline Status ConsumeJsonValue(simdjson::ondemand::value value) { - return VisitJsonValue( - value, ValidateJsonObject, ValidateJsonArray, - [](std::string_view) { return Status::OK(); }, [](bool) { return Status::OK(); }, - []() { return Status::OK(); }, [](int64_t) { return Status::OK(); }, - [](uint64_t) { return Status::OK(); }, [](double) { return Status::OK(); }, - [](simdjson::ondemand::value) { return Status::OK(); }); -} +ARROW_EXPORT Status ConsumeJsonValue(simdjson::ondemand::value value); -inline Status ValidateJsonObject(simdjson::ondemand::object object) { - for (auto field_result : object) { - ARROW_ASSIGN_OR_RAISE( - auto field, ResolveSimdjsonResult(field_result, "Failed to iterate JSON object")); - - RETURN_NOT_OK(ConsumeJsonValue(field.value())); - } - - return Status::OK(); -} - -inline Status ValidateJsonArray(simdjson::ondemand::array array) { - for (auto element_result : array) { - ARROW_ASSIGN_OR_RAISE( - auto value, - ResolveSimdjsonResult(element_result, "Failed to iterate JSON array")); - - RETURN_NOT_OK(ConsumeJsonValue(value)); - } - - return Status::OK(); -} - -inline Status ValidateJsonDocument(simdjson::ondemand::parser& parser, - simdjson::padded_string& json) { - ARROW_ASSIGN_OR_RAISE( - auto document, ResolveSimdjsonResult(parser.iterate(json), "Failed to parse JSON")); - - ARROW_ASSIGN_OR_RAISE(auto value, ResolveSimdjsonResult(document.get_value(), - "Failed to get JSON value")); - - return ConsumeJsonValue(value); -} +ARROW_EXPORT Status ValidateJsonDocument(simdjson::ondemand::parser& parser, + simdjson::padded_string& json); -} // namespace internal -} // namespace arrow +} // namespace arrow::internal diff --git a/cpp/src/parquet/encryption/file_system_key_material_store.cc b/cpp/src/parquet/encryption/file_system_key_material_store.cc index ece14b9664dc..4fce13f0d8c6 100644 --- a/cpp/src/parquet/encryption/file_system_key_material_store.cc +++ b/cpp/src/parquet/encryption/file_system_key_material_store.cc @@ -20,9 +20,9 @@ #include "arrow/buffer.h" #include "arrow/filesystem/filesystem.h" #include "arrow/filesystem/path_util.h" -#include "arrow/json/json_writer_internal.h" #include "arrow/json/object_parser.h" #include "arrow/result.h" +#include "arrow/util/json_writer_internal.h" #include "parquet/encryption/file_system_key_material_store.h" #include "parquet/encryption/key_material.h" diff --git a/cpp/src/parquet/encryption/key_material.cc b/cpp/src/parquet/encryption/key_material.cc index 305ad5969852..827e1e445025 100644 --- a/cpp/src/parquet/encryption/key_material.cc +++ b/cpp/src/parquet/encryption/key_material.cc @@ -17,8 +17,8 @@ #include -#include "arrow/json/json_writer_internal.h" #include "arrow/json/object_parser.h" +#include "arrow/util/json_writer_internal.h" #include "parquet/encryption/key_material.h" #include "parquet/encryption/key_metadata.h" diff --git a/cpp/src/parquet/encryption/key_metadata.cc b/cpp/src/parquet/encryption/key_metadata.cc index 94253c87e3e8..fd213169ff80 100644 --- a/cpp/src/parquet/encryption/key_metadata.cc +++ b/cpp/src/parquet/encryption/key_metadata.cc @@ -17,8 +17,8 @@ #include -#include "arrow/json/json_writer_internal.h" #include "arrow/json/object_parser.h" +#include "arrow/util/json_writer_internal.h" #include "parquet/encryption/key_metadata.h" #include "parquet/exception.h" diff --git a/cpp/src/parquet/encryption/local_wrap_kms_client.cc b/cpp/src/parquet/encryption/local_wrap_kms_client.cc index dcb6c49836e0..c2d336de6815 100644 --- a/cpp/src/parquet/encryption/local_wrap_kms_client.cc +++ b/cpp/src/parquet/encryption/local_wrap_kms_client.cc @@ -17,8 +17,8 @@ #include -#include "arrow/json/json_writer_internal.h" #include "arrow/json/object_parser.h" +#include "arrow/util/json_writer_internal.h" #include "arrow/util/secure_string.h" #include "parquet/encryption/key_toolkit_internal.h" diff --git a/cpp/src/parquet/geospatial/util_json_internal.cc b/cpp/src/parquet/geospatial/util_json_internal.cc index 44efcfc8d764..9bdc82fe1d8e 100644 --- a/cpp/src/parquet/geospatial/util_json_internal.cc +++ b/cpp/src/parquet/geospatial/util_json_internal.cc @@ -20,8 +20,8 @@ #include #include "arrow/extension_type.h" -#include "arrow/json/json_writer_internal.h" #include "arrow/result.h" +#include "arrow/util/json_writer_internal.h" #include "arrow/util/simdjson_internal.h" #include "arrow/util/string.h" diff --git a/cpp/src/parquet/printer.cc b/cpp/src/parquet/printer.cc index dd6a21913bf6..053183c41b42 100644 --- a/cpp/src/parquet/printer.cc +++ b/cpp/src/parquet/printer.cc @@ -25,7 +25,7 @@ #include #include -#include "arrow/json/json_writer_internal.h" +#include "arrow/util/json_writer_internal.h" #include "arrow/util/key_value_metadata.h" #include "arrow/util/string.h" diff --git a/cpp/src/parquet/types.cc b/cpp/src/parquet/types.cc index 534c79201c0f..ba80a27137a7 100644 --- a/cpp/src/parquet/types.cc +++ b/cpp/src/parquet/types.cc @@ -24,11 +24,11 @@ #include #include -#include "arrow/json/json_writer_internal.h" #include "arrow/util/checked_cast.h" #include "arrow/util/compression.h" #include "arrow/util/decimal.h" #include "arrow/util/float16.h" +#include "arrow/util/json_writer_internal.h" #include "arrow/util/logging_internal.h" #include "parquet/exception.h" From 86e85bd69b09e387b282bb1d51387856479cd381 Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Thu, 27 Aug 2026 17:28:37 +0200 Subject: [PATCH 2/4] simplify meson change --- cpp/meson.build | 1 + cpp/src/arrow/meson.build | 4 ++-- cpp/src/arrow/util/meson.build | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cpp/meson.build b/cpp/meson.build index 023877e19d11..5532d866db33 100644 --- a/cpp/meson.build +++ b/cpp/meson.build @@ -104,6 +104,7 @@ needs_testing = (get_option('testing').enabled() or needs_integration ) needs_json = get_option('json').enabled() or needs_testing +needs_simdjson = needs_json or needs_parquet needs_brotli = get_option('brotli').enabled() or needs_fuzzing needs_bz2 = get_option('bz2').enabled() needs_lz4 = get_option('lz4').enabled() diff --git a/cpp/src/arrow/meson.build b/cpp/src/arrow/meson.build index 298c7a5e02c1..5c475902d076 100644 --- a/cpp/src/arrow/meson.build +++ b/cpp/src/arrow/meson.build @@ -18,7 +18,7 @@ dl_dep = dependency('dl') threads_dep = dependency('threads') -if needs_json or needs_integration or needs_parquet +if needs_simdjson simdjson_dep = dependency('simdjson', allow_fallback: false, required: false) if not simdjson_dep.found() @@ -240,7 +240,7 @@ arrow_util_srcs = [ arrow_util_deps = [threads_dep] -if needs_json or needs_integration or needs_parquet +if needs_simdjson arrow_util_srcs += [ 'util/json_writer_internal.cc', 'util/simdjson_internal.cc', diff --git a/cpp/src/arrow/util/meson.build b/cpp/src/arrow/util/meson.build index 48a39072e3a6..456b8c61c7a3 100644 --- a/cpp/src/arrow/util/meson.build +++ b/cpp/src/arrow/util/meson.build @@ -223,7 +223,7 @@ utility_test_srcs = [ ] utility_test_deps = [arrow_test_dep_no_main] -if needs_json or needs_integration or needs_parquet +if needs_simdjson utility_test_srcs += ['json_writer_internal_test.cc'] utility_test_deps += [simdjson_dep] endif From 4d93dff9b3396f3484749fb1e4c6f8937c16ac1f Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Thu, 27 Aug 2026 19:13:49 +0200 Subject: [PATCH 3/4] set ARROW_JSON for odbc build --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 8888f52f721b..e65f6d3f174f 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -411,11 +411,11 @@ if(ARROW_AZURE) set(ARROW_WITH_AZURE_SDK ON) endif() -if(ARROW_JSON OR ARROW_FLIGHT_SQL_ODBC) +if(ARROW_JSON) set(ARROW_WITH_RAPIDJSON ON) endif() -if(ARROW_JSON) +if(ARROW_JSON OR ARROW_FLIGHT_SQL_ODBC) set(ARROW_WITH_SIMDJSON ON) endif() From ab2c41a1cea6ad48205f727c8602aaaa5ecda805 Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Fri, 28 Aug 2026 12:49:16 +0200 Subject: [PATCH 4/4] GH-50859: [C++][Parquet] Build ObjectParser with simdjson utilities Fully qualify simdjson utility calls so CMake unity builds do not resolve internal to arrow::json::internal. --- cpp/cmake_modules/DefineOptions.cmake | 3 +-- cpp/cmake_modules/ThirdpartyToolchain.cmake | 1 - cpp/src/arrow/CMakeLists.txt | 7 +++++-- cpp/src/arrow/json/object_parser.h | 2 +- cpp/src/arrow/meson.build | 2 +- cpp/src/arrow/util/CMakeLists.txt | 2 +- cpp/src/arrow/util/json_writer_internal.cc | 18 +++++++++--------- 7 files changed, 18 insertions(+), 17 deletions(-) diff --git a/cpp/cmake_modules/DefineOptions.cmake b/cpp/cmake_modules/DefineOptions.cmake index bfe4485aa912..1d12edd0613e 100644 --- a/cpp/cmake_modules/DefineOptions.cmake +++ b/cpp/cmake_modules/DefineOptions.cmake @@ -599,8 +599,7 @@ takes precedence over ccache if a storage backend is configured" ON) "Build support for encryption. Fail if OpenSSL is not found" OFF DEPENDS - ARROW_FILESYSTEM - ARROW_JSON) + ARROW_FILESYSTEM) #---------------------------------------------------------------------- set_option_category("Gandiva") diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index e65f6d3f174f..0fc8f234355a 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -383,7 +383,6 @@ if(ARROW_WITH_OPENTELEMETRY) endif() if(ARROW_PARQUET) - set(ARROW_WITH_RAPIDJSON ON) set(ARROW_WITH_SIMDJSON ON) set(ARROW_WITH_THRIFT ON) endif() diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index ed8174f7176a..122012efbeda 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -645,7 +645,11 @@ if(ARROW_WITH_OPENTELEMETRY) list(APPEND ARROW_UTIL_SRCS util/tracing_internal.cc) endif() if(ARROW_WITH_SIMDJSON) - list(APPEND ARROW_UTIL_SRCS util/json_writer_internal.cc util/simdjson_internal.cc) + list(APPEND + ARROW_UTIL_SRCS + json/object_parser.cc + util/json_writer_internal.cc + util/simdjson_internal.cc) endif() if(ARROW_WITH_SNAPPY) list(APPEND ARROW_UTIL_SRCS util/compression_snappy.cc) @@ -1097,7 +1101,6 @@ if(ARROW_JSON) json/chunker.cc json/converter.cc json/from_string.cc - json/object_parser.cc json/parser.cc json/reader.cc) foreach(ARROW_JSON_TARGET ${ARROW_JSON_TARGETS}) diff --git a/cpp/src/arrow/json/object_parser.h b/cpp/src/arrow/json/object_parser.h index 8035695e537c..bd04525efa50 100644 --- a/cpp/src/arrow/json/object_parser.h +++ b/cpp/src/arrow/json/object_parser.h @@ -29,7 +29,7 @@ namespace json { namespace internal { /// This class is a helper to parse a json object from a string. -/// It uses rapidjson::Document in implementation. +/// It uses simdjson in the implementation. class ARROW_EXPORT ObjectParser { public: ObjectParser(); diff --git a/cpp/src/arrow/meson.build b/cpp/src/arrow/meson.build index 5c475902d076..3315faccd860 100644 --- a/cpp/src/arrow/meson.build +++ b/cpp/src/arrow/meson.build @@ -242,6 +242,7 @@ arrow_util_deps = [threads_dep] if needs_simdjson arrow_util_srcs += [ + 'json/object_parser.cc', 'util/json_writer_internal.cc', 'util/simdjson_internal.cc', ] @@ -538,7 +539,6 @@ if needs_json 'json/chunker.cc', 'json/converter.cc', 'json/from_string.cc', - 'json/object_parser.cc', 'json/options.cc', 'json/parser.cc', 'json/reader.cc', diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt index f77c8faa0caf..2744737eb8f2 100644 --- a/cpp/src/arrow/util/CMakeLists.txt +++ b/cpp/src/arrow/util/CMakeLists.txt @@ -92,7 +92,7 @@ if(ARROW_WITH_SIMDJSON) SOURCES json_writer_internal_test.cc EXTRA_LINK_LIBS - simdjson::simdjson) + arrow::simdjson) endif() add_arrow_test(async-utility-test diff --git a/cpp/src/arrow/util/json_writer_internal.cc b/cpp/src/arrow/util/json_writer_internal.cc index 15175f0d313f..26d58d9b20f2 100644 --- a/cpp/src/arrow/util/json_writer_internal.cc +++ b/cpp/src/arrow/util/json_writer_internal.cc @@ -100,21 +100,21 @@ void JsonWriter::Double(double value) { } Status JsonWriter::WriteValue(sj::value value) { - return internal::VisitJsonValue( + return ::arrow::internal::VisitJsonValue( value, [&](sj::object object) -> Status { StartObject(); for (auto field : object) { - ARROW_ASSIGN_OR_RAISE( - auto key, internal::ResolveSimdjsonResult(field.unescaped_key(), - "Failed to get object key")); + ARROW_ASSIGN_OR_RAISE(auto key, + ::arrow::internal::ResolveSimdjsonResult( + field.unescaped_key(), "Failed to get object key")); Key(key); ARROW_ASSIGN_OR_RAISE(auto field_value, - internal::ResolveSimdjsonResult( + ::arrow::internal::ResolveSimdjsonResult( field.value(), "Failed to get object value")); RETURN_NOT_OK(WriteValue(field_value)); @@ -128,9 +128,9 @@ Status JsonWriter::WriteValue(sj::value value) { StartArray(); for (auto element : array) { - ARROW_ASSIGN_OR_RAISE( - auto element_value, - internal::ResolveSimdjsonResult(element, "Failed to iterate JSON array")); + ARROW_ASSIGN_OR_RAISE(auto element_value, + ::arrow::internal::ResolveSimdjsonResult( + element, "Failed to iterate JSON array")); RETURN_NOT_OK(WriteValue(element_value)); } @@ -170,7 +170,7 @@ Status JsonWriter::WriteValue(sj::value value) { }, [&](sj::value value) -> Status { - ARROW_ASSIGN_OR_RAISE(auto raw_json, internal::ResolveSimdjsonResult( + ARROW_ASSIGN_OR_RAISE(auto raw_json, ::arrow::internal::ResolveSimdjsonResult( simdjson::to_json_string(value), "Failed to get raw JSON")); RawValue(raw_json);