From 95a3b62ab06475d383416cec099fedd7b2645a13 Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Sat, 29 Aug 2026 23:53:35 +0530 Subject: [PATCH 01/21] Replace header --- cpp/src/arrow/json/parser.cc | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc index 53f856d8012e..2d99c9d9acea 100644 --- a/cpp/src/arrow/json/parser.cc +++ b/cpp/src/arrow/json/parser.cc @@ -27,10 +27,6 @@ #include #include -#include "arrow/json/rapidjson_defs.h" -#include "rapidjson/error/en.h" -#include "rapidjson/reader.h" - #include "arrow/array.h" #include "arrow/array/builder_binary.h" #include "arrow/buffer_builder.h" @@ -38,6 +34,7 @@ #include "arrow/util/bitset_stack_internal.h" #include "arrow/util/checked_cast.h" #include "arrow/util/logging_internal.h" +#include "arrow/util/simdjson_internal.h" #include "arrow/util/trie_internal.h" #include "arrow/visit_type_inline.h" @@ -48,7 +45,7 @@ using internal::checked_cast; namespace json { -namespace rj = arrow::rapidjson; +namespace sj = simdjson::ondemand; template static Status ParseError(T&&... t) { From d697915b3357a5990307392a48d63de9819e4359 Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Sun, 30 Aug 2026 01:27:51 +0530 Subject: [PATCH 02/21] replace parser logic --- cpp/src/arrow/json/parser.cc | 476 +++++++++++++----------------- cpp/src/arrow/json/reader_test.cc | 2 +- 2 files changed, 212 insertions(+), 266 deletions(-) diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc index 2d99c9d9acea..417967de3c86 100644 --- a/cpp/src/arrow/json/parser.cc +++ b/cpp/src/arrow/json/parser.cc @@ -52,6 +52,15 @@ static Status ParseError(T&&... t) { return Status::Invalid("JSON parse error: ", std::forward(t)...); } +static std::string_view TrimTrailingWhitespace(std::string_view value) { + while (!value.empty() && + (value.back() == ' ' || value.back() == '\t' || + value.back() == '\n' || value.back() == '\r')) { + value.remove_suffix(1); + } + return value; +} + const std::string& Kind::Name(Kind::type kind) { static const std::string names[] = { "null", "boolean", "number", "string", "array", "object", "number_or_string", @@ -644,8 +653,7 @@ class RawBuilderSet { /// Three implementations are provided for BlockParser, one for each /// UnexpectedFieldBehavior. However most of the logic is identical in each /// case, so the majority of the implementation is in this base class -class HandlerBase : public BlockParser, - public rj::BaseReaderHandler, HandlerBase> { +class HandlerBase : public BlockParser { public: explicit HandlerBase(MemoryPool* pool) : BlockParser(pool), @@ -662,68 +670,32 @@ class HandlerBase : public BlockParser, /// Accessor for a stored error Status Status Error() { return status_; } - /// \defgroup rapidjson-handler-interface functions expected by rj::Reader - /// - /// bool Key(const char* data, rj::SizeType size, ...) is omitted since - /// the behavior varies greatly between UnexpectedFieldBehaviors - /// - /// @{ - bool Null() { - status_ = builder_set_.AppendNull(builder_stack_.back(), field_index_, builder_); - return status_.ok(); + Status Null() { + return builder_set_.AppendNull(builder_stack_.back(), field_index_, builder_); } - bool Bool(bool value) { + Status Bool(bool value) { constexpr auto kind = Kind::kBoolean; if (ARROW_PREDICT_FALSE(builder_.kind != kind)) { - status_ = IllegallyChangedTo(kind); - return status_.ok(); + return IllegallyChangedTo(kind); } - status_ = Cast(builder_)->Append(value); - return status_.ok(); + return Cast(builder_)->Append(value); } - bool RawNumber(const char* data, rj::SizeType size, ...) { + Status RawNumber(std::string_view value) { if (builder_.kind == Kind::kNumberOrString) { - status_ = - AppendScalar(builder_, std::string_view(data, size)); - } else { - status_ = AppendScalar(builder_, std::string_view(data, size)); + return AppendScalar(builder_, value); } - return status_.ok(); + return AppendScalar(builder_, value); } - bool String(const char* data, rj::SizeType size, ...) { + Status String(std::string_view value) { if (builder_.kind == Kind::kNumberOrString) { - status_ = - AppendScalar(builder_, std::string_view(data, size)); - } else { - status_ = AppendScalar(builder_, std::string_view(data, size)); + return AppendScalar(builder_, value); } - return status_.ok(); - } - - bool StartObject() { - status_ = StartObjectImpl(); - return status_.ok(); - } - - bool EndObject(...) { - status_ = EndObjectImpl(); - return status_.ok(); - } - - bool StartArray() { - status_ = StartArrayImpl(); - return status_.ok(); + return AppendScalar(builder_, value); } - bool EndArray(rj::SizeType size) { - status_ = EndArrayImpl(size); - return status_.ok(); - } - /// @} - /// \brief Set up builders using an expected Schema Status Initialize(const std::shared_ptr& s) { auto type = struct_({}); @@ -759,43 +731,191 @@ class HandlerBase : public BlockParser, } protected: - template - Status DoParse(Handler& handler, Stream&& json, size_t json_size) { - constexpr auto parse_flags = rj::kParseIterativeFlag | rj::kParseNanAndInfFlag | - rj::kParseStopWhenDoneFlag | - rj::kParseNumbersAsStringsFlag; - - rj::Reader reader; - // ensure that the loop can exit when the block too large. - for (; num_rows_ < std::numeric_limits::max(); ++num_rows_) { - auto ok = reader.Parse(json, handler); - switch (ok.Code()) { - case rj::kParseErrorNone: - // parse the next object - continue; - case rj::kParseErrorDocumentEmpty: - if (json.Tell() < json_size) { - return ParseError(rj::GetParseError_En(ok.Code())); - } - // parsed all objects, finish - return Status::OK(); - case rj::kParseErrorTermination: - // handler emitted an error - return handler.Error(); - default: - // rj emitted an error - return ParseError(rj::GetParseError_En(ok.Code()), " in row ", num_rows_); + template + Status DoParse(Handler& handler, const std::shared_ptr& json) { + RETURN_NOT_OK(ReserveScalarStorage(json->size())); + + simdjson::padded_string padded_json( + reinterpret_cast(json->data()), json->size()); + + sj::parser parser; + + ARROW_ASSIGN_OR_RAISE( + auto stream, + internal::ResolveSimdjsonResult( + parser.iterate_many(padded_json), + "Failed to create JSON document stream")); + + for (auto document_result : stream) { + ARROW_ASSIGN_OR_RAISE( + auto document, + internal::ResolveSimdjsonResult( + document_result, "Failed to iterate JSON document stream")); + + if (num_rows_ == std::numeric_limits::max()) { + return Status::Invalid("Row count overflowed int32_t"); + } + + ARROW_ASSIGN_OR_RAISE( + auto value, + internal::ResolveSimdjsonResult( + document.get_value(), "JSON parse error: Failed to get JSON value")); + + RETURN_NOT_OK(ParseValue(handler, value)); + + ++num_rows_; + } + + if (stream.truncated_bytes() != 0) { + return ParseError("The document is empty"); + } + + return Status::OK(); + } + + template + Status MaybePromoteFromNull() { + if (builder_.kind != Kind::kNull) { + return Status::OK(); + } + + auto parent = builder_stack_.back(); + + if (parent.kind == Kind::kArray) { + auto list_builder = Cast(parent); + DCHECK_EQ(list_builder->value_builder(), builder_); + + RETURN_NOT_OK(builder_set_.MakeBuilder(builder_.index, &builder_)); + + list_builder = Cast(parent); + list_builder->value_builder(builder_); + } else { + auto struct_builder = Cast(parent); + DCHECK_EQ(struct_builder->field_builder(field_index_), builder_); + + RETURN_NOT_OK(builder_set_.MakeBuilder(builder_.index, &builder_)); + + struct_builder = Cast(parent); + struct_builder->field_builder(field_index_, builder_); + } + + return Status::OK(); + } + + template + Status ParseValue(Handler& handler, sj::value value) { + ARROW_ASSIGN_OR_RAISE( + auto type, + internal::ResolveSimdjsonResult( + value.type(), "Failed to determine JSON type")); + + switch (type) { + case sj::json_type::null: + return Null(); + + case sj::json_type::boolean: { + RETURN_NOT_OK(handler.template MaybePromoteFromNull()); + + ARROW_ASSIGN_OR_RAISE( + auto boolean, + internal::ResolveSimdjsonResult( + value.get_bool(), "Failed to get JSON boolean")); + return Bool(boolean); + } + + case sj::json_type::string: { + RETURN_NOT_OK(handler.template MaybePromoteFromNull()); + + ARROW_ASSIGN_OR_RAISE( + auto string, + internal::ResolveSimdjsonResult( + value.get_string(), "Failed to get JSON string")); + return String(string); + } + + case sj::json_type::number: { + RETURN_NOT_OK(handler.template MaybePromoteFromNull()); + return RawNumber(TrimTrailingWhitespace(value.raw_json_token())); } + + case sj::json_type::array: + RETURN_NOT_OK(handler.template MaybePromoteFromNull()); + return ParseArray(handler, value); + + case sj::json_type::object: + RETURN_NOT_OK(handler.template MaybePromoteFromNull()); + return ParseObject(handler, value); + + default: + return ParseError("Invalid value"); } - return Status::Invalid("Row count overflowed int32_t"); } template - Status DoParse(Handler& handler, const std::shared_ptr& json) { - RETURN_NOT_OK(ReserveScalarStorage(json->size())); - rj::MemoryStream ms(reinterpret_cast(json->data()), json->size()); - using InputStream = rj::EncodedInputStream, rj::MemoryStream>; - return DoParse(handler, InputStream(ms), static_cast(json->size())); + Status ParseArray(Handler& handler, sj::value value) { + RETURN_NOT_OK(StartArrayImpl()); + + ARROW_ASSIGN_OR_RAISE( + auto array, + internal::ResolveSimdjsonResult( + value.get_array(), "Failed to get JSON array")); + + size_t size = 0; + + for (auto element_result : array) { + ARROW_ASSIGN_OR_RAISE( + auto element, + internal::ResolveSimdjsonResult( + element_result, "Failed to iterate JSON array")); + + RETURN_NOT_OK(ParseValue(handler, element)); + ++size; + } + + return EndArrayImpl(size); + } + + template + Status ParseObject(Handler& handler, sj::value value) { + RETURN_NOT_OK(StartObjectImpl()); + + ARROW_ASSIGN_OR_RAISE( + auto object, + internal::ResolveSimdjsonResult( + value.get_object(), "Failed to get JSON object")); + + for (auto field_result : object) { + ARROW_ASSIGN_OR_RAISE( + auto field, + internal::ResolveSimdjsonResult( + field_result, "JSON parse error: Failed to iterate JSON object")); + + ARROW_ASSIGN_OR_RAISE( + auto key, + internal::ResolveSimdjsonResult( + field.unescaped_key(), "Failed to get JSON object key")); + + auto field_value = field.value(); + + RETURN_NOT_OK(ParseObjectField(handler, key, field_value)); + } + + return EndObjectImpl(); + } + + template + Status ParseObjectField(Handler& handler, std::string_view key, sj::value value) { + bool duplicate_keys = false; + + if (SetFieldBuilder(key, &duplicate_keys)) { + return ParseValue(handler, value); + } + + if (duplicate_keys) { + return status_; + } + + return handler.HandleUnexpectedField(key, value); } /// \defgroup handlerbase-append-methods append non-nested values @@ -884,7 +1004,7 @@ class HandlerBase : public BlockParser, return Status::OK(); } - Status EndArrayImpl(rj::SizeType size) { + Status EndArrayImpl(size_t size) { EndNested(); // append to list_builder here auto list_builder = Cast(builder_); @@ -952,20 +1072,10 @@ class Handler : public HandlerBase { return DoParse(*this, json); } - /// \ingroup rapidjson-handler-interface - /// - /// if an unexpected field is encountered, emit a parse error and bail - bool Key(const char* key, rj::SizeType len, ...) { - bool duplicate_keys = false; - if (ARROW_PREDICT_FALSE( - SetFieldBuilder(std::string_view(key, len), &duplicate_keys))) { - return true; - } - if (!duplicate_keys) { - status_ = ParseError("unexpected field"); - } - return false; + Status HandleUnexpectedField(std::string_view, sj::value) { + return ParseError("unexpected field"); } + }; template <> @@ -977,96 +1087,9 @@ class Handler : public HandlerBase { return DoParse(*this, json); } - bool Null() { - if (Skipping()) { - return true; - } - return HandlerBase::Null(); - } - - bool Bool(bool value) { - if (Skipping()) { - return true; - } - return HandlerBase::Bool(value); - } - - bool RawNumber(const char* data, rj::SizeType size, ...) { - if (Skipping()) { - return true; - } - return HandlerBase::RawNumber(data, size); - } - - bool String(const char* data, rj::SizeType size, ...) { - if (Skipping()) { - return true; - } - return HandlerBase::String(data, size); - } - - bool StartObject() { - ++depth_; - if (Skipping()) { - return true; - } - return HandlerBase::StartObject(); - } - - /// \ingroup rapidjson-handler-interface - /// - /// if an unexpected field is encountered, skip until its value has been consumed - bool Key(const char* key, rj::SizeType len, ...) { - MaybeStopSkipping(); - if (Skipping()) { - return true; - } - bool duplicate_keys = false; - if (ARROW_PREDICT_TRUE( - SetFieldBuilder(std::string_view(key, len), &duplicate_keys))) { - return true; - } - if (ARROW_PREDICT_FALSE(duplicate_keys)) { - return false; - } - skip_depth_ = depth_; - return true; - } - - bool EndObject(...) { - MaybeStopSkipping(); - --depth_; - if (Skipping()) { - return true; - } - return HandlerBase::EndObject(); - } - - bool StartArray() { - if (Skipping()) { - return true; - } - return HandlerBase::StartArray(); - } - - bool EndArray(rj::SizeType size) { - if (Skipping()) { - return true; - } - return HandlerBase::EndArray(size); - } - - private: - bool Skipping() { return depth_ >= skip_depth_; } - - void MaybeStopSkipping() { - if (skip_depth_ == depth_) { - skip_depth_ = std::numeric_limits::max(); - } + Status HandleUnexpectedField(std::string_view, sj::value) { + return Status::OK(); } - - int depth_ = 0; - int skip_depth_ = std::numeric_limits::max(); }; template <> @@ -1078,91 +1101,14 @@ class Handler : public HandlerBase { return DoParse(*this, json); } - bool Bool(bool value) { - if (ARROW_PREDICT_FALSE(MaybePromoteFromNull())) { - return false; - } - return HandlerBase::Bool(value); - } - - bool RawNumber(const char* data, rj::SizeType size, ...) { - if (ARROW_PREDICT_FALSE(MaybePromoteFromNull())) { - return false; - } - return HandlerBase::RawNumber(data, size); - } - - bool String(const char* data, rj::SizeType size, ...) { - if (ARROW_PREDICT_FALSE(MaybePromoteFromNull())) { - return false; - } - return HandlerBase::String(data, size); - } - - bool StartObject() { - if (ARROW_PREDICT_FALSE(MaybePromoteFromNull())) { - return false; - } - return HandlerBase::StartObject(); - } - - /// \ingroup rapidjson-handler-interface - /// - /// If an unexpected field is encountered, add a new builder to - /// the current parent builder. It is added as a NullBuilder with - /// (parent.length - 1) leading nulls. The next value parsed - /// will probably trigger promotion of this field from null - bool Key(const char* key, rj::SizeType len, ...) { - bool duplicate_keys = false; - if (ARROW_PREDICT_TRUE( - SetFieldBuilder(std::string_view(key, len), &duplicate_keys))) { - return true; - } - if (ARROW_PREDICT_FALSE(duplicate_keys)) { - return false; - } + Status HandleUnexpectedField(std::string_view key, sj::value value) { auto struct_builder = Cast(builder_stack_.back()); auto leading_nulls = static_cast(struct_builder->length() - 1); - builder_ = BuilderPtr(Kind::kNull, leading_nulls, true); - field_index_ = struct_builder->AddField(std::string_view(key, len), builder_); - return true; - } - bool StartArray() { - if (ARROW_PREDICT_FALSE(MaybePromoteFromNull())) { - return false; - } - return HandlerBase::StartArray(); - } + builder_ = BuilderPtr(Kind::kNull, leading_nulls, true); + field_index_ = struct_builder->AddField(key, builder_); - private: - // return true if a terminal error was encountered - template - bool MaybePromoteFromNull() { - if (ARROW_PREDICT_TRUE(builder_.kind != Kind::kNull)) { - return false; - } - auto parent = builder_stack_.back(); - if (parent.kind == Kind::kArray) { - auto list_builder = Cast(parent); - DCHECK_EQ(list_builder->value_builder(), builder_); - status_ = builder_set_.MakeBuilder(builder_.index, &builder_); - if (ARROW_PREDICT_FALSE(!status_.ok())) { - return true; - } - list_builder = Cast(parent); - list_builder->value_builder(builder_); - } else { - auto struct_builder = Cast(parent); - DCHECK_EQ(struct_builder->field_builder(field_index_), builder_); - status_ = builder_set_.MakeBuilder(builder_.index, &builder_); - if (ARROW_PREDICT_FALSE(!status_.ok())) { - return true; - } - struct_builder = Cast(parent); - struct_builder->field_builder(field_index_, builder_); - } - return false; + return ParseValue(*this, value); } }; diff --git a/cpp/src/arrow/json/reader_test.cc b/cpp/src/arrow/json/reader_test.cc index 7dcd30a0eb0b..53c082cbb844 100644 --- a/cpp/src/arrow/json/reader_test.cc +++ b/cpp/src/arrow/json/reader_test.cc @@ -714,7 +714,7 @@ TEST_P(StreamingReaderTest, PropagateParsingErrors) { EXPECT_RAISES_WITH_MESSAGE_THAT( Invalid, ::testing::StartsWith( - "Invalid: JSON parse error: Missing a comma or '}' after an object member"), + "Invalid: JSON parse error"), reader->ReadNext(&batch)); EXPECT_EQ(reader->bytes_processed(), 13); AssertReadEnd(reader); From 475e7cd61ad751f226b6ce5b34ba2fe098a92bf7 Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Sun, 30 Aug 2026 01:35:50 +0530 Subject: [PATCH 03/21] Lint Fix --- cpp/src/arrow/json/parser.cc | 66 ++++++++++++------------------- cpp/src/arrow/json/reader_test.cc | 8 ++-- 2 files changed, 28 insertions(+), 46 deletions(-) diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc index 417967de3c86..0433ff88791d 100644 --- a/cpp/src/arrow/json/parser.cc +++ b/cpp/src/arrow/json/parser.cc @@ -53,9 +53,8 @@ static Status ParseError(T&&... t) { } static std::string_view TrimTrailingWhitespace(std::string_view value) { - while (!value.empty() && - (value.back() == ' ' || value.back() == '\t' || - value.back() == '\n' || value.back() == '\r')) { + while (!value.empty() && (value.back() == ' ' || value.back() == '\t' || + value.back() == '\n' || value.back() == '\r')) { value.remove_suffix(1); } return value; @@ -735,22 +734,19 @@ class HandlerBase : public BlockParser { Status DoParse(Handler& handler, const std::shared_ptr& json) { RETURN_NOT_OK(ReserveScalarStorage(json->size())); - simdjson::padded_string padded_json( - reinterpret_cast(json->data()), json->size()); + simdjson::padded_string padded_json(reinterpret_cast(json->data()), + json->size()); sj::parser parser; - ARROW_ASSIGN_OR_RAISE( - auto stream, - internal::ResolveSimdjsonResult( - parser.iterate_many(padded_json), - "Failed to create JSON document stream")); + ARROW_ASSIGN_OR_RAISE(auto stream, internal::ResolveSimdjsonResult( + parser.iterate_many(padded_json), + "Failed to create JSON document stream")); for (auto document_result : stream) { ARROW_ASSIGN_OR_RAISE( - auto document, - internal::ResolveSimdjsonResult( - document_result, "Failed to iterate JSON document stream")); + auto document, internal::ResolveSimdjsonResult( + document_result, "Failed to iterate JSON document stream")); if (num_rows_ == std::numeric_limits::max()) { return Status::Invalid("Row count overflowed int32_t"); @@ -758,8 +754,8 @@ class HandlerBase : public BlockParser { ARROW_ASSIGN_OR_RAISE( auto value, - internal::ResolveSimdjsonResult( - document.get_value(), "JSON parse error: Failed to get JSON value")); + internal::ResolveSimdjsonResult(document.get_value(), + "JSON parse error: Failed to get JSON value")); RETURN_NOT_OK(ParseValue(handler, value)); @@ -804,10 +800,8 @@ class HandlerBase : public BlockParser { template Status ParseValue(Handler& handler, sj::value value) { - ARROW_ASSIGN_OR_RAISE( - auto type, - internal::ResolveSimdjsonResult( - value.type(), "Failed to determine JSON type")); + ARROW_ASSIGN_OR_RAISE(auto type, internal::ResolveSimdjsonResult( + value.type(), "Failed to determine JSON type")); switch (type) { case sj::json_type::null: @@ -817,9 +811,8 @@ class HandlerBase : public BlockParser { RETURN_NOT_OK(handler.template MaybePromoteFromNull()); ARROW_ASSIGN_OR_RAISE( - auto boolean, - internal::ResolveSimdjsonResult( - value.get_bool(), "Failed to get JSON boolean")); + auto boolean, internal::ResolveSimdjsonResult(value.get_bool(), + "Failed to get JSON boolean")); return Bool(boolean); } @@ -827,9 +820,8 @@ class HandlerBase : public BlockParser { RETURN_NOT_OK(handler.template MaybePromoteFromNull()); ARROW_ASSIGN_OR_RAISE( - auto string, - internal::ResolveSimdjsonResult( - value.get_string(), "Failed to get JSON string")); + auto string, internal::ResolveSimdjsonResult(value.get_string(), + "Failed to get JSON string")); return String(string); } @@ -855,18 +847,15 @@ class HandlerBase : public BlockParser { Status ParseArray(Handler& handler, sj::value value) { RETURN_NOT_OK(StartArrayImpl()); - ARROW_ASSIGN_OR_RAISE( - auto array, - internal::ResolveSimdjsonResult( - value.get_array(), "Failed to get JSON array")); + ARROW_ASSIGN_OR_RAISE(auto array, internal::ResolveSimdjsonResult( + value.get_array(), "Failed to get JSON array")); size_t size = 0; for (auto element_result : array) { ARROW_ASSIGN_OR_RAISE( - auto element, - internal::ResolveSimdjsonResult( - element_result, "Failed to iterate JSON array")); + auto element, internal::ResolveSimdjsonResult(element_result, + "Failed to iterate JSON array")); RETURN_NOT_OK(ParseValue(handler, element)); ++size; @@ -881,8 +870,7 @@ class HandlerBase : public BlockParser { ARROW_ASSIGN_OR_RAISE( auto object, - internal::ResolveSimdjsonResult( - value.get_object(), "Failed to get JSON object")); + internal::ResolveSimdjsonResult(value.get_object(), "Failed to get JSON object")); for (auto field_result : object) { ARROW_ASSIGN_OR_RAISE( @@ -891,9 +879,8 @@ class HandlerBase : public BlockParser { field_result, "JSON parse error: Failed to iterate JSON object")); ARROW_ASSIGN_OR_RAISE( - auto key, - internal::ResolveSimdjsonResult( - field.unescaped_key(), "Failed to get JSON object key")); + auto key, internal::ResolveSimdjsonResult(field.unescaped_key(), + "Failed to get JSON object key")); auto field_value = field.value(); @@ -1075,7 +1062,6 @@ class Handler : public HandlerBase { Status HandleUnexpectedField(std::string_view, sj::value) { return ParseError("unexpected field"); } - }; template <> @@ -1087,9 +1073,7 @@ class Handler : public HandlerBase { return DoParse(*this, json); } - Status HandleUnexpectedField(std::string_view, sj::value) { - return Status::OK(); - } + Status HandleUnexpectedField(std::string_view, sj::value) { return Status::OK(); } }; template <> diff --git a/cpp/src/arrow/json/reader_test.cc b/cpp/src/arrow/json/reader_test.cc index 53c082cbb844..ffe33c00111a 100644 --- a/cpp/src/arrow/json/reader_test.cc +++ b/cpp/src/arrow/json/reader_test.cc @@ -711,11 +711,9 @@ TEST_P(StreamingReaderTest, PropagateParsingErrors) { EXPECT_EQ(reader->bytes_processed(), 13); ASSERT_BATCHES_EQUAL(*RecordBatchFromJSON(test_schema, R"([{"n":10000}])"), *batch); - EXPECT_RAISES_WITH_MESSAGE_THAT( - Invalid, - ::testing::StartsWith( - "Invalid: JSON parse error"), - reader->ReadNext(&batch)); + EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, + ::testing::StartsWith("Invalid: JSON parse error"), + reader->ReadNext(&batch)); EXPECT_EQ(reader->bytes_processed(), 13); AssertReadEnd(reader); EXPECT_EQ(reader->bytes_processed(), 13); From 8dcd806fea224eb2fc11197cdb123a5268220380 Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Sun, 30 Aug 2026 01:47:14 +0530 Subject: [PATCH 04/21] Fix CI --- cpp/src/arrow/json/parser.cc | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc index 0433ff88791d..e0d79999de49 100644 --- a/cpp/src/arrow/json/parser.cc +++ b/cpp/src/arrow/json/parser.cc @@ -60,6 +60,15 @@ static std::string_view TrimTrailingWhitespace(std::string_view value) { return value; } +static bool IsWhitespaceOnly(std::string_view value) { + for (const auto c : value) { + if (c != ' ' && c != '\t' && c != '\n' && c != '\r') { + return false; + } + } + return true; +} + const std::string& Kind::Name(Kind::type kind) { static const std::string names[] = { "null", "boolean", "number", "string", "array", "object", "number_or_string", @@ -734,6 +743,13 @@ class HandlerBase : public BlockParser { Status DoParse(Handler& handler, const std::shared_ptr& json) { RETURN_NOT_OK(ReserveScalarStorage(json->size())); + const std::string_view input(reinterpret_cast(json->data()), + json->size()); + + if (IsWhitespaceOnly(input)) { + return Status::OK(); + } + simdjson::padded_string padded_json(reinterpret_cast(json->data()), json->size()); @@ -995,7 +1011,8 @@ class HandlerBase : public BlockParser { EndNested(); // append to list_builder here auto list_builder = Cast(builder_); - return list_builder->Append(size); + DCHECK_LE(size, std::numeric_limits::max()); + return list_builder->Append(static_cast(size)); } /// helper method for StartArray and StartObject From b939078cac32649a8074b083cc9d12b89a082d57 Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Sun, 30 Aug 2026 11:12:51 +0530 Subject: [PATCH 05/21] Update Python test expectations --- python/pyarrow/tests/test_json.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/pyarrow/tests/test_json.py b/python/pyarrow/tests/test_json.py index ac2a027cfa24..8a8237cbc24c 100644 --- a/python/pyarrow/tests/test_json.py +++ b/python/pyarrow/tests/test_json.py @@ -476,8 +476,7 @@ def test_bad_middle_parse(self): } with pytest.raises( pa.ArrowInvalid, - match="JSON parse error:\ - Missing a comma or '}' after an object member*" + match="JSON parse error:" ): reader.read_next_batch() From d5b27fd8bb3175f6c737df5bbafef73c0c326556 Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Sun, 30 Aug 2026 11:26:47 +0530 Subject: [PATCH 06/21] resolve namespace error --- cpp/src/arrow/json/parser.cc | 42 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc index e0d79999de49..263366114c9b 100644 --- a/cpp/src/arrow/json/parser.cc +++ b/cpp/src/arrow/json/parser.cc @@ -755,13 +755,13 @@ class HandlerBase : public BlockParser { sj::parser parser; - ARROW_ASSIGN_OR_RAISE(auto stream, internal::ResolveSimdjsonResult( + ARROW_ASSIGN_OR_RAISE(auto stream, arrow::internal::ResolveSimdjsonResult( parser.iterate_many(padded_json), "Failed to create JSON document stream")); for (auto document_result : stream) { ARROW_ASSIGN_OR_RAISE( - auto document, internal::ResolveSimdjsonResult( + auto document, arrow::internal::ResolveSimdjsonResult( document_result, "Failed to iterate JSON document stream")); if (num_rows_ == std::numeric_limits::max()) { @@ -770,8 +770,8 @@ class HandlerBase : public BlockParser { ARROW_ASSIGN_OR_RAISE( auto value, - internal::ResolveSimdjsonResult(document.get_value(), - "JSON parse error: Failed to get JSON value")); + arrow::internal::ResolveSimdjsonResult( + document.get_value(), "JSON parse error: Failed to get JSON value")); RETURN_NOT_OK(ParseValue(handler, value)); @@ -816,7 +816,7 @@ class HandlerBase : public BlockParser { template Status ParseValue(Handler& handler, sj::value value) { - ARROW_ASSIGN_OR_RAISE(auto type, internal::ResolveSimdjsonResult( + ARROW_ASSIGN_OR_RAISE(auto type, arrow::internal::ResolveSimdjsonResult( value.type(), "Failed to determine JSON type")); switch (type) { @@ -826,18 +826,18 @@ class HandlerBase : public BlockParser { case sj::json_type::boolean: { RETURN_NOT_OK(handler.template MaybePromoteFromNull()); - ARROW_ASSIGN_OR_RAISE( - auto boolean, internal::ResolveSimdjsonResult(value.get_bool(), - "Failed to get JSON boolean")); + ARROW_ASSIGN_OR_RAISE(auto boolean, + arrow::internal::ResolveSimdjsonResult( + value.get_bool(), "Failed to get JSON boolean")); return Bool(boolean); } case sj::json_type::string: { RETURN_NOT_OK(handler.template MaybePromoteFromNull()); - ARROW_ASSIGN_OR_RAISE( - auto string, internal::ResolveSimdjsonResult(value.get_string(), - "Failed to get JSON string")); + ARROW_ASSIGN_OR_RAISE(auto string, + arrow::internal::ResolveSimdjsonResult( + value.get_string(), "Failed to get JSON string")); return String(string); } @@ -863,15 +863,15 @@ class HandlerBase : public BlockParser { Status ParseArray(Handler& handler, sj::value value) { RETURN_NOT_OK(StartArrayImpl()); - ARROW_ASSIGN_OR_RAISE(auto array, internal::ResolveSimdjsonResult( + ARROW_ASSIGN_OR_RAISE(auto array, arrow::internal::ResolveSimdjsonResult( value.get_array(), "Failed to get JSON array")); size_t size = 0; for (auto element_result : array) { - ARROW_ASSIGN_OR_RAISE( - auto element, internal::ResolveSimdjsonResult(element_result, - "Failed to iterate JSON array")); + ARROW_ASSIGN_OR_RAISE(auto element, + arrow::internal::ResolveSimdjsonResult( + element_result, "Failed to iterate JSON array")); RETURN_NOT_OK(ParseValue(handler, element)); ++size; @@ -885,18 +885,18 @@ class HandlerBase : public BlockParser { RETURN_NOT_OK(StartObjectImpl()); ARROW_ASSIGN_OR_RAISE( - auto object, - internal::ResolveSimdjsonResult(value.get_object(), "Failed to get JSON object")); + auto object, arrow::internal::ResolveSimdjsonResult(value.get_object(), + "Failed to get JSON object")); for (auto field_result : object) { ARROW_ASSIGN_OR_RAISE( auto field, - internal::ResolveSimdjsonResult( + arrow::internal::ResolveSimdjsonResult( field_result, "JSON parse error: Failed to iterate JSON object")); - ARROW_ASSIGN_OR_RAISE( - auto key, internal::ResolveSimdjsonResult(field.unescaped_key(), - "Failed to get JSON object key")); + ARROW_ASSIGN_OR_RAISE(auto key, + arrow::internal::ResolveSimdjsonResult( + field.unescaped_key(), "Failed to get JSON object key")); auto field_value = field.value(); From bc1b9d7baed45a4d71fc554818750974b62750fe Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Mon, 7 Sep 2026 09:05:46 +0530 Subject: [PATCH 07/21] make parser persistent --- cpp/src/arrow/json/parser.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc index 263366114c9b..6a6933e76b19 100644 --- a/cpp/src/arrow/json/parser.cc +++ b/cpp/src/arrow/json/parser.cc @@ -753,10 +753,8 @@ class HandlerBase : public BlockParser { simdjson::padded_string padded_json(reinterpret_cast(json->data()), json->size()); - sj::parser parser; - ARROW_ASSIGN_OR_RAISE(auto stream, arrow::internal::ResolveSimdjsonResult( - parser.iterate_many(padded_json), + parser_.iterate_many(padded_json), "Failed to create JSON document stream")); for (auto document_result : stream) { @@ -1062,6 +1060,7 @@ class HandlerBase : public BlockParser { // top of this stack == field_index_ std::vector field_index_stack_; StringBuilder scalar_values_builder_; + sj::parser parser_; }; template From 3eacd65a299eb19fbca766d182511ae8d12ddd58 Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Tue, 8 Sep 2026 18:14:59 +0530 Subject: [PATCH 08/21] use padded_string_view --- cpp/src/arrow/json/parser.cc | 55 +++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc index 6a6933e76b19..80e4ffdf6871 100644 --- a/cpp/src/arrow/json/parser.cc +++ b/cpp/src/arrow/json/parser.cc @@ -750,37 +750,48 @@ class HandlerBase : public BlockParser { return Status::OK(); } - simdjson::padded_string padded_json(reinterpret_cast(json->data()), - json->size()); + auto parse = [&](const auto& input) -> Status { + ARROW_ASSIGN_OR_RAISE(auto stream, arrow::internal::ResolveSimdjsonResult( + parser_.iterate_many(input), + "Failed to create JSON document stream")); + + for (auto document_result : stream) { + ARROW_ASSIGN_OR_RAISE( + auto document, + arrow::internal::ResolveSimdjsonResult( + document_result, "Failed to iterate JSON document stream")); + + if (num_rows_ == std::numeric_limits::max()) { + return Status::Invalid("Row count overflowed int32_t"); + } - ARROW_ASSIGN_OR_RAISE(auto stream, arrow::internal::ResolveSimdjsonResult( - parser_.iterate_many(padded_json), - "Failed to create JSON document stream")); + ARROW_ASSIGN_OR_RAISE( + auto value, + arrow::internal::ResolveSimdjsonResult( + document.get_value(), "JSON parse error: Failed to get JSON value")); - for (auto document_result : stream) { - ARROW_ASSIGN_OR_RAISE( - auto document, arrow::internal::ResolveSimdjsonResult( - document_result, "Failed to iterate JSON document stream")); + RETURN_NOT_OK(ParseValue(handler, value)); - if (num_rows_ == std::numeric_limits::max()) { - return Status::Invalid("Row count overflowed int32_t"); + ++num_rows_; } - ARROW_ASSIGN_OR_RAISE( - auto value, - arrow::internal::ResolveSimdjsonResult( - document.get_value(), "JSON parse error: Failed to get JSON value")); - - RETURN_NOT_OK(ParseValue(handler, value)); + if (stream.truncated_bytes() != 0) { + return ParseError("The document is empty"); + } - ++num_rows_; - } + return Status::OK(); + }; - if (stream.truncated_bytes() != 0) { - return ParseError("The document is empty"); + if (json->capacity() - json->size() >= + static_cast(simdjson::SIMDJSON_PADDING)) { + const auto padded_json = simdjson::padded_string_view( + reinterpret_cast(json->data()), json->size(), json->capacity()); + return parse(padded_json); } - return Status::OK(); + simdjson::padded_string padded_json(reinterpret_cast(json->data()), + json->size()); + return parse(padded_json); } template From 27d40e6c01c6a260171f9d5c392856f07c8277e6 Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Wed, 16 Sep 2026 14:33:01 +0530 Subject: [PATCH 09/21] fix emscripten --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 56f781bb7cf6..b65a13c8a0f6 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2833,6 +2833,7 @@ function(build_simdjson) URL_HASH "SHA256=${ARROW_SIMDJSON_BUILD_SHA256_CHECKSUM}") prepare_fetchcontent() + set(SIMDJSON_ENABLE_THREADS ${ARROW_ENABLE_THREADING}) fetchcontent_makeavailable(simdjson) From f247da72bb5e4ca7894a274fd7384777250d3b42 Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Thu, 17 Sep 2026 10:58:33 +0530 Subject: [PATCH 10/21] Address feedback --- cpp/src/arrow/json/parser.cc | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc index 80e4ffdf6871..b83590997309 100644 --- a/cpp/src/arrow/json/parser.cc +++ b/cpp/src/arrow/json/parser.cc @@ -693,15 +693,17 @@ class HandlerBase : public BlockParser { Status RawNumber(std::string_view value) { if (builder_.kind == Kind::kNumberOrString) { return AppendScalar(builder_, value); + } else { + return AppendScalar(builder_, value); } - return AppendScalar(builder_, value); } Status String(std::string_view value) { if (builder_.kind == Kind::kNumberOrString) { return AppendScalar(builder_, value); + } else { + return AppendScalar(builder_, value); } - return AppendScalar(builder_, value); } /// \brief Set up builders using an expected Schema @@ -829,8 +831,12 @@ class HandlerBase : public BlockParser { value.type(), "Failed to determine JSON type")); switch (type) { - case sj::json_type::null: + case sj::json_type::null: { + ARROW_ASSIGN_OR_RAISE([[maybe_unused]] auto is_null, + arrow::internal::ResolveSimdjsonResult( + value.is_null(), "Failed to validate JSON null")); return Null(); + } case sj::json_type::boolean: { RETURN_NOT_OK(handler.template MaybePromoteFromNull()); From 78a6431c98edd07c68e7d44344c9f8f676f38100 Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Mon, 21 Sep 2026 19:45:19 +0530 Subject: [PATCH 11/21] Add comment --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index b65a13c8a0f6..366ea0e60fc7 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2833,6 +2833,9 @@ function(build_simdjson) URL_HASH "SHA256=${ARROW_SIMDJSON_BUILD_SHA256_CHECKSUM}") prepare_fetchcontent() + + # Keep simdjson's threading configuration consistent with Arrow's, + # which is required for Emscripten where Arrow threading is disabled. set(SIMDJSON_ENABLE_THREADS ${ARROW_ENABLE_THREADING}) fetchcontent_makeavailable(simdjson) From 59bee5bf79f88edb6d4f09103f262526d164efed Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Mon, 21 Sep 2026 19:46:11 +0530 Subject: [PATCH 12/21] Make ConsumeJsonWhitespace a common helper --- cpp/src/arrow/json/chunker.cc | 12 +----------- cpp/src/arrow/json/parser.cc | 13 ++++--------- cpp/src/arrow/util/simdjson_internal.cc | 20 ++++++++++++++++++++ cpp/src/arrow/util/simdjson_internal.h | 2 ++ 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/cpp/src/arrow/json/chunker.cc b/cpp/src/arrow/json/chunker.cc index f69c45b93867..1b6f2f3d3c61 100644 --- a/cpp/src/arrow/json/chunker.cc +++ b/cpp/src/arrow/json/chunker.cc @@ -32,16 +32,6 @@ namespace arrow { namespace json { namespace { -// XXX We could try to SIMD-accelerate this routine but it's called only -// once per chunk and also will presumably examine a minimal amount of bytes. -int64_t ConsumeWhitespace(std::string_view view) { - const auto ws_count = view.find_first_not_of(" \t\r\n"); - if (ws_count == std::string_view::npos) { - return view.size(); - } - return static_cast(ws_count); -} - Status ConsumeDocument(simdjson::ondemand::document_stream::iterator& it) { ARROW_ASSIGN_OR_RAISE( auto document, internal::ResolveSimdjsonResult(*it, "Failed to get JSON document")); @@ -160,7 +150,7 @@ class ParsingBoundaryFinder : public BoundaryFinder { if (consumed_length > 0) { // If we found at least one document, also consume its trailing whitespace // to avoid stray bytes at the end of the stream. - consumed_length += ConsumeWhitespace(input.substr(consumed_length)); + consumed_length += internal::ConsumeJsonWhitespace(input.substr(consumed_length), /*trailing=*/false); } return consumed_length; } diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc index b83590997309..5682953ab543 100644 --- a/cpp/src/arrow/json/parser.cc +++ b/cpp/src/arrow/json/parser.cc @@ -52,14 +52,6 @@ static Status ParseError(T&&... t) { return Status::Invalid("JSON parse error: ", std::forward(t)...); } -static std::string_view TrimTrailingWhitespace(std::string_view value) { - while (!value.empty() && (value.back() == ' ' || value.back() == '\t' || - value.back() == '\n' || value.back() == '\r')) { - value.remove_suffix(1); - } - return value; -} - static bool IsWhitespaceOnly(std::string_view value) { for (const auto c : value) { if (c != ' ' && c != '\t' && c != '\n' && c != '\r') { @@ -858,7 +850,10 @@ class HandlerBase : public BlockParser { case sj::json_type::number: { RETURN_NOT_OK(handler.template MaybePromoteFromNull()); - return RawNumber(TrimTrailingWhitespace(value.raw_json_token())); + auto raw_number = value.raw_json_token(); + raw_number.remove_suffix( + internal::ConsumeJsonWhitespace(raw_number, /*trailing=*/true)); + return RawNumber(raw_number); } case sj::json_type::array: diff --git a/cpp/src/arrow/util/simdjson_internal.cc b/cpp/src/arrow/util/simdjson_internal.cc index 146b48d9eb7f..363431fd8d08 100644 --- a/cpp/src/arrow/util/simdjson_internal.cc +++ b/cpp/src/arrow/util/simdjson_internal.cc @@ -581,4 +581,24 @@ Status ValidateJsonDocument(simdjson::ondemand::parser& parser, return ConsumeJsonValue(value); } +// XXX We could try to SIMD-accelerate this routine but it's called only +// once per chunk and also will presumably examine a minimal amount of bytes. +int64_t ConsumeJsonWhitespace(std::string_view view, bool trailing) { + if (!trailing) { + const auto pos = view.find_first_not_of(" \t\r\n"); + return static_cast( + pos == std::string_view::npos ? view.size() : pos); + } + + int64_t count = 0; + while (count < static_cast(view.size())) { + const auto c = view[view.size() - count - 1]; + if (c != ' ' && c != '\t' && c != '\r' && c != '\n') { + break; + } + ++count; + } + return count; +} + } // namespace arrow::internal diff --git a/cpp/src/arrow/util/simdjson_internal.h b/cpp/src/arrow/util/simdjson_internal.h index 5c52f7648c6a..90f293b30788 100644 --- a/cpp/src/arrow/util/simdjson_internal.h +++ b/cpp/src/arrow/util/simdjson_internal.h @@ -350,4 +350,6 @@ ARROW_EXPORT Status ConsumeJsonValue(simdjson::ondemand::value value); ARROW_EXPORT Status ValidateJsonDocument(simdjson::ondemand::parser& parser, simdjson::padded_string& json); +ARROW_EXPORT int64_t ConsumeJsonWhitespace(std::string_view view, bool trailing); + } // namespace arrow::internal From 18d14922d47e684624a4d1d1501108685236d270 Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Mon, 21 Sep 2026 21:06:39 +0530 Subject: [PATCH 13/21] Remove Handler --- cpp/src/arrow/json/parser.cc | 264 ++++++++++++----------------------- 1 file changed, 91 insertions(+), 173 deletions(-) diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc index 5682953ab543..9bfe40d29fc6 100644 --- a/cpp/src/arrow/json/parser.cc +++ b/cpp/src/arrow/json/parser.cc @@ -136,7 +136,7 @@ class RawArrayBuilder; /// \brief packed pointer to a RawArrayBuilder /// -/// RawArrayBuilders are stored in HandlerBase, +/// RawArrayBuilders are stored in ParseImpl, /// which allows storage of their indices (uint32_t) instead of a full pointer. /// BuilderPtr is also tagged with the json kind and nullable properties /// so those can be accessed before dereferencing the builder. @@ -650,30 +650,49 @@ class RawBuilderSet { arenas_; }; -/// Three implementations are provided for BlockParser, one for each -/// UnexpectedFieldBehavior. However most of the logic is identical in each -/// case, so the majority of the implementation is in this base class -class HandlerBase : public BlockParser { +/// Parser implementation for BlockParser. +class ParseImpl : public BlockParser { public: - explicit HandlerBase(MemoryPool* pool) + explicit ParseImpl(MemoryPool* pool, UnexpectedFieldBehavior unexpected_field_behavior) : BlockParser(pool), + unexpected_field_behavior_(unexpected_field_behavior), builder_set_(pool), field_index_(-1), scalar_values_builder_(pool) {} - /// Retrieve a pointer to a builder from a BuilderPtr template enable_if_t*> Cast(BuilderPtr builder) { return builder_set_.Cast(builder); } - /// Accessor for a stored error Status Status Error() { return status_; } Status Null() { return builder_set_.AppendNull(builder_stack_.back(), field_index_, builder_); } + Status HandleUnexpectedField(std::string_view key, sj::value value) { + switch (unexpected_field_behavior_) { + case UnexpectedFieldBehavior::Error: + return ParseError("unexpected field"); + + case UnexpectedFieldBehavior::Ignore: + return Status::OK(); + + case UnexpectedFieldBehavior::InferType: { + auto struct_builder = Cast(builder_stack_.back()); + auto leading_nulls = static_cast(struct_builder->length() - 1); + + builder_ = BuilderPtr(Kind::kNull, leading_nulls, true); + field_index_ = struct_builder->AddField(key, builder_); + + return ParseValue(value); + } + } + + return Status::OK(); + } + Status Bool(bool value) { constexpr auto kind = Kind::kBoolean; if (ARROW_PREDICT_FALSE(builder_.kind != kind)) { @@ -698,7 +717,6 @@ class HandlerBase : public BlockParser { } } - /// \brief Set up builders using an expected Schema Status Initialize(const std::shared_ptr& s) { auto type = struct_({}); if (s) { @@ -707,13 +725,14 @@ class HandlerBase : public BlockParser { return builder_set_.MakeBuilder(*type, 0, &builder_); } + Status Parse(const std::shared_ptr& json) override { return DoParse(json); } + Status Finish(std::shared_ptr* parsed) override { std::shared_ptr scalar_values; RETURN_NOT_OK(scalar_values_builder_.Finish(&scalar_values)); return builder_set_.Finish(scalar_values, builder_, parsed); } - /// \brief Emit path of current field for debugging purposes std::string Path() { std::string path; for (size_t i = 0; i < builder_stack_.size(); ++i) { @@ -733,8 +752,7 @@ class HandlerBase : public BlockParser { } protected: - template - Status DoParse(Handler& handler, const std::shared_ptr& json) { + Status DoParse(const std::shared_ptr& json) { RETURN_NOT_OK(ReserveScalarStorage(json->size())); const std::string_view input(reinterpret_cast(json->data()), @@ -764,7 +782,7 @@ class HandlerBase : public BlockParser { arrow::internal::ResolveSimdjsonResult( document.get_value(), "JSON parse error: Failed to get JSON value")); - RETURN_NOT_OK(ParseValue(handler, value)); + RETURN_NOT_OK(ParseValue(value)); ++num_rows_; } @@ -817,8 +835,7 @@ class HandlerBase : public BlockParser { return Status::OK(); } - template - Status ParseValue(Handler& handler, sj::value value) { + Status ParseValue(sj::value value) { ARROW_ASSIGN_OR_RAISE(auto type, arrow::internal::ResolveSimdjsonResult( value.type(), "Failed to determine JSON type")); @@ -831,7 +848,7 @@ class HandlerBase : public BlockParser { } case sj::json_type::boolean: { - RETURN_NOT_OK(handler.template MaybePromoteFromNull()); + RETURN_NOT_OK(MaybePromoteFromNull()); ARROW_ASSIGN_OR_RAISE(auto boolean, arrow::internal::ResolveSimdjsonResult( @@ -840,7 +857,7 @@ class HandlerBase : public BlockParser { } case sj::json_type::string: { - RETURN_NOT_OK(handler.template MaybePromoteFromNull()); + RETURN_NOT_OK(MaybePromoteFromNull()); ARROW_ASSIGN_OR_RAISE(auto string, arrow::internal::ResolveSimdjsonResult( @@ -849,7 +866,7 @@ class HandlerBase : public BlockParser { } case sj::json_type::number: { - RETURN_NOT_OK(handler.template MaybePromoteFromNull()); + RETURN_NOT_OK(MaybePromoteFromNull()); auto raw_number = value.raw_json_token(); raw_number.remove_suffix( internal::ConsumeJsonWhitespace(raw_number, /*trailing=*/true)); @@ -857,21 +874,28 @@ class HandlerBase : public BlockParser { } case sj::json_type::array: - RETURN_NOT_OK(handler.template MaybePromoteFromNull()); - return ParseArray(handler, value); + RETURN_NOT_OK(MaybePromoteFromNull()); + return ParseArray(value); case sj::json_type::object: - RETURN_NOT_OK(handler.template MaybePromoteFromNull()); - return ParseObject(handler, value); + RETURN_NOT_OK(MaybePromoteFromNull()); + return ParseObject(value); default: return ParseError("Invalid value"); } } - template - Status ParseArray(Handler& handler, sj::value value) { - RETURN_NOT_OK(StartArrayImpl()); + Status ParseArray(sj::value value) { + constexpr auto kind = Kind::kArray; + if (ARROW_PREDICT_FALSE(builder_.kind != kind)) { + return IllegallyChangedTo(kind); + } + + StartNested(); + + auto list_builder = Cast(builder_); + builder_ = list_builder->value_builder(); ARROW_ASSIGN_OR_RAISE(auto array, arrow::internal::ResolveSimdjsonResult( value.get_array(), "Failed to get JSON array")); @@ -883,16 +907,26 @@ class HandlerBase : public BlockParser { arrow::internal::ResolveSimdjsonResult( element_result, "Failed to iterate JSON array")); - RETURN_NOT_OK(ParseValue(handler, element)); + RETURN_NOT_OK(ParseValue(element)); ++size; } - return EndArrayImpl(size); + EndNested(); + + DCHECK_LE(size, std::numeric_limits::max()); + return list_builder->Append(static_cast(size)); } - template - Status ParseObject(Handler& handler, sj::value value) { - RETURN_NOT_OK(StartObjectImpl()); + Status ParseObject(sj::value value) { + constexpr auto kind = Kind::kObject; + if (ARROW_PREDICT_FALSE(builder_.kind != kind)) { + return IllegallyChangedTo(kind); + } + + auto struct_builder = Cast(builder_); + absent_fields_stack_.Push(struct_builder->num_fields(), true); + StartNested(); + RETURN_NOT_OK(struct_builder->Append()); ARROW_ASSIGN_OR_RAISE( auto object, arrow::internal::ResolveSimdjsonResult(value.get_object(), @@ -908,33 +942,44 @@ class HandlerBase : public BlockParser { arrow::internal::ResolveSimdjsonResult( field.unescaped_key(), "Failed to get JSON object key")); - auto field_value = field.value(); + RETURN_NOT_OK(ParseObjectField(key, field.value())); + } + + auto parent = builder_stack_.back(); + auto expected_count = absent_fields_stack_.TopSize(); + + for (int i = 0; i < expected_count; ++i) { + if (!absent_fields_stack_[i]) { + continue; + } + + auto field_builder = Cast(parent)->field_builder(i); + if (ARROW_PREDICT_FALSE(!field_builder.nullable)) { + return ParseError("a required field was absent"); + } - RETURN_NOT_OK(ParseObjectField(handler, key, field_value)); + RETURN_NOT_OK(builder_set_.AppendNull(parent, i, field_builder)); } - return EndObjectImpl(); + absent_fields_stack_.Pop(); + EndNested(); + return Status::OK(); } - template - Status ParseObjectField(Handler& handler, std::string_view key, sj::value value) { + Status ParseObjectField(std::string_view key, sj::value value) { bool duplicate_keys = false; if (SetFieldBuilder(key, &duplicate_keys)) { - return ParseValue(handler, value); + return ParseValue(value); } if (duplicate_keys) { return status_; } - return handler.HandleUnexpectedField(key, value); + return HandleUnexpectedField(key, value); } - /// \defgroup handlerbase-append-methods append non-nested values - /// - /// @{ - template Status AppendScalar(BuilderPtr builder, std::string_view scalar) { if (ARROW_PREDICT_FALSE(builder.kind != kind)) { @@ -948,23 +993,6 @@ class HandlerBase : public BlockParser { return Status::OK(); } - /// @} - - Status StartObjectImpl() { - constexpr auto kind = Kind::kObject; - if (ARROW_PREDICT_FALSE(builder_.kind != kind)) { - return IllegallyChangedTo(kind); - } - auto struct_builder = Cast(builder_); - absent_fields_stack_.Push(struct_builder->num_fields(), true); - StartNested(); - return struct_builder->Append(); - } - - /// \brief helper for Key() functions - /// - /// sets the field builder with name key, or returns false if - /// there is no field with that name bool SetFieldBuilder(std::string_view key, bool* duplicate_keys) { auto parent = Cast(builder_stack_.back()); field_index_ = parent->GetFieldIndex(key); @@ -974,8 +1002,6 @@ class HandlerBase : public BlockParser { if (field_index_ < absent_fields_stack_.TopSize()) { *duplicate_keys = !absent_fields_stack_[field_index_]; } else { - // When field_index is beyond the range of absent_fields_stack_ we have a duplicated - // field that wasn't declared in schema or previous records. *duplicate_keys = true; } if (*duplicate_keys) { @@ -987,56 +1013,12 @@ class HandlerBase : public BlockParser { return true; } - Status EndObjectImpl() { - auto parent = builder_stack_.back(); - - auto expected_count = absent_fields_stack_.TopSize(); - for (int i = 0; i < expected_count; ++i) { - if (!absent_fields_stack_[i]) { - continue; - } - auto field_builder = Cast(parent)->field_builder(i); - if (ARROW_PREDICT_FALSE(!field_builder.nullable)) { - return ParseError("a required field was absent"); - } - RETURN_NOT_OK(builder_set_.AppendNull(parent, i, field_builder)); - } - absent_fields_stack_.Pop(); - EndNested(); - return Status::OK(); - } - - Status StartArrayImpl() { - constexpr auto kind = Kind::kArray; - if (ARROW_PREDICT_FALSE(builder_.kind != kind)) { - return IllegallyChangedTo(kind); - } - StartNested(); - // append to the list builder in EndArrayImpl - builder_ = Cast(builder_)->value_builder(); - return Status::OK(); - } - - Status EndArrayImpl(size_t size) { - EndNested(); - // append to list_builder here - auto list_builder = Cast(builder_); - DCHECK_LE(size, std::numeric_limits::max()); - return list_builder->Append(static_cast(size)); - } - - /// helper method for StartArray and StartObject - /// adds the current builder to a stack so its - /// children can be visited and parsed. void StartNested() { field_index_stack_.push_back(field_index_); field_index_ = -1; builder_stack_.push_back(builder_); } - /// helper method for EndArray and EndObject - /// replaces the current builder with its parent - /// so parsing of the parent can continue void EndNested() { field_index_ = field_index_stack_.back(); field_index_stack_.pop_back(); @@ -1049,7 +1031,6 @@ class HandlerBase : public BlockParser { " to ", Kind::Name(illegally_changed_to), " in row ", num_rows_); } - /// Reserve storage for scalars, these can occupy almost all of the JSON buffer Status ReserveScalarStorage(int64_t size) override { auto available_storage = scalar_values_builder_.value_data_capacity() - scalar_values_builder_.value_data_length(); @@ -1059,90 +1040,27 @@ class HandlerBase : public BlockParser { return scalar_values_builder_.ReserveData(size - available_storage); } + UnexpectedFieldBehavior unexpected_field_behavior_; Status status_; RawBuilderSet builder_set_; BuilderPtr builder_; - // top of this stack is the parent of builder_ std::vector builder_stack_; - // top of this stack refers to the fields of the highest *StructBuilder* - // in builder_stack_ (list builders don't have absent fields) BitsetStack absent_fields_stack_; - // index of builder_ within its parent int field_index_; - // top of this stack == field_index_ std::vector field_index_stack_; StringBuilder scalar_values_builder_; sj::parser parser_; }; -template -class Handler; - -template <> -class Handler : public HandlerBase { - public: - using HandlerBase::HandlerBase; - - Status Parse(const std::shared_ptr& json) override { - return DoParse(*this, json); - } - - Status HandleUnexpectedField(std::string_view, sj::value) { - return ParseError("unexpected field"); - } -}; - -template <> -class Handler : public HandlerBase { - public: - using HandlerBase::HandlerBase; - - Status Parse(const std::shared_ptr& json) override { - return DoParse(*this, json); - } - - Status HandleUnexpectedField(std::string_view, sj::value) { return Status::OK(); } -}; - -template <> -class Handler : public HandlerBase { - public: - using HandlerBase::HandlerBase; - - Status Parse(const std::shared_ptr& json) override { - return DoParse(*this, json); - } - - Status HandleUnexpectedField(std::string_view key, sj::value value) { - auto struct_builder = Cast(builder_stack_.back()); - auto leading_nulls = static_cast(struct_builder->length() - 1); - - builder_ = BuilderPtr(Kind::kNull, leading_nulls, true); - field_index_ = struct_builder->AddField(key, builder_); - - return ParseValue(*this, value); - } -}; - Status BlockParser::Make(MemoryPool* pool, const ParseOptions& options, std::unique_ptr* out) { DCHECK(options.unexpected_field_behavior == UnexpectedFieldBehavior::InferType || options.explicit_schema != nullptr); - switch (options.unexpected_field_behavior) { - case UnexpectedFieldBehavior::Ignore: { - *out = std::make_unique>(pool); - break; - } - case UnexpectedFieldBehavior::Error: { - *out = std::make_unique>(pool); - break; - } - case UnexpectedFieldBehavior::InferType: - *out = std::make_unique>(pool); - break; - } - return static_cast(**out).Initialize(options.explicit_schema); + auto parser = std::make_unique(pool, options.unexpected_field_behavior); + RETURN_NOT_OK(parser->Initialize(options.explicit_schema)); + *out = std::move(parser); + return Status::OK(); } Status BlockParser::Make(const ParseOptions& options, std::unique_ptr* out) { From 89e0a6f9e49d653f96bec8bd4462331344e0c1f6 Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Mon, 21 Sep 2026 22:42:20 +0530 Subject: [PATCH 14/21] Remove isWhitespaceOnly --- cpp/src/arrow/json/parser.cc | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc index 9bfe40d29fc6..7203d5cb6e90 100644 --- a/cpp/src/arrow/json/parser.cc +++ b/cpp/src/arrow/json/parser.cc @@ -52,15 +52,6 @@ static Status ParseError(T&&... t) { return Status::Invalid("JSON parse error: ", std::forward(t)...); } -static bool IsWhitespaceOnly(std::string_view value) { - for (const auto c : value) { - if (c != ' ' && c != '\t' && c != '\n' && c != '\r') { - return false; - } - } - return true; -} - const std::string& Kind::Name(Kind::type kind) { static const std::string names[] = { "null", "boolean", "number", "string", "array", "object", "number_or_string", @@ -757,8 +748,9 @@ class ParseImpl : public BlockParser { const std::string_view input(reinterpret_cast(json->data()), json->size()); - - if (IsWhitespaceOnly(input)) { + + const int64_t input_size = input.size(); + if (internal::ConsumeJsonWhitespace(input, /*trailing=*/false) == input_size) { return Status::OK(); } From d07e7efc1a56755faff8fc9fd864f9a121a89beb Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Tue, 22 Sep 2026 00:44:05 +0530 Subject: [PATCH 15/21] Fix Bug --- cpp/src/arrow/json/parser.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc index 7203d5cb6e90..9debcd84f53c 100644 --- a/cpp/src/arrow/json/parser.cc +++ b/cpp/src/arrow/json/parser.cc @@ -748,7 +748,7 @@ class ParseImpl : public BlockParser { const std::string_view input(reinterpret_cast(json->data()), json->size()); - + const int64_t input_size = input.size(); if (internal::ConsumeJsonWhitespace(input, /*trailing=*/false) == input_size) { return Status::OK(); @@ -886,8 +886,7 @@ class ParseImpl : public BlockParser { StartNested(); - auto list_builder = Cast(builder_); - builder_ = list_builder->value_builder(); + builder_ = Cast(builder_)->value_builder(); ARROW_ASSIGN_OR_RAISE(auto array, arrow::internal::ResolveSimdjsonResult( value.get_array(), "Failed to get JSON array")); @@ -905,6 +904,7 @@ class ParseImpl : public BlockParser { EndNested(); + auto list_builder = Cast(builder_); DCHECK_LE(size, std::numeric_limits::max()); return list_builder->Append(static_cast(size)); } From ff707ad10c6185f011d25d055a98bc7c2a28a4f6 Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Tue, 22 Sep 2026 00:44:16 +0530 Subject: [PATCH 16/21] Lint Fix --- cpp/src/arrow/json/chunker.cc | 3 ++- cpp/src/arrow/util/simdjson_internal.cc | 3 +-- cpp/src/arrow/util/simdjson_internal.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/src/arrow/json/chunker.cc b/cpp/src/arrow/json/chunker.cc index 1b6f2f3d3c61..02c8ec9c33b4 100644 --- a/cpp/src/arrow/json/chunker.cc +++ b/cpp/src/arrow/json/chunker.cc @@ -150,7 +150,8 @@ class ParsingBoundaryFinder : public BoundaryFinder { if (consumed_length > 0) { // If we found at least one document, also consume its trailing whitespace // to avoid stray bytes at the end of the stream. - consumed_length += internal::ConsumeJsonWhitespace(input.substr(consumed_length), /*trailing=*/false); + consumed_length += internal::ConsumeJsonWhitespace(input.substr(consumed_length), + /*trailing=*/false); } return consumed_length; } diff --git a/cpp/src/arrow/util/simdjson_internal.cc b/cpp/src/arrow/util/simdjson_internal.cc index 363431fd8d08..a6681809871e 100644 --- a/cpp/src/arrow/util/simdjson_internal.cc +++ b/cpp/src/arrow/util/simdjson_internal.cc @@ -586,8 +586,7 @@ Status ValidateJsonDocument(simdjson::ondemand::parser& parser, int64_t ConsumeJsonWhitespace(std::string_view view, bool trailing) { if (!trailing) { const auto pos = view.find_first_not_of(" \t\r\n"); - return static_cast( - pos == std::string_view::npos ? view.size() : pos); + return static_cast(pos == std::string_view::npos ? view.size() : pos); } int64_t count = 0; diff --git a/cpp/src/arrow/util/simdjson_internal.h b/cpp/src/arrow/util/simdjson_internal.h index 90f293b30788..3579645bf071 100644 --- a/cpp/src/arrow/util/simdjson_internal.h +++ b/cpp/src/arrow/util/simdjson_internal.h @@ -350,6 +350,6 @@ ARROW_EXPORT Status ConsumeJsonValue(simdjson::ondemand::value value); ARROW_EXPORT Status ValidateJsonDocument(simdjson::ondemand::parser& parser, simdjson::padded_string& json); -ARROW_EXPORT int64_t ConsumeJsonWhitespace(std::string_view view, bool trailing); +ARROW_EXPORT int64_t ConsumeJsonWhitespace(std::string_view view, bool trailing); } // namespace arrow::internal From 7b0d5bfd46bee3b20516d9017e60a53e7389d189 Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Tue, 22 Sep 2026 01:54:39 +0530 Subject: [PATCH 17/21] Add rok's test and fix code --- cpp/src/arrow/json/parser.cc | 4 +++- cpp/src/arrow/json/parser_test.cc | 10 ++++++++++ cpp/src/arrow/json/reader_test.cc | 16 ++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc index 9debcd84f53c..dca2e3228a82 100644 --- a/cpp/src/arrow/json/parser.cc +++ b/cpp/src/arrow/json/parser.cc @@ -668,7 +668,7 @@ class ParseImpl : public BlockParser { return ParseError("unexpected field"); case UnexpectedFieldBehavior::Ignore: - return Status::OK(); + return internal::ConsumeJsonValue(value); case UnexpectedFieldBehavior::InferType: { auto struct_builder = Cast(builder_stack_.back()); @@ -860,6 +860,8 @@ class ParseImpl : public BlockParser { case sj::json_type::number: { RETURN_NOT_OK(MaybePromoteFromNull()); auto raw_number = value.raw_json_token(); + RETURN_NOT_OK(arrow::internal::ResolveSimdjsonResult( + value.get_number(), "Failed to parse JSON number")); raw_number.remove_suffix( internal::ConsumeJsonWhitespace(raw_number, /*trailing=*/true)); return RawNumber(raw_number); diff --git a/cpp/src/arrow/json/parser_test.cc b/cpp/src/arrow/json/parser_test.cc index 1b107aa020fd..a173bfb5df77 100644 --- a/cpp/src/arrow/json/parser_test.cc +++ b/cpp/src/arrow/json/parser_test.cc @@ -321,5 +321,15 @@ TEST(BlockParser, AdHoc) { R"([{"c":true, "d": "1991-02-03"}, {"c":false, "d":"2019-04-01"}])"}); } +TEST(BlockParserWithSchema, ValidateIgnoredFields) { + auto options = ParseOptions::Defaults(); + options.explicit_schema = schema({field("known", int64())}); + options.unexpected_field_behavior = UnexpectedFieldBehavior::Ignore; + + std::shared_ptr parsed; + ASSERT_RAISES(Invalid, + ParseFromString(options, R"({"known": 1, "ignored": [1,]})", &parsed)); +} + } // namespace json } // namespace arrow diff --git a/cpp/src/arrow/json/reader_test.cc b/cpp/src/arrow/json/reader_test.cc index ffe33c00111a..62e55650551b 100644 --- a/cpp/src/arrow/json/reader_test.cc +++ b/cpp/src/arrow/json/reader_test.cc @@ -1030,5 +1030,21 @@ TEST_F(AsyncStreamingReaderTest, StressSharedIoAndCpuExecutor) { AssertBatchSequenceEquals(expected.batches, batches); } +TEST(ReaderTest, FailOnMalformedNumbers) { + auto read_options = ReadOptions::Defaults(); + auto parse_options = ParseOptions::Defaults(); + read_options.use_threads = false; + + const std::vector malformed = { + R"({"a": 01})", + R"({"a": 1.})", + }; + + for (const auto& json : malformed) { + auto result = ReadToTable(json, read_options, parse_options); + EXPECT_TRUE(result.status().IsInvalid()) << result.status().ToString(); + } +} + } // namespace json } // namespace arrow From fbe9f5baa3713e1136d883d0154e89999b7ce85a Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Tue, 22 Sep 2026 17:15:19 +0530 Subject: [PATCH 18/21] Address Feedback --- cpp/src/arrow/json/parser.cc | 42 ++++++++++++++++++------- cpp/src/arrow/json/parser_test.cc | 1 + cpp/src/arrow/json/reader_test.cc | 13 +++++--- cpp/src/arrow/util/simdjson_internal.cc | 17 ++++------ 4 files changed, 46 insertions(+), 27 deletions(-) diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc index dca2e3228a82..d6cdf3515286 100644 --- a/cpp/src/arrow/json/parser.cc +++ b/cpp/src/arrow/json/parser.cc @@ -651,13 +651,12 @@ class ParseImpl : public BlockParser { field_index_(-1), scalar_values_builder_(pool) {} + /// Retrieve a pointer to a builder from a BuilderPtr template enable_if_t*> Cast(BuilderPtr builder) { return builder_set_.Cast(builder); } - Status Error() { return status_; } - Status Null() { return builder_set_.AppendNull(builder_stack_.back(), field_index_, builder_); } @@ -671,10 +670,12 @@ class ParseImpl : public BlockParser { return internal::ConsumeJsonValue(value); case UnexpectedFieldBehavior::InferType: { + // If an unexpected field is encountered, add a NullBuilder with leading nulls. + // The next value parsed will promote this field to its inferred type. auto struct_builder = Cast(builder_stack_.back()); auto leading_nulls = static_cast(struct_builder->length() - 1); - builder_ = BuilderPtr(Kind::kNull, leading_nulls, true); + builder_ = BuilderPtr(Kind::kNull, leading_nulls, /*nullable=*/true); field_index_ = struct_builder->AddField(key, builder_); return ParseValue(value); @@ -708,6 +709,7 @@ class ParseImpl : public BlockParser { } } + /// \brief Set up builders using an expected Schema Status Initialize(const std::shared_ptr& s) { auto type = struct_({}); if (s) { @@ -716,14 +718,13 @@ class ParseImpl : public BlockParser { return builder_set_.MakeBuilder(*type, 0, &builder_); } - Status Parse(const std::shared_ptr& json) override { return DoParse(json); } - Status Finish(std::shared_ptr* parsed) override { std::shared_ptr scalar_values; RETURN_NOT_OK(scalar_values_builder_.Finish(&scalar_values)); return builder_set_.Finish(scalar_values, builder_, parsed); } + /// \brief Emit path of current field for debugging purposes std::string Path() { std::string path; for (size_t i = 0; i < builder_stack_.size(); ++i) { @@ -743,7 +744,7 @@ class ParseImpl : public BlockParser { } protected: - Status DoParse(const std::shared_ptr& json) { + Status Parse(const std::shared_ptr& json) override { RETURN_NOT_OK(ReserveScalarStorage(json->size())); const std::string_view input(reinterpret_cast(json->data()), @@ -793,6 +794,7 @@ class ParseImpl : public BlockParser { return parse(padded_json); } + // padded_string makes a copy of the input buffer. simdjson::padded_string padded_json(reinterpret_cast(json->data()), json->size()); return parse(padded_json); @@ -875,9 +877,10 @@ class ParseImpl : public BlockParser { RETURN_NOT_OK(MaybePromoteFromNull()); return ParseObject(value); - default: + case sj::json_type::unknown: return ParseError("Invalid value"); } + return Status::OK(); } Status ParseArray(sj::value value) { @@ -893,7 +896,7 @@ class ParseImpl : public BlockParser { ARROW_ASSIGN_OR_RAISE(auto array, arrow::internal::ResolveSimdjsonResult( value.get_array(), "Failed to get JSON array")); - size_t size = 0; + int64_t size = 0; for (auto element_result : array) { ARROW_ASSIGN_OR_RAISE(auto element, @@ -940,21 +943,18 @@ class ParseImpl : public BlockParser { } auto parent = builder_stack_.back(); - auto expected_count = absent_fields_stack_.TopSize(); + auto expected_count = absent_fields_stack_.TopSize(); for (int i = 0; i < expected_count; ++i) { if (!absent_fields_stack_[i]) { continue; } - auto field_builder = Cast(parent)->field_builder(i); if (ARROW_PREDICT_FALSE(!field_builder.nullable)) { return ParseError("a required field was absent"); } - RETURN_NOT_OK(builder_set_.AppendNull(parent, i, field_builder)); } - absent_fields_stack_.Pop(); EndNested(); return Status::OK(); @@ -987,6 +987,10 @@ class ParseImpl : public BlockParser { return Status::OK(); } + /// \brief helper for parsing object fields. + /// + /// Sets the field builder with the given name, or returns false if + /// there is no such field or the field was already specified. bool SetFieldBuilder(std::string_view key, bool* duplicate_keys) { auto parent = Cast(builder_stack_.back()); field_index_ = parent->GetFieldIndex(key); @@ -996,6 +1000,8 @@ class ParseImpl : public BlockParser { if (field_index_ < absent_fields_stack_.TopSize()) { *duplicate_keys = !absent_fields_stack_[field_index_]; } else { + // When field_index is beyond the range of absent_fields_stack_ we have a duplicated + // field that wasn't declared in schema or previous records. *duplicate_keys = true; } if (*duplicate_keys) { @@ -1007,12 +1013,18 @@ class ParseImpl : public BlockParser { return true; } + /// helper method for ParseArray and ParseObject + /// adds the current builder to a stack so its + /// children can be visited and parsed. void StartNested() { field_index_stack_.push_back(field_index_); field_index_ = -1; builder_stack_.push_back(builder_); } + /// helper method for EndArray and EndObject + /// replaces the current builder with its parent + /// so parsing of the parent can continue void EndNested() { field_index_ = field_index_stack_.back(); field_index_stack_.pop_back(); @@ -1025,6 +1037,7 @@ class ParseImpl : public BlockParser { " to ", Kind::Name(illegally_changed_to), " in row ", num_rows_); } + /// Reserve storage for scalars, these can occupy almost all of the JSON buffer Status ReserveScalarStorage(int64_t size) override { auto available_storage = scalar_values_builder_.value_data_capacity() - scalar_values_builder_.value_data_length(); @@ -1038,9 +1051,14 @@ class ParseImpl : public BlockParser { Status status_; RawBuilderSet builder_set_; BuilderPtr builder_; + // top of this stack is the parent of builder_ std::vector builder_stack_; + // top of this stack refers to the fields of the highest *StructBuilder* + // in builder_stack_ (list builders don't have absent fields) BitsetStack absent_fields_stack_; + // index of builder_ within its parent int field_index_; + // top of this stack == field_index_ std::vector field_index_stack_; StringBuilder scalar_values_builder_; sj::parser parser_; diff --git a/cpp/src/arrow/json/parser_test.cc b/cpp/src/arrow/json/parser_test.cc index a173bfb5df77..366a9b3f2a6a 100644 --- a/cpp/src/arrow/json/parser_test.cc +++ b/cpp/src/arrow/json/parser_test.cc @@ -327,6 +327,7 @@ TEST(BlockParserWithSchema, ValidateIgnoredFields) { options.unexpected_field_behavior = UnexpectedFieldBehavior::Ignore; std::shared_ptr parsed; + // Ignored fields should still be validated for malformed JSON. ASSERT_RAISES(Invalid, ParseFromString(options, R"({"known": 1, "ignored": [1,]})", &parsed)); } diff --git a/cpp/src/arrow/json/reader_test.cc b/cpp/src/arrow/json/reader_test.cc index 62e55650551b..b8e3244d7066 100644 --- a/cpp/src/arrow/json/reader_test.cc +++ b/cpp/src/arrow/json/reader_test.cc @@ -1033,16 +1033,21 @@ TEST_F(AsyncStreamingReaderTest, StressSharedIoAndCpuExecutor) { TEST(ReaderTest, FailOnMalformedNumbers) { auto read_options = ReadOptions::Defaults(); auto parse_options = ParseOptions::Defaults(); - read_options.use_threads = false; const std::vector malformed = { R"({"a": 01})", R"({"a": 1.})", }; - for (const auto& json : malformed) { - auto result = ReadToTable(json, read_options, parse_options); - EXPECT_TRUE(result.status().IsInvalid()) << result.status().ToString(); + // Malformed numbers should be rejected regardless of whether parsing is threaded. + for (const bool use_threads : {false, true}) { + read_options.use_threads = use_threads; + + for (const auto& json : malformed) { + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::StartsWith("Invalid: Failed to parse JSON number"), + ReadToTable(json, read_options, parse_options)); + } } } diff --git a/cpp/src/arrow/util/simdjson_internal.cc b/cpp/src/arrow/util/simdjson_internal.cc index a6681809871e..c15f9fcb4a2f 100644 --- a/cpp/src/arrow/util/simdjson_internal.cc +++ b/cpp/src/arrow/util/simdjson_internal.cc @@ -581,23 +581,18 @@ Status ValidateJsonDocument(simdjson::ondemand::parser& parser, return ConsumeJsonValue(value); } -// XXX We could try to SIMD-accelerate this routine but it's called only -// once per chunk and also will presumably examine a minimal amount of bytes. +/// Returns the number of leading whitespace characters when trailing is false, +/// or the number of trailing whitespace characters when trailing is true. +// XXX We could try to SIMD-accelerate this routine. int64_t ConsumeJsonWhitespace(std::string_view view, bool trailing) { if (!trailing) { const auto pos = view.find_first_not_of(" \t\r\n"); return static_cast(pos == std::string_view::npos ? view.size() : pos); } - int64_t count = 0; - while (count < static_cast(view.size())) { - const auto c = view[view.size() - count - 1]; - if (c != ' ' && c != '\t' && c != '\r' && c != '\n') { - break; - } - ++count; - } - return count; + const auto pos = view.find_last_not_of(" \t\r\n"); + return static_cast(pos == std::string_view::npos ? view.size() + : view.size() - pos - 1); } } // namespace arrow::internal From b27b054cb7a02384bae366a574020aa5e6cd1843 Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Wed, 23 Sep 2026 17:51:45 +0530 Subject: [PATCH 19/21] Address Feedback 2 --- cpp/src/arrow/json/parser.cc | 28 ++++++++++------------ cpp/src/arrow/json/parser_test.cc | 31 +++++++++++++++---------- cpp/src/arrow/json/reader_test.cc | 6 ++--- cpp/src/arrow/util/simdjson_internal.cc | 2 +- cpp/src/arrow/util/simdjson_internal.h | 2 +- python/pyarrow/tests/test_json.py | 4 ++-- 6 files changed, 38 insertions(+), 35 deletions(-) diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc index d6cdf3515286..0e606ef2f78b 100644 --- a/cpp/src/arrow/json/parser.cc +++ b/cpp/src/arrow/json/parser.cc @@ -781,7 +781,7 @@ class ParseImpl : public BlockParser { } if (stream.truncated_bytes() != 0) { - return ParseError("The document is empty"); + return ParseError("JSON document was truncated"); } return Status::OK(); @@ -794,7 +794,8 @@ class ParseImpl : public BlockParser { return parse(padded_json); } - // padded_string makes a copy of the input buffer. + // TODO(GH-51463): Investigate allocating input buffers with enough capacity + // for SIMDJSON padding to avoid copying in this case. simdjson::padded_string padded_json(reinterpret_cast(json->data()), json->size()); return parse(padded_json); @@ -878,7 +879,7 @@ class ParseImpl : public BlockParser { return ParseObject(value); case sj::json_type::unknown: - return ParseError("Invalid value"); + return ParseError("Invalid JSON value"); } return Status::OK(); } @@ -961,16 +962,12 @@ class ParseImpl : public BlockParser { } Status ParseObjectField(std::string_view key, sj::value value) { - bool duplicate_keys = false; + ARROW_ASSIGN_OR_RAISE(auto found, SetFieldBuilder(key)); - if (SetFieldBuilder(key, &duplicate_keys)) { + if (found) { return ParseValue(value); } - if (duplicate_keys) { - return status_; - } - return HandleUnexpectedField(key, value); } @@ -991,22 +988,22 @@ class ParseImpl : public BlockParser { /// /// Sets the field builder with the given name, or returns false if /// there is no such field or the field was already specified. - bool SetFieldBuilder(std::string_view key, bool* duplicate_keys) { + Result SetFieldBuilder(std::string_view key) { auto parent = Cast(builder_stack_.back()); field_index_ = parent->GetFieldIndex(key); if (ARROW_PREDICT_FALSE(field_index_ == -1)) { return false; } + bool duplicate_keys; if (field_index_ < absent_fields_stack_.TopSize()) { - *duplicate_keys = !absent_fields_stack_[field_index_]; + duplicate_keys = !absent_fields_stack_[field_index_]; } else { // When field_index is beyond the range of absent_fields_stack_ we have a duplicated // field that wasn't declared in schema or previous records. - *duplicate_keys = true; + duplicate_keys = true; } - if (*duplicate_keys) { - status_ = ParseError("Column(", Path(), ") was specified twice in row ", num_rows_); - return false; + if (duplicate_keys) { + return ParseError("Column(", Path(), ") was specified twice in row ", num_rows_); } builder_ = parent->field_builder(field_index_); absent_fields_stack_[field_index_] = false; @@ -1048,7 +1045,6 @@ class ParseImpl : public BlockParser { } UnexpectedFieldBehavior unexpected_field_behavior_; - Status status_; RawBuilderSet builder_set_; BuilderPtr builder_; // top of this stack is the parent of builder_ diff --git a/cpp/src/arrow/json/parser_test.cc b/cpp/src/arrow/json/parser_test.cc index 366a9b3f2a6a..0b560043d442 100644 --- a/cpp/src/arrow/json/parser_test.cc +++ b/cpp/src/arrow/json/parser_test.cc @@ -235,6 +235,24 @@ TEST(BlockParserWithSchema, FailOnIncompleteJson) { ASSERT_RAISES(Invalid, ParseFromString(options, "{\"a\":0, \"b\"", &parsed)); } +TEST(BlockParserWithSchema, ValidateIgnoredFields) { + auto options = ParseOptions::Defaults(); + options.explicit_schema = schema({field("known", int64())}); + options.unexpected_field_behavior = UnexpectedFieldBehavior::Ignore; + + std::shared_ptr parsed; + // Ignored fields should still be validated for malformed JSON. + Status error = ParseFromString(options, R"({"known": 1, "ignored": [1,]})", &parsed); + ASSERT_RAISES(Invalid, error); + EXPECT_THAT(error.message(), testing::StartsWith("Invalid JSON value")); +} + +TEST(BlockParserWithSchema, NumberWithWhitespace) { + auto options = ParseOptions::Defaults(); + options.explicit_schema = schema({field("a", int64())}); + AssertParseColumns(options, R"({"a": 123 })", {field("a", utf8())}, {R"(["123"])"}); +} + TEST(BlockParser, Basics) { auto options = ParseOptions::Defaults(); options.unexpected_field_behavior = UnexpectedFieldBehavior::InferType; @@ -305,7 +323,7 @@ TEST(BlockParser, FailOnInvalidEOF) { auto status = ParseFromString(ParseOptions::Defaults(), "}", &parsed); ASSERT_RAISES(Invalid, status); EXPECT_THAT(status.message(), - ::testing::StartsWith("JSON parse error: The document is empty")); + ::testing::StartsWith("JSON parse error: JSON document was truncated")); } TEST(BlockParser, AdHoc) { @@ -321,16 +339,5 @@ TEST(BlockParser, AdHoc) { R"([{"c":true, "d": "1991-02-03"}, {"c":false, "d":"2019-04-01"}])"}); } -TEST(BlockParserWithSchema, ValidateIgnoredFields) { - auto options = ParseOptions::Defaults(); - options.explicit_schema = schema({field("known", int64())}); - options.unexpected_field_behavior = UnexpectedFieldBehavior::Ignore; - - std::shared_ptr parsed; - // Ignored fields should still be validated for malformed JSON. - ASSERT_RAISES(Invalid, - ParseFromString(options, R"({"known": 1, "ignored": [1,]})", &parsed)); -} - } // namespace json } // namespace arrow diff --git a/cpp/src/arrow/json/reader_test.cc b/cpp/src/arrow/json/reader_test.cc index b8e3244d7066..dba835606291 100644 --- a/cpp/src/arrow/json/reader_test.cc +++ b/cpp/src/arrow/json/reader_test.cc @@ -696,10 +696,10 @@ TEST_P(StreamingReaderTest, PropagateParsingErrors) { read_options_.block_size = 16; EXPECT_RAISES_WITH_MESSAGE_THAT( - Invalid, ::testing::StartsWith("Invalid: JSON parse error: Invalid value"), + Invalid, ::testing::StartsWith("Invalid: JSON parse error: Invalid JSON value"), MakeReader(bad_first_block)); EXPECT_RAISES_WITH_MESSAGE_THAT( - Invalid, ::testing::StartsWith("Invalid: JSON parse error: Invalid value"), + Invalid, ::testing::StartsWith("Invalid: JSON parse error: Invalid JSON value"), MakeReader(bad_first_block_after_empty)); std::shared_ptr batch; @@ -1039,7 +1039,7 @@ TEST(ReaderTest, FailOnMalformedNumbers) { R"({"a": 1.})", }; - // Malformed numbers should be rejected regardless of whether parsing is threaded. + // Malformed numbers should be rejected for (const bool use_threads : {false, true}) { read_options.use_threads = use_threads; diff --git a/cpp/src/arrow/util/simdjson_internal.cc b/cpp/src/arrow/util/simdjson_internal.cc index c15f9fcb4a2f..d845eab0d473 100644 --- a/cpp/src/arrow/util/simdjson_internal.cc +++ b/cpp/src/arrow/util/simdjson_internal.cc @@ -581,7 +581,7 @@ Status ValidateJsonDocument(simdjson::ondemand::parser& parser, return ConsumeJsonValue(value); } -/// Returns the number of leading whitespace characters when trailing is false, +/// Returns the position of the first non-whitespace character when trailing is false, /// or the number of trailing whitespace characters when trailing is true. // XXX We could try to SIMD-accelerate this routine. int64_t ConsumeJsonWhitespace(std::string_view view, bool trailing) { diff --git a/cpp/src/arrow/util/simdjson_internal.h b/cpp/src/arrow/util/simdjson_internal.h index 3579645bf071..d5816522bee0 100644 --- a/cpp/src/arrow/util/simdjson_internal.h +++ b/cpp/src/arrow/util/simdjson_internal.h @@ -266,7 +266,7 @@ Status VisitJsonValue(simdjson::ondemand::value value, ObjectFn&& object_fn, } case simdjson::ondemand::json_type::unknown: - return Status::Invalid("Unknown JSON type"); + return Status::Invalid("Invalid JSON value"); } return Status::Invalid("Unreachable"); diff --git a/python/pyarrow/tests/test_json.py b/python/pyarrow/tests/test_json.py index 8a8237cbc24c..e7ab574ead48 100644 --- a/python/pyarrow/tests/test_json.py +++ b/python/pyarrow/tests/test_json.py @@ -452,7 +452,7 @@ def test_bad_first_parse(self): read_options = ReadOptions() read_options.block_size = 16 with pytest.raises(pa.ArrowInvalid, - match="JSON parse error: Invalid value.*"): + match="JSON parse error: Invalid JSON value.*"): self.open_bytes(bad_first_block, read_options=read_options) def test_bad_middle_parse_after_empty(self): @@ -460,7 +460,7 @@ def test_bad_middle_parse_after_empty(self): read_options = ReadOptions() read_options.block_size = 16 with pytest.raises(pa.ArrowInvalid, - match="JSON parse error: Invalid value.*"): + match="JSON parse error: Invalid JSON value.*"): self.open_bytes(bad_first_block, read_options=read_options) def test_bad_middle_parse(self): From 6b0b6a4ad9ed05a111d1532d54a66257f2e9eb08 Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Mon, 28 Sep 2026 21:41:01 +0530 Subject: [PATCH 20/21] rebase conflict --- cpp/src/arrow/json/chunker.cc | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/cpp/src/arrow/json/chunker.cc b/cpp/src/arrow/json/chunker.cc index 3e8c3ee50dd3..d073f9405408 100644 --- a/cpp/src/arrow/json/chunker.cc +++ b/cpp/src/arrow/json/chunker.cc @@ -32,16 +32,6 @@ namespace arrow { namespace json { namespace { -// XXX We could try to SIMD-accelerate this routine but it's called only -// once per chunk and also will presumably examine a minimal amount of bytes. -int64_t ConsumeWhitespace(std::string_view view) { - const auto ws_count = view.find_first_not_of(" \t\r\n"); - if (ws_count == std::string_view::npos) { - return view.size(); - } - return static_cast(ws_count); -} - // A BoundaryFinder implementation that assumes JSON objects can contain raw newlines, // and uses the structural indexes computed by simdjson to delimit them. class ParsingBoundaryFinder : public BoundaryFinder { From 1c977a4288f1c3a5a62f6e02840ea68798e4a804 Mon Sep 17 00:00:00 2001 From: Aaditya Srinivasan Date: Tue, 29 Sep 2026 16:49:01 +0530 Subject: [PATCH 21/21] Remove SetFieldBuilder --- cpp/src/arrow/json/parser.cc | 44 ++++++++++++------------------------ 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc index 0e606ef2f78b..2ef339a19f74 100644 --- a/cpp/src/arrow/json/parser.cc +++ b/cpp/src/arrow/json/parser.cc @@ -962,37 +962,10 @@ class ParseImpl : public BlockParser { } Status ParseObjectField(std::string_view key, sj::value value) { - ARROW_ASSIGN_OR_RAISE(auto found, SetFieldBuilder(key)); - - if (found) { - return ParseValue(value); - } - - return HandleUnexpectedField(key, value); - } - - template - Status AppendScalar(BuilderPtr builder, std::string_view scalar) { - if (ARROW_PREDICT_FALSE(builder.kind != kind)) { - return IllegallyChangedTo(kind); - } - auto index = static_cast(scalar_values_builder_.length()); - auto value_length = static_cast(scalar.size()); - RETURN_NOT_OK(Cast(builder)->Append(index, value_length)); - RETURN_NOT_OK(scalar_values_builder_.Reserve(1)); - scalar_values_builder_.UnsafeAppend(scalar); - return Status::OK(); - } - - /// \brief helper for parsing object fields. - /// - /// Sets the field builder with the given name, or returns false if - /// there is no such field or the field was already specified. - Result SetFieldBuilder(std::string_view key) { auto parent = Cast(builder_stack_.back()); field_index_ = parent->GetFieldIndex(key); if (ARROW_PREDICT_FALSE(field_index_ == -1)) { - return false; + return HandleUnexpectedField(key, value); } bool duplicate_keys; if (field_index_ < absent_fields_stack_.TopSize()) { @@ -1007,7 +980,20 @@ class ParseImpl : public BlockParser { } builder_ = parent->field_builder(field_index_); absent_fields_stack_[field_index_] = false; - return true; + return ParseValue(value); + } + + template + Status AppendScalar(BuilderPtr builder, std::string_view scalar) { + if (ARROW_PREDICT_FALSE(builder.kind != kind)) { + return IllegallyChangedTo(kind); + } + auto index = static_cast(scalar_values_builder_.length()); + auto value_length = static_cast(scalar.size()); + RETURN_NOT_OK(Cast(builder)->Append(index, value_length)); + RETURN_NOT_OK(scalar_values_builder_.Reserve(1)); + scalar_values_builder_.UnsafeAppend(scalar); + return Status::OK(); } /// helper method for ParseArray and ParseObject