Add strict duplicate-key parsing and JuliaC coverage

Project.toml

@@ -1,6 +1,6 @@
 name = "JSON"
 uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
-version = "1.6.1"
+version = "1.7.0"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
@@ -28,8 +28,9 @@ julia = "1.9"
 [extras]
 Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45"
 ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Arrow", "Tar", "Test"]
+test = ["Arrow", "Pkg", "Tar", "Test"]

src/JSON.jl

@@ -15,14 +15,29 @@ export JSONText, StructUtils, @noarg, @kwarg, @defaults, @tags, @choosetype, @no
     eval(Expr(:public,
         :parse, :parse!, :parsefile, :parsefile!,
         :lazy, :lazyfile, :LazyValue,
-        :isvalidjson,
+        :isvalidjson, :DuplicateKeyError,
         :json, :print,
         :lower, :lift,
         :omit_null, :omit_empty,
         :Object, :Null, :Omit, :JSONStyle,
     ))
 end
 
+"""
+    JSON.DuplicateKeyError
+
+Error thrown when `duplicate_keys=:error` encounters a repeated object key.
+`key` is the decoded JSON key and `position` is its one-based byte position.
+"""
+struct DuplicateKeyError <: Exception
+    key::String
+    position::Int
+end
+
+function Base.showerror(io::IO, err::DuplicateKeyError)
+    Base.print(io, "duplicate JSON object key ", repr(err.key), " at byte position ", err.position)
+end
+
 @enum Error InvalidJSON UnexpectedEOF ExpectedOpeningObjectChar ExpectedOpeningQuoteChar ExpectedOpeningArrayChar ExpectedClosingArrayChar ExpectedComma ExpectedColon ExpectedNewline InvalidChar InvalidNumber InvalidUTF16
 
 @noinline function invalid(error, buf, pos::Int, T)

src/lazy.jl

@@ -59,6 +59,7 @@ Currently supported keyword arguments include:
   - `inf::String = "Infinity"`: the string that will be used to parse `Inf` if `allownan=true`
   - `nan::String = "NaN"`: the string that will be sued to parse `NaN` if `allownan=true`
   - `jsonlines::Bool = false`: whether the JSON input should be treated as an implicit array, with newlines separating individual JSON elements with no leading `'['` or trailing `']'` characters. Common in logging or streaming workflows. Defaults to `true` when used with `JSON.parsefile` and the filename extension is `.jsonl` or `ndjson`. Note this ensures that parsing will _always_ return an array at the root-level.
+  - `duplicate_keys::Symbol = :overwrite`: how repeated object keys are handled. `:overwrite` preserves the default last-value-wins behavior. `:error` throws [`JSON.DuplicateKeyError`](@ref).
   - `isroot::Bool = true`: whether this is the root LazyValue encompassing the entire json buffer. If `false` parses only the first JSON value and ignores trailing characters.
 
 Note that validation is only fully done on `null`, `true`, and `false`,
@@ -81,6 +82,7 @@ function lazy end
     inf::String = "Infinity"
     nan::String = "NaN"
     jsonlines::Bool = false
+    duplicate_keys::Symbol = :overwrite
 end
 
 lazy(io::Union{IO, Base.AbstractCmd}; kw...) = lazy(Base.read(io); kw...)
@@ -91,6 +93,8 @@ lazyfile(file; jsonlines::Union{Bool, Nothing}=nothing, kw...) = open(io -> lazy
 lazyfile
 
 function lazy(buf::Union{AbstractVector{UInt8}, AbstractString}; isroot::Bool=true, kw...)
+    opts = LazyOptions(; kw...)
+    opts.duplicate_keys in (:overwrite, :error) || throw(ArgumentError("`duplicate_keys` must be `:overwrite` or `:error`, got `$(repr(opts.duplicate_keys))`"))
     if !applicable(pointer, buf, 1) || (buf isa AbstractVector{UInt8} && !isone(only(strides(buf))))
         if buf isa AbstractString
             buf = String(buf)
@@ -118,7 +122,7 @@ function lazy(buf::Union{AbstractVector{UInt8}, AbstractString}; isroot::Bool=tr
     # detect and ignore UTF-8 BOM
     pos = (len >= 3 && getbyte(buf, pos) == 0xef && getbyte(buf, pos + 1) == 0xbb && getbyte(buf, pos + 2) == 0xbf) ? pos + 3 : pos
     @nextbyte
-    return _lazy(buf, pos, len, b, LazyOptions(; kw...), isroot)
+    return _lazy(buf, pos, len, b, opts, isroot)
 
 @label invalid
     invalid(error, buf, pos, Any)
@@ -262,6 +266,7 @@ function applyobject(keyvalfunc, x::LazyValues)
     buf = getbuf(x)
     len = getlength(buf)
     opts = getopts(x)
+    seen = opts.duplicate_keys === :error ? Set{String}() : nothing
     b = getbyte(buf, pos)
     if b != UInt8('{')
         error = ExpectedOpeningObjectChar
@@ -272,8 +277,14 @@ function applyobject(keyvalfunc, x::LazyValues)
     b == UInt8('}') && return pos + 1
     while true
         # parsestring returns key as a PtrString
+        keypos = pos
         GC.@preserve buf begin
             key, pos = @inline parsestring(LazyValue(buf, pos, JSONTypes.STRING, opts, false))
+            if seen !== nothing
+                decoded = convert(String, key)
+                decoded in seen && throw(DuplicateKeyError(decoded, keypos))
+                push!(seen, decoded)
+            end
             @nextbyte
             if b != UInt8(':')
                 error = ExpectedColon
@@ -378,7 +389,7 @@ function applyarray(keyvalfunc, x::LazyValues)
         # for jsonlines, we need to make sure that recursive
         # lazy values *don't* consider individual lines *also*
         # to be jsonlines
-        opts = LazyOptions(; allownan=opts.allownan, ninf=opts.ninf, inf=opts.inf, nan=opts.nan, jsonlines=false)
+        opts = LazyOptions(; allownan=opts.allownan, ninf=opts.ninf, inf=opts.inf, nan=opts.nan, jsonlines=false, duplicate_keys=opts.duplicate_keys)
     end
     i = 1
     while true

test/json_trim_public_entrypoints.jl

@@ -0,0 +1,72 @@
+using JSON
+
+const ARRAY_JSON = "[1,2,3]"
+const STRING_JSON = "\"Ada\""
+const TRIM_FIXTURE_DIR = joinpath(@__DIR__, "trim")
+
+JSON.@nonstruct struct TrimCode
+    value::String
+end
+
+JSON.lower(x::TrimCode) = x.value
+
+function checked(cond::Bool, msg::String)::Nothing
+    cond || error(msg)
+    return nothing
+end
+
+function exercise_lazy_entrypoints()::Nothing
+    checked(JSON.lazy(ARRAY_JSON) isa JSON.LazyValue, "lazy failed")
+    checked(JSON.lazyfile(joinpath(TRIM_FIXTURE_DIR, "value.json")) isa JSON.LazyValue, "lazyfile failed")
+    return nothing
+end
+
+function exercise_parse_entrypoints()::Nothing
+    # Materializing JSON.parse currently pulls in verifier failures in parser
+    # and StructUtils error paths, so keep this read workload to trim-safe
+    # lazy entrypoints until those verifier issues can be chased down.
+    checked(JSON.lazy("7") isa JSON.LazyValue, "numeric lazy detection failed")
+    checked(JSON.lazy(IOBuffer(STRING_JSON)) isa JSON.LazyValue, "IO lazy detection failed")
+    return nothing
+end
+
+function exercise_write_entrypoints()::Nothing
+    obj = JSON.Object{String, Int}("score" => 7)
+    obj[:score] = 10
+    checked(obj.score == 10, "Object property access failed")
+    checked(haskey(obj, "score"), "Object setindex! failed")
+    delete!(obj, :score)
+    checked(!haskey(obj, "score"), "Object delete! failed")
+
+    checked(JSON.json([1, 2, 3]) == ARRAY_JSON, "json string output failed")
+
+    io = IOBuffer()
+    JSON.json(io, [1, 2, 3]; pretty = 2)
+    checked(String(take!(io)) == "[\n  1,\n  2,\n  3\n]", "pretty IO json output failed")
+
+    print_io = IOBuffer()
+    JSON.print(print_io, [1, 2, 3], 2)
+    checked(String(take!(print_io)) == "[\n  1,\n  2,\n  3\n]", "JSON.print failed")
+
+    jsonlines = JSON.json([[1], [2]]; jsonlines = true)
+    checked(jsonlines == "[1]\n[2]\n", "jsonlines write failed")
+    checked(JSON.json(JSON.JSONText("{\"raw\":true}")) == "{\"raw\":true}", "JSONText write failed")
+    checked(JSON.json(JSON.Null()) == "null", "JSON.Null write failed")
+    checked(JSON.json(TrimCode("beta")) == "\"beta\"", "custom lower write failed")
+    return nothing
+end
+
+function run_json_trim_public_entrypoints()::Nothing
+    exercise_lazy_entrypoints()
+    exercise_parse_entrypoints()
+    exercise_write_entrypoints()
+    return nothing
+end
+
+function @main(args::Vector{String})::Cint
+    _ = args
+    run_json_trim_public_entrypoints()
+    return 0
+end
+
+Base.Experimental.entrypoint(main, (Vector{String},))

test/parse.jl

@@ -835,3 +835,27 @@ end
     # isroot=false with typed parse
     @test JSON.parse("{\"a\": 1, \"b\": 2.0, \"c\": \"hi\"} trailing", D; isroot=false) == D(1, 2.0, "hi")
 end
+
+@testset "duplicate object keys" begin
+    input = "{\"a\":1,\"a\":2}"
+    @test JSON.parse(input) == JSON.Object("a" => 2)
+
+    err = try
+        JSON.parse(input; duplicate_keys=:error)
+        nothing
+    catch e
+        e
+    end
+    @test err isa JSON.DuplicateKeyError
+    @test err.key == "a"
+    @test err.position == 8
+    @test occursin("duplicate JSON object key", sprint(showerror, err))
+
+    @test_throws JSON.DuplicateKeyError JSON.parse("{\"outer\":{\"x\":1,\"x\":2}}"; duplicate_keys=:error)
+    @test_throws JSON.DuplicateKeyError JSON.parse("{\"a\":1,\"\\u0061\":2}"; duplicate_keys=:error)
+    @test_throws JSON.DuplicateKeyError JSON.parse("{\"a\":1}\n{\"b\":1,\"b\":2}"; jsonlines=true, duplicate_keys=:error)
+    @test_throws JSON.DuplicateKeyError JSON.parse(input, Dict{String, Int}; duplicate_keys=:error)
+    @test JSON.isvalidjson(input)
+    @test !JSON.isvalidjson(input; duplicate_keys=:error)
+    @test_throws ArgumentError JSON.parse("{}"; duplicate_keys=:keep_first)
+end

test/runtests.jl

@@ -8,6 +8,7 @@ include(joinpath(dirname(pathof(JSON)), "../test/json.jl"))
 if Sys.WORD_SIZE == 64
     include(joinpath(dirname(pathof(JSON)), "../test/arrow.jl"))
 end
+include(joinpath(dirname(pathof(JSON)), "../test/trim_compile_tests.jl"))
 
 function tar_files(tarball::String)
     data = Dict{String, Vector{UInt8}}()

test/trim/Project.toml

@@ -0,0 +1,2 @@
+[deps]
+JuliaC = "acedd4c2-ced6-4a15-accc-2607eb759ba2"

test/trim/value.json

@@ -0,0 +1 @@
+"Ada"