diff --git a/CHANGELOG.md b/CHANGELOG.md index 12e5f82..9aaea01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Changed + +- **Breaking**: `CDF_CHAR`/`CDF_UCHAR` variables have eltype `String` (trailing null padding stripped); `StaticString` is removed. Avoids a recompile for each new string width ([#79]). + ## [0.3.0] - 2026-10-02 ### Changed @@ -29,5 +33,6 @@ [0.3.0]: https://github.com/JuliaSpacePhysics/CommonDataFormat.jl/compare/v0.2.0...v0.3.0 [0.2.0]: https://github.com/JuliaSpacePhysics/CommonDataFormat.jl/releases/tag/v0.2.0 +[#79]: https://github.com/JuliaSpacePhysics/CommonDataFormat.jl/pull/79 [#74]: https://github.com/JuliaSpacePhysics/CommonDataFormat.jl/pull/74 [#46]: https://github.com/JuliaSpacePhysics/CommonDataFormat.jl/pull/46 diff --git a/src/CommonDataFormat.jl b/src/CommonDataFormat.jl index e982346..f0487f3 100644 --- a/src/CommonDataFormat.jl +++ b/src/CommonDataFormat.jl @@ -38,7 +38,6 @@ end include("epochs.jl") include("enums.jl") include("types.jl") -include("staticstring.jl") include("parsing.jl") include("decompress.jl") include("records/records.jl") diff --git a/src/dataset.jl b/src/dataset.jl index 240dbfc..9c0554b 100644 --- a/src/dataset.jl +++ b/src/dataset.jl @@ -102,13 +102,9 @@ end function _variable(cdf, name, vdr, ::Val{M}) where {M} dims = (map(Int, record_sizes(vdr, cdf, Val(M)))..., Int(vdr.max_rec) + 1) code = Int(vdr.data_type) - if code == CDF_CHAR || code == CDF_UCHAR # eltype depends on runtime num_elems - T = StaticString{Int(vdr.num_elems),UInt8} - return CDFVariable{T,M + 1,typeof(vdr),typeof(cdf)}(name, vdr, cdf, dims) - end # Branch to static constructor per element type return Base.Cartesian.@nif( - 16, + 18, d -> code == CODE_TYPE_PAIRS[d][1], d -> _construct(cdf, name, vdr, dims, CODE_TYPE_PAIRS[d][2]), d -> throw(ArgumentError("unsupported CDF data type $code")) diff --git a/src/enums.jl b/src/enums.jl index 0608a62..d2ece4b 100644 --- a/src/enums.jl +++ b/src/enums.jl @@ -51,6 +51,7 @@ const CODE_TYPE_PAIRS = ( (33, TT2000), (31, Epoch), (32, Epoch16), (1, Int8), (2, Int16), (4, Int32), (8, Int64), (11, UInt8), (12, UInt16), (14, UInt32), (41, Int8), + (51, String), (52, String), ) const type_map = Dict{CDFDataType, Type}(CDFDataType(c) => T for (c, T) in CODE_TYPE_PAIRS) diff --git a/src/loading/variable.jl b/src/loading/variable.jl index 1f03c5c..fe58c68 100644 --- a/src/loading/variable.jl +++ b/src/loading/variable.jl @@ -101,12 +101,43 @@ function DiskArrays.readblock!(var::CDFVariable{T,N}, dest::AbstractArray{T}, ra copyto!(dest, DiskArrays.readblock!(var, Array{T}(undef, size(dest)), ranges...)) return dest end + GC.@preserve dest _readblock!(Ptr{UInt8}(_ptr(dest)), sizeof(T), _block_args(var, ranges)..., var.vdr, var.parentdataset) + is_big_endian_encoding(var) && _byte_swap!(dest) + return dest +end + +# CHAR elements are `num_elems`-byte fields, null-padded; the string ends at the trailing-null run. +function DiskArrays.readblock!(var::CDFVariable{String,N}, dest::AbstractArray{String}, ranges::Vararg{AbstractUnitRange{<:Integer},N}) where {N} + N > 0 && @boundscheck checkbounds(var, ranges...) + isempty(dest) && return dest + if !(dest isa Array) + copyto!(dest, DiskArrays.readblock!(var, Array{String}(undef, size(dest)), ranges...)) + return dest + end + w = Int(var.vdr.num_elems) + bytes = Vector{UInt8}(undef, w * length(dest)) + GC.@preserve bytes _readblock!(pointer(bytes), w, _block_args(var, ranges)..., var.vdr, var.parentdataset) + _fill_strings!(vec(dest), bytes, w) + return dest +end + +function _fill_strings!(dest::Vector{String}, bytes::Vector{UInt8}, w::Int) + GC.@preserve bytes for k in eachindex(dest) + o = (k - 1) * w + n = w + while n > 0 && iszero(@inbounds bytes[o + n]) + n -= 1 + end + dest[k] = unsafe_string(pointer(bytes, o + 1), n) + end + return dest +end + +function _block_args(var::CDFVariable, ranges) rdims = Int[Base.front(var.dims)...] other_ranges = UnitRange{Int}[Int(first(r)):Int(last(r)) for r in Base.front(ranges)] rec_range = Int(first(ranges[end])):Int(last(ranges[end])) - GC.@preserve dest _readblock!(Ptr{UInt8}(_ptr(dest)), sizeof(T), rdims, other_ranges, rec_range, var.vdr, var.parentdataset) - is_big_endian_encoding(var) && _byte_swap!(dest) - return dest + return rdims, other_ranges, rec_range end function _readblock!(dest::Ptr{UInt8}, esz::Int, rdims::Vector{Int}, other_ranges::Vector{UnitRange{Int}}, rec_range::UnitRange{Int}, vdr, ds) diff --git a/src/parsing.jl b/src/parsing.jl index 7bfa1ae..add478a 100644 --- a/src/parsing.jl +++ b/src/parsing.jl @@ -93,7 +93,6 @@ end # Big-endian files: swap each `unit`-byte word in place (0 = raw bytes, no swap). _swap_unit(::Type{T}) where {T} = sizeof(T) -_swap_unit(::Type{<:StaticString}) = 0 _swap_unit(::Type{Epoch16}) = 8 # two Float64s function _byte_swap!(p::Ptr{UInt8}, nbytes::Int, unit::Int) unit <= 1 && return diff --git a/src/precompile.jl b/src/precompile.jl index 367e53d..93e2318 100644 --- a/src/precompile.jl +++ b/src/precompile.jl @@ -1,7 +1,7 @@ # Seed one variable per common (eltype, ndims) shape, per CDF v2/v3 offset width. Every new combination costs DiskArrays indexing compiles, so rarer -# shapes (integers, Epoch16, >3-D, extra string widths) are left out. -_workload_key(var) = (eltype(var) <: StaticString ? StaticString : eltype(var), ndims(var), recordsize_type(var.parentdataset)) -_workload_maxdims(T) = T <: AbstractFloat ? 3 : T <: StaticString ? 2 : T in (TT2000, Epoch) ? 1 : 0 +# shapes (integers, Epoch16, >3-D) are left out. +_workload_key(var) = (eltype(var), ndims(var), recordsize_type(var.parentdataset)) +_workload_maxdims(T) = T <: AbstractFloat ? 3 : T === String ? 2 : T in (TT2000, Epoch) ? 1 : 0 PrecompileTools.@setup_workload begin files = joinpath.(@__DIR__, "../data", ("elb_l2_epdef_20210914_v01.cdf", "a_cdf_with_compressed_vars.cdf", "ac_h2_sis_20101105_v06.cdf")) diff --git a/src/staticstring.jl b/src/staticstring.jl deleted file mode 100644 index 62413fa..0000000 --- a/src/staticstring.jl +++ /dev/null @@ -1,95 +0,0 @@ -# https://github.com/mkitti/StaticStrings.jl -# https://github.com/JuliaPy/PythonCall.jl/blob/main/src/Utils/Utils.jl - -struct StaticString{N, T} <: AbstractString - codeunits::NTuple{N, T} - StaticString{N, T}(codeunits::NTuple{N, T}) where {N, T} = new{N, T}(codeunits) -end - -function Base.iterate(x::StaticString{N, UInt8}, i::Int = 1) where {N} - i > ncodeunits(x) && return - cs = x.codeunits - c = @inbounds cs[i] - if (c & 0x80) == 0x00 - return (reinterpret(Char, UInt32(c) << 24), i + 1) - elseif (c & 0x40) == 0x00 - nothing - elseif (c & 0x20) == 0x00 - if @inbounds (i ≤ N - 1) && ((cs[i + 1] & 0xC0) == 0x80) - return ( - reinterpret(Char, (UInt32(cs[i]) << 24) | (UInt32(cs[i + 1]) << 16)), - i + 2, - ) - end - elseif (c & 0x10) == 0x00 - if @inbounds (i ≤ N - 2) && ((cs[i + 1] & 0xC0) == 0x80) && ((cs[i + 2] & 0xC0) == 0x80) - return ( - reinterpret( - Char, - (UInt32(cs[i]) << 24) | - (UInt32(cs[i + 1]) << 16) | - (UInt32(cs[i + 2]) << 8), - ), - i + 3, - ) - end - elseif (c & 0x08) == 0x00 - if @inbounds (i ≤ N - 3) && - ((cs[i + 1] & 0xC0) == 0x80) && - ((cs[i + 2] & 0xC0) == 0x80) && - ((cs[i + 3] & 0xC0) == 0x80) - return ( - reinterpret( - Char, - (UInt32(cs[i]) << 24) | - (UInt32(cs[i + 1]) << 16) | - (UInt32(cs[i + 2]) << 8) | - UInt32(cs[i + 3]), - ), - i + 4, - ) - end - end - throw(StringIndexError(x, i)) -end - -function Base.String(x::StaticString) - n = ncodeunits(x) - b = Base.StringVector(n) - @inbounds for i in 1:n - b[i] = x.codeunits[i] - end - return String(b) -end - -# CDF CHAR values are fixed-width null-padded; the string ends at the trailing-null run -# so length/collect/String agree with iterate truncating there. -@inline function Base.ncodeunits(s::StaticString{N}) where {N} - cs = s.codeunits - n = N - while n > 0 && iszero(@inbounds cs[n]) - n -= 1 - end - return n -end -Base.codeunit(::StaticString{N, T}) where {N, T} = T -Base.@propagate_inbounds Base.codeunit(s::StaticString, i::Int) = s.codeunits[i] - -function StaticString(cu::Base.CodeUnits{T}) where {T} - N = length(cu) - return StaticString{N, T}(NTuple{N, T}(cu)) -end -StaticString(s::AbstractString) = StaticString(codeunits(s)) - -Base.isvalid(s::StaticString, i::Int) = checkbounds(Bool, s, i) && Base._thisind_str(s, i) == i - -# Specialized display machinery, costing compile time for each distinct CDF CHAR width -Base.show(io::IO, s::StaticString) = show(io, String(s)) - -function Base.show(io::IO, ::MIME"text/plain", a::AbstractArray{<:StaticString}) - summary(io, a) - isempty(a) && return - print(io, ":\n") - Base.print_array(io, map(String, a)) - return -end diff --git a/test/comprehensive_test.jl b/test/comprehensive_test.jl index 0259fa6..8429bd4 100644 --- a/test/comprehensive_test.jl +++ b/test/comprehensive_test.jl @@ -126,10 +126,8 @@ ds = CDFDataset(file) @test Set(keys(ds)) == Set(keys(EXPECTED_VARIABLES)) end -@testset "StaticString" begin - using CommonDataFormat: StaticString - @test typeof(ds["var_string"][1]) == StaticString{16, UInt8} - @test String(ds["var_string"][1]) == "This is a string" +@testset "CHAR" begin + @test ds["var_string"][1] == "This is a string" end @testset "DateTime Conversions" begin diff --git a/test/parsing_test.jl b/test/parsing_test.jl index 8d280a5..891616f 100644 --- a/test/parsing_test.jl +++ b/test/parsing_test.jl @@ -41,3 +41,8 @@ end @test Dict(la)["VAR_TYPE"] == "metadata" @test length(ds.vattrib["FIELDNAM"]) == length(ds) end + +@testset "CHAR null padding" begin + bytes = UInt8['a', 'b', 0, 0, 'c', 0, 'd', 0] + @test CommonDataFormat._fill_strings!(Vector{String}(undef, 2), bytes, 4) == ["ab", "c\0d"] +end diff --git a/test/runtests.jl b/test/runtests.jl index a784074..e009b65 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -11,9 +11,6 @@ include("cdf2_test.jl") include("CommonDataModelExt_test.jl") include("decompress_test.jl") include("parsing_test.jl") -@testset "StaticString" begin - include("staticstring.jl") -end @testset "Aqua" begin using Aqua diff --git a/test/staticstring.jl b/test/staticstring.jl deleted file mode 100644 index d2029ae..0000000 --- a/test/staticstring.jl +++ /dev/null @@ -1,51 +0,0 @@ -using CommonDataFormat: StaticString -using Test - -@testset "Basic operations" begin - s = "Hello, World!" - ss = StaticString(s) - - @test StaticString(codeunits(s)) == s - @test ss == s - @test String(ss) == s - @test !isempty(ss) - @test contains(ss, "World") - @test occursin("World", ss) - @test replace(ss, "World" => "Julia") == "Hello, Julia!" - - @test codeunit(ss) == UInt8 -end - -@testset "Null padding and UTF-8" begin - # null padding: iterate/length/collect/String must agree - pad = StaticString{8, UInt8}((UInt8('a'), UInt8('b'), zeros(UInt8, 6)...)) - @test ncodeunits(pad) == 2 - @test length(pad) == 2 - @test collect(pad) == ['a', 'b'] - @test String(pad) == "ab" - @test pad == "ab" - @test isempty(StaticString{4, UInt8}(ntuple(_ -> 0x00, 4))) - - # multi-byte UTF-8 indexing - s = StaticString("héllo") - @test collect(s) == collect("héllo") - @test thisind(s, 3) == 2 - @test isvalid(s, 2) && !isvalid(s, 3) - @test length(s) == 5 && ncodeunits(s) == 6 -end - -@testset "show" begin - pad = StaticString{8, UInt8}((UInt8('a'), UInt8('b'), zeros(UInt8, 6)...)) - @test sprint(show, pad) == "\"ab\"" # null padding trimmed, quoted like String - @test sprint(show, StaticString("a\"\n")) == sprint(show, "a\"\n") - - disp(x) = sprint(show, MIME"text/plain"(), x) - for a in (fill(pad, 3), fill(pad, 2, 2)) - header, body = split(disp(a), ":\n"; limit = 2) - # element rendering delegates to String; only the summary keeps the real eltype - @test body == split(disp(map(String, a)), ":\n"; limit = 2)[2] - @test occursin("StaticString{8, UInt8}", header) - end - empty = StaticString{4, UInt8}[] - @test disp(empty) == sprint(summary, empty) -end