From b5f40f21aa72db8ee5f68453a96846fefd19d214 Mon Sep 17 00:00:00 2001 From: Beforerr Date: Mon, 5 Oct 2026 22:18:01 -0600 Subject: [PATCH] perf: share compiled read paths across r/z-variables and CDF v2 files Claude-Session: https://claude.ai/code/session_01SJ8TCKtj357ZjhLh9Y7dwQ --- src/dataset.jl | 9 +---- src/loading/attribute.jl | 3 +- src/loading/variable.jl | 2 +- src/precompile.jl | 12 +++---- src/records/vdr.jl | 72 ++++++++++++---------------------------- src/variable.jl | 2 +- test/runtests.jl | 4 +-- 7 files changed, 35 insertions(+), 69 deletions(-) diff --git a/src/dataset.jl b/src/dataset.jl index 543f832..8f06d41 100644 --- a/src/dataset.jl +++ b/src/dataset.jl @@ -68,14 +68,7 @@ majority(cdf::CDFDataset) = majority(cdf.cdr) throw(ArgumentError("Unknown property $name")) end -# Load the (z or r) VDR at a known offset -function _vdr_at(cdf::CDFDataset{FST}, offset::Int) where {FST} - buffer = parent(cdf) - record_type = read_be(buffer, offset + 1 + sizeof(FST), Int32) - @assert record_type in (8, 3) - return record_type == 8 ? VDR{FST}(buffer, offset) : - rVDR{FST}(buffer, offset) -end +_vdr_at(cdf::CDFDataset{FST}, offset::Int) where {FST} = VDR{FST}(parent(cdf), offset) # Name is the first variable-width field of a VDR; everything before it is fixed size. vdr_name(buffer, offset, ::Type{FST}) where {FST} = readname(buffer, offset + 45 + 5 * sizeof(FST)) diff --git a/src/loading/attribute.jl b/src/loading/attribute.jl index 4ebdc13..96b7539 100644 --- a/src/loading/attribute.jl +++ b/src/loading/attribute.jl @@ -114,7 +114,8 @@ Base.haskey(la::LazyVAttrib, name) = false # https://github.com/SciQLop/PyISTP/blob/0a565c39c73dd800934bc379dd7c2e00c28d23d0/pyistp/_impl.py#L16 function _get_attributes(name, value, cdf) if occursin("LABL_PTR", name) - return cdf[value][:] + # `Array` skips DiskArrays indexing + return vec(Array(cdf[value])) end return value end diff --git a/src/loading/variable.jl b/src/loading/variable.jl index 90a0786..1f03c5c 100644 --- a/src/loading/variable.jl +++ b/src/loading/variable.jl @@ -203,7 +203,7 @@ function collect_vxr_entries!(entries::Vector{VVREntry}, src, offset, ::Type{Fie return entries end -function variable_compression(vdr::AbstractVDR{FieldSizeT}, cdf) where {FieldSizeT} +function variable_compression(vdr::VDR{FieldSizeT}, cdf) where {FieldSizeT} offset_value = Int(vdr.cpr_or_spr_offset) if is_compressed(vdr) && offset_value != 0 buffer = parent(cdf) diff --git a/src/precompile.jl b/src/precompile.jl index 2ebd38e..367e53d 100644 --- a/src/precompile.jl +++ b/src/precompile.jl @@ -1,11 +1,11 @@ -# Seed one variable per common (eltype, ndims) shape; every new pair costs DiskArrays -# indexing compiles, so rarer shapes (integers, Epoch16, >3-D, extra string widths) are left out. -_workload_key(var) = (eltype(var) <: StaticString ? StaticString : eltype(var), ndims(var)) +# Seed one variable per common (eltype, ndims) shape, per CDF v2/v3 offset width. Every new combination costs DiskArrays indexing compiles, so rarer +# shapes (integers, Epoch16, >3-D, extra string widths) are left out. +_workload_key(var) = (eltype(var) <: StaticString ? StaticString : eltype(var), ndims(var), recordsize_type(var.parentdataset)) _workload_maxdims(T) = T <: AbstractFloat ? 3 : T <: StaticString ? 2 : T in (TT2000, Epoch) ? 1 : 0 PrecompileTools.@setup_workload begin - files = joinpath.(@__DIR__, "../data", ("elb_l2_epdef_20210914_v01.cdf", "a_cdf_with_compressed_vars.cdf")) - seen = Set{Tuple{Any, Int}}() + files = joinpath.(@__DIR__, "../data", ("elb_l2_epdef_20210914_v01.cdf", "a_cdf_with_compressed_vars.cdf", "ac_h2_sis_20101105_v06.cdf")) + seen = Set{Tuple{Any, Int, DataType}}() PrecompileTools.@compile_workload begin for (i, file) in enumerate(files) @@ -15,7 +15,7 @@ PrecompileTools.@setup_workload begin key = _workload_key(var) (isempty(var) || key[2] > _workload_maxdims(key[1]) || key in seen) && continue push!(seen, key) - get(var.attrib, "FILLVAL", nothing) + Dict(var.attrib) Array(var) var[1] var[ntuple(_ -> Colon(), ndims(var) - 1)..., 1:1] diff --git a/src/records/vdr.jl b/src/records/vdr.jl index 09d7ce7..5f29895 100644 --- a/src/records/vdr.jl +++ b/src/records/vdr.jl @@ -1,31 +1,9 @@ # VDR loading functionality -abstract type AbstractVDR{FST} <: Record end - -struct rVDR{FST} <: AbstractVDR{FST} - vdr_next::FST # Offset to next VDR in chain - data_type::Int32 # CDF data type - max_rec::Int32 # Maximum record number (-1 if none) - vxr_head::FST # Variable indeX Record head - vxr_tail::FST # Variable indeX Record tail - flags::Int32 # Variable flags - s_records::Int32 # Sparse records flag - rfu_b::RInt32 # Reserved field B - rfu_c::RInt32 # Reserved field C - rfu_f::RInt32 # Reserved field F - num_elems::Int32 # Number of elements (for strings) - num::Int32 # Variable number - cpr_or_spr_offset::FST # Compression/Sparseness Parameters Record offset - # blocking_factor::Int32 - # name::S # Variable name - pos::Int -end - """ -Variable Descriptor Record (VDR) - describes a single variable +Variable Descriptor Record (VDR) - describes a single r- or z-variable. """ -struct VDR{FST} <: AbstractVDR{FST} - # header::Header +struct VDR{FST} <: Record vdr_next::FST # Offset to next VDR in chain data_type::Int32 # CDF data type max_rec::Int32 # Maximum record number (-1 if none) @@ -37,14 +15,13 @@ struct VDR{FST} <: AbstractVDR{FST} rfu_c::RInt32 # Reserved field C rfu_f::RInt32 # Reserved field F num_elems::Int32 # Number of elements (for strings) - num::Int32 # Variable number + num::Int32 # Variable number (r- and z-variables are numbered independently) cpr_or_spr_offset::FST # Compression/Sparseness Parameters Record offset # blocking_factor::Int32 # name::S # Variable name - num_dims::Int32 # Number of dimensions - pos::Int - # z_dim_sizes::Tuple{Vararg{Int32}} # Dimension sizes (z-variables only) - # dim_varys::Tuple{Vararg{Int32}} # Dimension variance flags + zvar::Bool + num_dims::Int32 # zNumDims; unused for r-variables, whose dims come from the GDR + pos::Int # zDimSizes (z) / DimVarys (r) end # zNumDims (zVDRs) / DimVarys (rVDRs) start right after the fixed-width name field. @@ -53,33 +30,28 @@ _after_name(offset, ::Type{FieldSizeT}) where {FieldSizeT} = FieldSizeT == Int64 """ VDR{FieldSizeT}(buffer, offset) -Load a z-Variable Descriptor Record from the buffer at the specified offset. +Load a z- or r-Variable Descriptor Record from the buffer at the specified offset. """ @inline function VDR{FieldSizeT}(buffer::Vector{UInt8}, offset) where {FieldSizeT} - pos = check_record_type(8, buffer, offset, FieldSizeT) + record_type = get_record_type(buffer, offset, FieldSizeT) + @assert record_type in (8, 3) + zvar = record_type == 8 + pos = offset + sizeof(FieldSizeT) + 5 fields, _ = read_be_fields(buffer, pos, VDR{FieldSizeT}, Val(1:13)) - z_num_dims, pos = read_be_i(buffer, _after_name(offset, FieldSizeT), Int32) - return VDR{FieldSizeT}(fields..., z_num_dims, pos) -end - -""" - rVDR{FieldSizeT}(buffer, offset) - -Load an r-Variable Descriptor Record from the buffer at the specified offset. -""" -@inline function rVDR{FieldSizeT}(buffer::Vector{UInt8}, offset) where {FieldSizeT} - pos = check_record_type(3, buffer, offset, FieldSizeT) - fields, _ = read_be_fields(buffer, pos, rVDR{FieldSizeT}, Val(1:13)) - return rVDR{FieldSizeT}(fields..., _after_name(offset, FieldSizeT)) + after_name = _after_name(offset, FieldSizeT) + zvar || return VDR{FieldSizeT}(fields..., false, Int32(0), after_name) + z_num_dims, pos = read_be_i(buffer, after_name, Int32) + return VDR{FieldSizeT}(fields..., true, z_num_dims, pos) end function record_sizes(vdr::VDR, cdf, ::Val{M}) where {M} + vdr.zvar || return _rvar_record_sizes(vdr, cdf, Val(M)) vdr.num_dims == M || throw(DimensionMismatch("variable has $(vdr.num_dims) dimensions, expected $M")) return read_be(parent(cdf), vdr.pos, Val(M), Int32) end -function record_sizes(vdr::rVDR, cdf, ::Val{M}) where {M} +function _rvar_record_sizes(vdr::VDR, cdf, ::Val{M}) where {M} gdr = GDR(cdf) buf = parent(cdf) sizes_pos = gdr.pos + sizeof(Int64) + 3 * sizeof(Int32) @@ -94,8 +66,8 @@ function record_sizes(vdr::rVDR, cdf, ::Val{M}) where {M} return ntuple(i -> sizes[i], Val(M)) end -num_record_dims(vdr::VDR, cdf) = Int(vdr.num_dims) -function num_record_dims(vdr::rVDR, cdf) +function num_record_dims(vdr::VDR, cdf) + vdr.zvar && return Int(vdr.num_dims) n = 0 for i in 1:Int(GDR(cdf).r_num_dims) n += read_be(parent(cdf), vdr.pos + (i - 1) * 4, Int32) != 0 @@ -104,7 +76,7 @@ function num_record_dims(vdr::rVDR, cdf) end -function Base.show(io::IO, vdr::AbstractVDR) +function Base.show(io::IO, vdr::VDR) print(io, "VDR: ", CDFDataType(vdr.data_type)) is_nrv(vdr) && print(io, " [NRV]") is_compressed(vdr) && print(io, " [compressed]") @@ -116,7 +88,7 @@ end # 1 Whether or not a pad value is specified for this variable. Set indicates that a pad value has been specified. Clear indicates that a pad value has not been specified. The PadValue field described below is only present if a pad value has been specified. # 2 Whether or not a compression method might be applied to this variable data. Set indicates that a compression is chosen by the user and the data might be compressed, depending on the data size and content. If the compressed data becomes larger than its uncompressed data, no compression is applied and the data are stored as uncompressed, even the compression bit is set. The compressed data is stored in Compressed Variable Value Record (CVVR) while uncompressed data go into Variable Value Record (VVR). Clear indicates that a compression will not be used. The CPRorSPRoffset field provides the offset of the Compressed Parameters Record if this compression bit is set and the compression used. -function read_vvrs(vdr::AbstractVDR{FieldSizeT}, cdf) where {FieldSizeT} +function read_vvrs(vdr::VDR{FieldSizeT}, cdf) where {FieldSizeT} entries = Vector{VVREntry}() sizehint!(entries, 1) collect_vxr_entries!(entries, parent(cdf), Int(vdr.vxr_head), FieldSizeT) @@ -126,4 +98,4 @@ end is_record_varying(vdr) = !is_nrv(vdr) """Whether or not the variable is a non-record variable""" is_nrv(vdr) = (vdr.flags & 0x01) == 0 -is_compressed(vdr::AbstractVDR) = (vdr.flags & 0x04) != 0 +is_compressed(vdr::VDR) = (vdr.flags & 0x04) != 0 diff --git a/src/variable.jl b/src/variable.jl index 6c4f2e5..eda1227 100644 --- a/src/variable.jl +++ b/src/variable.jl @@ -64,7 +64,7 @@ end Base.getindex(var::CDFVariable, name::String) = var.attrib[name] Base.haskey(var::CDFVariable, name::String) = haskey(var.attrib, name) -attrib(var::CDFVariable) = LazyVAttrib(var.parentdataset, var.vdr.num, var.vdr isa VDR) +attrib(var::CDFVariable) = LazyVAttrib(var.parentdataset, var.vdr.num, var.vdr.zvar) attrib(var::CDFVariable, name::String) = get(attrib(var), name) is_record_varying(v::CDFVariable) = is_record_varying(v.vdr) diff --git a/test/runtests.jl b/test/runtests.jl index 403d2c6..bf563b8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -152,7 +152,7 @@ end file = data_path("ac_h0_mfi_20230102_v07.cdf") ds = CDFDataset(file) var = ds["BGSEc"] - @test var.vdr isa CommonDataFormat.rVDR + @test !var.vdr.zvar @test size(var) == (3, 5400) end @@ -160,6 +160,6 @@ end file = download_test_data("https://github.com/JuliaSpacePhysics/CommonDataFormat.jl/releases/download/v0.1.8/omni2_h0_mrg1hr_20150101_v01.cdf") ds = CDFDataset(file) var = ds["Epoch"] - @test var.vdr isa CommonDataFormat.rVDR + @test !var.vdr.zvar @test size(var) == (4344,) end