From b5f40f21aa72db8ee5f68453a96846fefd19d214 Mon Sep 17 00:00:00 2001 From: Beforerr Date: Mon, 5 Oct 2026 22:18:01 -0600 Subject: [PATCH 1/2] perf: share compiled read paths across r/z-variables and CDF v2 files Claude-Session: https://claude.ai/code/session_01SJ8TCKtj357ZjhLh9Y7dwQ --- src/dataset.jl | 9 +---- src/loading/attribute.jl | 3 +- src/loading/variable.jl | 2 +- src/precompile.jl | 12 +++---- src/records/vdr.jl | 72 ++++++++++++---------------------------- src/variable.jl | 2 +- test/runtests.jl | 4 +-- 7 files changed, 35 insertions(+), 69 deletions(-) diff --git a/src/dataset.jl b/src/dataset.jl index 543f832..8f06d41 100644 --- a/src/dataset.jl +++ b/src/dataset.jl @@ -68,14 +68,7 @@ majority(cdf::CDFDataset) = majority(cdf.cdr) throw(ArgumentError("Unknown property $name")) end -# Load the (z or r) VDR at a known offset -function _vdr_at(cdf::CDFDataset{FST}, offset::Int) where {FST} - buffer = parent(cdf) - record_type = read_be(buffer, offset + 1 + sizeof(FST), Int32) - @assert record_type in (8, 3) - return record_type == 8 ? VDR{FST}(buffer, offset) : - rVDR{FST}(buffer, offset) -end +_vdr_at(cdf::CDFDataset{FST}, offset::Int) where {FST} = VDR{FST}(parent(cdf), offset) # Name is the first variable-width field of a VDR; everything before it is fixed size. vdr_name(buffer, offset, ::Type{FST}) where {FST} = readname(buffer, offset + 45 + 5 * sizeof(FST)) diff --git a/src/loading/attribute.jl b/src/loading/attribute.jl index 4ebdc13..96b7539 100644 --- a/src/loading/attribute.jl +++ b/src/loading/attribute.jl @@ -114,7 +114,8 @@ Base.haskey(la::LazyVAttrib, name) = false # https://github.com/SciQLop/PyISTP/blob/0a565c39c73dd800934bc379dd7c2e00c28d23d0/pyistp/_impl.py#L16 function _get_attributes(name, value, cdf) if occursin("LABL_PTR", name) - return cdf[value][:] + # `Array` skips DiskArrays indexing + return vec(Array(cdf[value])) end return value end diff --git a/src/loading/variable.jl b/src/loading/variable.jl index 90a0786..1f03c5c 100644 --- a/src/loading/variable.jl +++ b/src/loading/variable.jl @@ -203,7 +203,7 @@ function collect_vxr_entries!(entries::Vector{VVREntry}, src, offset, ::Type{Fie return entries end -function variable_compression(vdr::AbstractVDR{FieldSizeT}, cdf) where {FieldSizeT} +function variable_compression(vdr::VDR{FieldSizeT}, cdf) where {FieldSizeT} offset_value = Int(vdr.cpr_or_spr_offset) if is_compressed(vdr) && offset_value != 0 buffer = parent(cdf) diff --git a/src/precompile.jl b/src/precompile.jl index 2ebd38e..367e53d 100644 --- a/src/precompile.jl +++ b/src/precompile.jl @@ -1,11 +1,11 @@ -# Seed one variable per common (eltype, ndims) shape; every new pair costs DiskArrays -# indexing compiles, so rarer shapes (integers, Epoch16, >3-D, extra string widths) are left out. -_workload_key(var) = (eltype(var) <: StaticString ? StaticString : eltype(var), ndims(var)) +# Seed one variable per common (eltype, ndims) shape, per CDF v2/v3 offset width. Every new combination costs DiskArrays indexing compiles, so rarer +# shapes (integers, Epoch16, >3-D, extra string widths) are left out. +_workload_key(var) = (eltype(var) <: StaticString ? StaticString : eltype(var), ndims(var), recordsize_type(var.parentdataset)) _workload_maxdims(T) = T <: AbstractFloat ? 3 : T <: StaticString ? 2 : T in (TT2000, Epoch) ? 1 : 0 PrecompileTools.@setup_workload begin - files = joinpath.(@__DIR__, "../data", ("elb_l2_epdef_20210914_v01.cdf", "a_cdf_with_compressed_vars.cdf")) - seen = Set{Tuple{Any, Int}}() + files = joinpath.(@__DIR__, "../data", ("elb_l2_epdef_20210914_v01.cdf", "a_cdf_with_compressed_vars.cdf", "ac_h2_sis_20101105_v06.cdf")) + seen = Set{Tuple{Any, Int, DataType}}() PrecompileTools.@compile_workload begin for (i, file) in enumerate(files) @@ -15,7 +15,7 @@ PrecompileTools.@setup_workload begin key = _workload_key(var) (isempty(var) || key[2] > _workload_maxdims(key[1]) || key in seen) && continue push!(seen, key) - get(var.attrib, "FILLVAL", nothing) + Dict(var.attrib) Array(var) var[1] var[ntuple(_ -> Colon(), ndims(var) - 1)..., 1:1] diff --git a/src/records/vdr.jl b/src/records/vdr.jl index 09d7ce7..5f29895 100644 --- a/src/records/vdr.jl +++ b/src/records/vdr.jl @@ -1,31 +1,9 @@ # VDR loading functionality -abstract type AbstractVDR{FST} <: Record end - -struct rVDR{FST} <: AbstractVDR{FST} - vdr_next::FST # Offset to next VDR in chain - data_type::Int32 # CDF data type - max_rec::Int32 # Maximum record number (-1 if none) - vxr_head::FST # Variable indeX Record head - vxr_tail::FST # Variable indeX Record tail - flags::Int32 # Variable flags - s_records::Int32 # Sparse records flag - rfu_b::RInt32 # Reserved field B - rfu_c::RInt32 # Reserved field C - rfu_f::RInt32 # Reserved field F - num_elems::Int32 # Number of elements (for strings) - num::Int32 # Variable number - cpr_or_spr_offset::FST # Compression/Sparseness Parameters Record offset - # blocking_factor::Int32 - # name::S # Variable name - pos::Int -end - """ -Variable Descriptor Record (VDR) - describes a single variable +Variable Descriptor Record (VDR) - describes a single r- or z-variable. """ -struct VDR{FST} <: AbstractVDR{FST} - # header::Header +struct VDR{FST} <: Record vdr_next::FST # Offset to next VDR in chain data_type::Int32 # CDF data type max_rec::Int32 # Maximum record number (-1 if none) @@ -37,14 +15,13 @@ struct VDR{FST} <: AbstractVDR{FST} rfu_c::RInt32 # Reserved field C rfu_f::RInt32 # Reserved field F num_elems::Int32 # Number of elements (for strings) - num::Int32 # Variable number + num::Int32 # Variable number (r- and z-variables are numbered independently) cpr_or_spr_offset::FST # Compression/Sparseness Parameters Record offset # blocking_factor::Int32 # name::S # Variable name - num_dims::Int32 # Number of dimensions - pos::Int - # z_dim_sizes::Tuple{Vararg{Int32}} # Dimension sizes (z-variables only) - # dim_varys::Tuple{Vararg{Int32}} # Dimension variance flags + zvar::Bool + num_dims::Int32 # zNumDims; unused for r-variables, whose dims come from the GDR + pos::Int # zDimSizes (z) / DimVarys (r) end # zNumDims (zVDRs) / DimVarys (rVDRs) start right after the fixed-width name field. @@ -53,33 +30,28 @@ _after_name(offset, ::Type{FieldSizeT}) where {FieldSizeT} = FieldSizeT == Int64 """ VDR{FieldSizeT}(buffer, offset) -Load a z-Variable Descriptor Record from the buffer at the specified offset. +Load a z- or r-Variable Descriptor Record from the buffer at the specified offset. """ @inline function VDR{FieldSizeT}(buffer::Vector{UInt8}, offset) where {FieldSizeT} - pos = check_record_type(8, buffer, offset, FieldSizeT) + record_type = get_record_type(buffer, offset, FieldSizeT) + @assert record_type in (8, 3) + zvar = record_type == 8 + pos = offset + sizeof(FieldSizeT) + 5 fields, _ = read_be_fields(buffer, pos, VDR{FieldSizeT}, Val(1:13)) - z_num_dims, pos = read_be_i(buffer, _after_name(offset, FieldSizeT), Int32) - return VDR{FieldSizeT}(fields..., z_num_dims, pos) -end - -""" - rVDR{FieldSizeT}(buffer, offset) - -Load an r-Variable Descriptor Record from the buffer at the specified offset. -""" -@inline function rVDR{FieldSizeT}(buffer::Vector{UInt8}, offset) where {FieldSizeT} - pos = check_record_type(3, buffer, offset, FieldSizeT) - fields, _ = read_be_fields(buffer, pos, rVDR{FieldSizeT}, Val(1:13)) - return rVDR{FieldSizeT}(fields..., _after_name(offset, FieldSizeT)) + after_name = _after_name(offset, FieldSizeT) + zvar || return VDR{FieldSizeT}(fields..., false, Int32(0), after_name) + z_num_dims, pos = read_be_i(buffer, after_name, Int32) + return VDR{FieldSizeT}(fields..., true, z_num_dims, pos) end function record_sizes(vdr::VDR, cdf, ::Val{M}) where {M} + vdr.zvar || return _rvar_record_sizes(vdr, cdf, Val(M)) vdr.num_dims == M || throw(DimensionMismatch("variable has $(vdr.num_dims) dimensions, expected $M")) return read_be(parent(cdf), vdr.pos, Val(M), Int32) end -function record_sizes(vdr::rVDR, cdf, ::Val{M}) where {M} +function _rvar_record_sizes(vdr::VDR, cdf, ::Val{M}) where {M} gdr = GDR(cdf) buf = parent(cdf) sizes_pos = gdr.pos + sizeof(Int64) + 3 * sizeof(Int32) @@ -94,8 +66,8 @@ function record_sizes(vdr::rVDR, cdf, ::Val{M}) where {M} return ntuple(i -> sizes[i], Val(M)) end -num_record_dims(vdr::VDR, cdf) = Int(vdr.num_dims) -function num_record_dims(vdr::rVDR, cdf) +function num_record_dims(vdr::VDR, cdf) + vdr.zvar && return Int(vdr.num_dims) n = 0 for i in 1:Int(GDR(cdf).r_num_dims) n += read_be(parent(cdf), vdr.pos + (i - 1) * 4, Int32) != 0 @@ -104,7 +76,7 @@ function num_record_dims(vdr::rVDR, cdf) end -function Base.show(io::IO, vdr::AbstractVDR) +function Base.show(io::IO, vdr::VDR) print(io, "VDR: ", CDFDataType(vdr.data_type)) is_nrv(vdr) && print(io, " [NRV]") is_compressed(vdr) && print(io, " [compressed]") @@ -116,7 +88,7 @@ end # 1 Whether or not a pad value is specified for this variable. Set indicates that a pad value has been specified. Clear indicates that a pad value has not been specified. The PadValue field described below is only present if a pad value has been specified. # 2 Whether or not a compression method might be applied to this variable data. Set indicates that a compression is chosen by the user and the data might be compressed, depending on the data size and content. If the compressed data becomes larger than its uncompressed data, no compression is applied and the data are stored as uncompressed, even the compression bit is set. The compressed data is stored in Compressed Variable Value Record (CVVR) while uncompressed data go into Variable Value Record (VVR). Clear indicates that a compression will not be used. The CPRorSPRoffset field provides the offset of the Compressed Parameters Record if this compression bit is set and the compression used. -function read_vvrs(vdr::AbstractVDR{FieldSizeT}, cdf) where {FieldSizeT} +function read_vvrs(vdr::VDR{FieldSizeT}, cdf) where {FieldSizeT} entries = Vector{VVREntry}() sizehint!(entries, 1) collect_vxr_entries!(entries, parent(cdf), Int(vdr.vxr_head), FieldSizeT) @@ -126,4 +98,4 @@ end is_record_varying(vdr) = !is_nrv(vdr) """Whether or not the variable is a non-record variable""" is_nrv(vdr) = (vdr.flags & 0x01) == 0 -is_compressed(vdr::AbstractVDR) = (vdr.flags & 0x04) != 0 +is_compressed(vdr::VDR) = (vdr.flags & 0x04) != 0 diff --git a/src/variable.jl b/src/variable.jl index 6c4f2e5..eda1227 100644 --- a/src/variable.jl +++ b/src/variable.jl @@ -64,7 +64,7 @@ end Base.getindex(var::CDFVariable, name::String) = var.attrib[name] Base.haskey(var::CDFVariable, name::String) = haskey(var.attrib, name) -attrib(var::CDFVariable) = LazyVAttrib(var.parentdataset, var.vdr.num, var.vdr isa VDR) +attrib(var::CDFVariable) = LazyVAttrib(var.parentdataset, var.vdr.num, var.vdr.zvar) attrib(var::CDFVariable, name::String) = get(attrib(var), name) is_record_varying(v::CDFVariable) = is_record_varying(v.vdr) diff --git a/test/runtests.jl b/test/runtests.jl index 403d2c6..bf563b8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -152,7 +152,7 @@ end file = data_path("ac_h0_mfi_20230102_v07.cdf") ds = CDFDataset(file) var = ds["BGSEc"] - @test var.vdr isa CommonDataFormat.rVDR + @test !var.vdr.zvar @test size(var) == (3, 5400) end @@ -160,6 +160,6 @@ end file = download_test_data("https://github.com/JuliaSpacePhysics/CommonDataFormat.jl/releases/download/v0.1.8/omni2_h0_mrg1hr_20150101_v01.cdf") ds = CDFDataset(file) var = ds["Epoch"] - @test var.vdr isa CommonDataFormat.rVDR + @test !var.vdr.zvar @test size(var) == (4344,) end From 47dea4f63b19c90c49fcd2d855b34199097c88c9 Mon Sep 17 00:00:00 2001 From: Beforerr Date: Tue, 6 Oct 2026 07:01:18 -0600 Subject: [PATCH 2/2] fix: drop NOVARY dimensions of z-variables NOVARY dimensions aren't physically stored, so counting them read past each variable's data. r-variables already dropped them; z-variables now share that path. Matches CDFpp and cdflib. --- data/make_novary_z.py | 16 ++++++++++++++ data/novary_z.cdf | Bin 0 -> 1054 bytes src/records/vdr.jl | 50 +++++++++++++++++++++++------------------- test/runtests.jl | 5 +++++ 4 files changed, 49 insertions(+), 22 deletions(-) create mode 100644 data/make_novary_z.py create mode 100644 data/novary_z.cdf diff --git a/data/make_novary_z.py b/data/make_novary_z.py new file mode 100644 index 0000000..7fb4d7c --- /dev/null +++ b/data/make_novary_z.py @@ -0,0 +1,16 @@ +# z-var "novary": declared (2,3,4) with dim 2 NOVARY, so each record stores 2x4 values. +# cdflib/CDFpp writers ignore Dim_Vary, so write (2,1,4) and patch the zVDR's dim 2 to size 3, NOVARY. +import sys, struct, numpy as np +from cdflib.cdfwrite import CDF +out = sys.argv[1] +c = CDF(out, cdf_spec={"Majority": "row_major"}, delete=True) +data = np.arange(5 * 2 * 4, dtype=np.int32).reshape(5, 2, 1, 4) +c.write_var({"Variable": "novary", "Data_Type": 4, "Num_Elements": 1, "Rec_Vary": True, + "Var_Type": "zVariable", "Dim_Sizes": [2, 1, 4]}, var_attrs={}, var_data=data) +c.close() +b = bytearray(open(out, "rb").read()) +p = b.find(b"novary".ljust(256, b"\0")) + 256 # zNumDims follows the 256-byte name +assert b.count(b"novary") == 1 and struct.unpack(">i", b[p:p + 4])[0] == 3 +b[p + 8:p + 12] = struct.pack(">i", 3) # zDimSizes[2] +b[p + 20:p + 24] = struct.pack(">i", 0) # DimVarys[2] +open(out, "wb").write(b) diff --git a/data/novary_z.cdf b/data/novary_z.cdf new file mode 100644 index 0000000000000000000000000000000000000000..4ca4cccee0f2c2e61ef85f8a2f79b13aeb2242d5 GIT binary patch literal 1054 zcmX^6nSqgk;s1XIFkrL*Qb0ZkFggGUW+3JSVm2rnNWlTqe<0v;&d<%w&r@(oEJ;*w z%P-1JEK$&Kc5&0>$|xx*D7MnqPfkhGOD|4K*2_yQPSi`!FXIX>NK8&u2*{``&P*P$?|%*iaNP9)bj!(1&nQuFPR%PxE#h)7N=?m6O3f+JQSePH zs?15uOHnW~FgG_;2n}}R(g-b1Em9~hEhxw@DzQ=k*{ch5fG$+TAt$rAq!{iJO)j`+ zC}e~H^)sQG!MFjH&msrogTf8UhK4nS$7p=C6iH47D9R1QtVn7>@c;yjB|w4$%!Cju zKnjF`fO!%U%?@FMVi*E|d?Y&=r+|#f%P#|lBuwRqq(SK%6n`L^1r`S&8W>m&Krs-= z17cn<3qpj+cXKcZeLRxI@SpcEw@_mM1H%Dz&0i}5oRqx2?_YR#Ca+>k@5JcjaS8MU3 literal 0 HcmV?d00001 diff --git a/src/records/vdr.jl b/src/records/vdr.jl index 5f29895..81f40f9 100644 --- a/src/records/vdr.jl +++ b/src/records/vdr.jl @@ -44,35 +44,41 @@ Load a z- or r-Variable Descriptor Record from the buffer at the specified offse return VDR{FieldSizeT}(fields..., true, z_num_dims, pos) end -function record_sizes(vdr::VDR, cdf, ::Val{M}) where {M} - vdr.zvar || return _rvar_record_sizes(vdr, cdf, Val(M)) - vdr.num_dims == M || - throw(DimensionMismatch("variable has $(vdr.num_dims) dimensions, expected $M")) - return read_be(parent(cdf), vdr.pos, Val(M), Int32) +# (declared dim count, position of their sizes, position of their DimVarys) +function _dim_layout(vdr::VDR, cdf) + vdr.zvar && return Int(vdr.num_dims), vdr.pos, vdr.pos + 4 * Int(vdr.num_dims) + gdr = GDR(cdf) + return Int(gdr.r_num_dims), gdr.pos + sizeof(Int64) + 3 * sizeof(Int32), vdr.pos end -function _rvar_record_sizes(vdr::VDR, cdf, ::Val{M}) where {M} - gdr = GDR(cdf) - buf = parent(cdf) - sizes_pos = gdr.pos + sizeof(Int64) + 3 * sizeof(Int32) - sizes = zeros(Int32, M) - count = 0 - for i in 1:Int(gdr.r_num_dims) - read_be(buf, vdr.pos + (i - 1) * 4, Int32) == 0 && continue - count += 1 - count <= M && (sizes[count] = read_be(buf, sizes_pos + (i - 1) * 4, Int32)) +_varies(buf, varys_pos, i) = read_be(buf, varys_pos + (i - 1) * 4, Int32) != 0 + +# Declared index of the `j`-th varying dimension +function _varying_dim(buf, varys_pos, j) + i = 0 + while j > 0 + i += 1 + j -= _varies(buf, varys_pos, i) end - count == M || throw(DimensionMismatch("variable has $count dimensions, expected $M")) - return ntuple(i -> sizes[i], Val(M)) + return i +end + +# NOVARY dimensions aren't physically stored, so they aren't array dimensions either. +# `M` must be `num_record_dims(vdr, cdf)`. +function record_sizes(vdr::VDR, cdf, ::Val{M}) where {M} + n, sizes_pos, varys_pos = _dim_layout(vdr, cdf) + buf = parent(cdf) + n == M && return read_be(buf, sizes_pos, Val(M), Int32) + return ntuple(j -> read_be(buf, sizes_pos + (_varying_dim(buf, varys_pos, j) - 1) * 4, Int32), Val(M)) end function num_record_dims(vdr::VDR, cdf) - vdr.zvar && return Int(vdr.num_dims) - n = 0 - for i in 1:Int(GDR(cdf).r_num_dims) - n += read_be(parent(cdf), vdr.pos + (i - 1) * 4, Int32) != 0 + n, _, varys_pos = _dim_layout(vdr, cdf) + count = 0 + for i in 1:n + count += _varies(parent(cdf), varys_pos, i) end - return n + return count end diff --git a/test/runtests.jl b/test/runtests.jl index bf563b8..a784074 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -120,6 +120,11 @@ end @test occursin("compressed", string(var.vdr)) end +@testset "NOVARY z-variable dimension" begin + v = CDFDataset(data_path("novary_z.cdf"))["novary"] # row-major, declared (2,3,4), dim 2 NOVARY + @test Array(v) == permutedims(reshape(Int32(0):Int32(39), 4, 2, 5), (2, 1, 3)) +end + @testset "CHECK_VARIABLES - Variable structure verification" begin file = data_path("omni_coho1hr_merged_mag_plasma_20240901_v01.cdf") ds = CDFDataset(file)