perf: share compiled read paths across r/z-variables and CDF v2 files

src/dataset.jl

@@ -68,14 +68,7 @@ majority(cdf::CDFDataset) = majority(cdf.cdr)
     throw(ArgumentError("Unknown property $name"))
 end
 
-# Load the (z or r) VDR at a known offset
-function _vdr_at(cdf::CDFDataset{FST}, offset::Int) where {FST}
-    buffer = parent(cdf)
-    record_type = read_be(buffer, offset + 1 + sizeof(FST), Int32)
-    @assert record_type in (8, 3)
-    return record_type == 8 ? VDR{FST}(buffer, offset) :
-        rVDR{FST}(buffer, offset)
-end
+_vdr_at(cdf::CDFDataset{FST}, offset::Int) where {FST} = VDR{FST}(parent(cdf), offset)
 
 # Name is the first variable-width field of a VDR; everything before it is fixed size.
 vdr_name(buffer, offset, ::Type{FST}) where {FST} = readname(buffer, offset + 45 + 5 * sizeof(FST))

src/loading/attribute.jl

@@ -114,7 +114,8 @@ Base.haskey(la::LazyVAttrib, name) = false
 # https://github.com/SciQLop/PyISTP/blob/0a565c39c73dd800934bc379dd7c2e00c28d23d0/pyistp/_impl.py#L16
 function _get_attributes(name, value, cdf)
     if occursin("LABL_PTR", name)
-        return cdf[value][:]
+        # `Array` skips DiskArrays indexing
+        return vec(Array(cdf[value]))
     end
     return value
 end

src/loading/variable.jl

@@ -203,7 +203,7 @@ function collect_vxr_entries!(entries::Vector{VVREntry}, src, offset, ::Type{Fie
     return entries
 end
 
-function variable_compression(vdr::AbstractVDR{FieldSizeT}, cdf) where {FieldSizeT}
+function variable_compression(vdr::VDR{FieldSizeT}, cdf) where {FieldSizeT}
     offset_value = Int(vdr.cpr_or_spr_offset)
     if is_compressed(vdr) && offset_value != 0
         buffer = parent(cdf)

src/precompile.jl

@@ -1,11 +1,11 @@
-# Seed one variable per common (eltype, ndims) shape; every new pair costs DiskArrays
-# indexing compiles, so rarer shapes (integers, Epoch16, >3-D, extra string widths) are left out.
-_workload_key(var) = (eltype(var) <: StaticString ? StaticString : eltype(var), ndims(var))
+# Seed one variable per common (eltype, ndims) shape, per CDF v2/v3 offset width. Every new combination costs DiskArrays indexing compiles, so rarer
+# shapes (integers, Epoch16, >3-D, extra string widths) are left out.
+_workload_key(var) = (eltype(var) <: StaticString ? StaticString : eltype(var), ndims(var), recordsize_type(var.parentdataset))
 _workload_maxdims(T) = T <: AbstractFloat ? 3 : T <: StaticString ? 2 : T in (TT2000, Epoch) ? 1 : 0
 
 PrecompileTools.@setup_workload begin
-    files = joinpath.(@__DIR__, "../data", ("elb_l2_epdef_20210914_v01.cdf", "a_cdf_with_compressed_vars.cdf"))
-    seen = Set{Tuple{Any, Int}}()
+    files = joinpath.(@__DIR__, "../data", ("elb_l2_epdef_20210914_v01.cdf", "a_cdf_with_compressed_vars.cdf", "ac_h2_sis_20101105_v06.cdf"))
+    seen = Set{Tuple{Any, Int, DataType}}()
 
     PrecompileTools.@compile_workload begin
         for (i, file) in enumerate(files)
@@ -15,7 +15,7 @@ PrecompileTools.@setup_workload begin
                 key = _workload_key(var)
                 (isempty(var) || key[2] > _workload_maxdims(key[1]) || key in seen) && continue
                 push!(seen, key)
-                get(var.attrib, "FILLVAL", nothing)
+                Dict(var.attrib)
                 Array(var)
                 var[1]
                 var[ntuple(_ -> Colon(), ndims(var) - 1)..., 1:1]

src/records/vdr.jl

@@ -1,31 +1,9 @@
 # VDR loading functionality
 
-abstract type AbstractVDR{FST} <: Record end
-
-struct rVDR{FST} <: AbstractVDR{FST}
-    vdr_next::FST     # Offset to next VDR in chain
-    data_type::Int32    # CDF data type
-    max_rec::Int32       # Maximum record number (-1 if none)
-    vxr_head::FST     # Variable indeX Record head
-    vxr_tail::FST     # Variable indeX Record tail
-    flags::Int32        # Variable flags
-    s_records::Int32    # Sparse records flag
-    rfu_b::RInt32        # Reserved field B
-    rfu_c::RInt32        # Reserved field C
-    rfu_f::RInt32        # Reserved field F
-    num_elems::Int32    # Number of elements (for strings)
-    num::Int32          # Variable number
-    cpr_or_spr_offset::FST  # Compression/Sparseness Parameters Record offset
-    # blocking_factor::Int32
-    # name::S         # Variable name
-    pos::Int
-end
-
 """
-Variable Descriptor Record (VDR) - describes a single variable
+Variable Descriptor Record (VDR) - describes a single r- or z-variable.
 """
-struct VDR{FST} <: AbstractVDR{FST}
-    # header::Header
+struct VDR{FST} <: Record
     vdr_next::FST     # Offset to next VDR in chain
     data_type::Int32    # CDF data type
     max_rec::Int32       # Maximum record number (-1 if none)
@@ -37,14 +15,13 @@ struct VDR{FST} <: AbstractVDR{FST}
     rfu_c::RInt32        # Reserved field C
     rfu_f::RInt32        # Reserved field F
     num_elems::Int32    # Number of elements (for strings)
-    num::Int32          # Variable number
+    num::Int32          # Variable number (r- and z-variables are numbered independently)
     cpr_or_spr_offset::FST  # Compression/Sparseness Parameters Record offset
     # blocking_factor::Int32
     # name::S         # Variable name
-    num_dims::Int32   # Number of dimensions
-    pos::Int
-    # z_dim_sizes::Tuple{Vararg{Int32}}  # Dimension sizes (z-variables only)
-    # dim_varys::Tuple{Vararg{Int32}}    # Dimension variance flags
+    zvar::Bool
+    num_dims::Int32   # zNumDims; unused for r-variables, whose dims come from the GDR
+    pos::Int          # zDimSizes (z) / DimVarys (r)
 end
 
 # zNumDims (zVDRs) / DimVarys (rVDRs) start right after the fixed-width name field.
@@ -53,33 +30,28 @@ _after_name(offset, ::Type{FieldSizeT}) where {FieldSizeT} = FieldSizeT == Int64
 """
     VDR{FieldSizeT}(buffer, offset)
 
-Load a z-Variable Descriptor Record from the buffer at the specified offset.
+Load a z- or r-Variable Descriptor Record from the buffer at the specified offset.
 """
 @inline function VDR{FieldSizeT}(buffer::Vector{UInt8}, offset) where {FieldSizeT}
-    pos = check_record_type(8, buffer, offset, FieldSizeT)
+    record_type = get_record_type(buffer, offset, FieldSizeT)
+    @assert record_type in (8, 3)
+    zvar = record_type == 8
+    pos = offset + sizeof(FieldSizeT) + 5
     fields, _ = read_be_fields(buffer, pos, VDR{FieldSizeT}, Val(1:13))
-    z_num_dims, pos = read_be_i(buffer, _after_name(offset, FieldSizeT), Int32)
-    return VDR{FieldSizeT}(fields..., z_num_dims, pos)
-end
-
-"""
-    rVDR{FieldSizeT}(buffer, offset)
-
-Load an r-Variable Descriptor Record from the buffer at the specified offset.
-"""
-@inline function rVDR{FieldSizeT}(buffer::Vector{UInt8}, offset) where {FieldSizeT}
-    pos = check_record_type(3, buffer, offset, FieldSizeT)
-    fields, _ = read_be_fields(buffer, pos, rVDR{FieldSizeT}, Val(1:13))
-    return rVDR{FieldSizeT}(fields..., _after_name(offset, FieldSizeT))
+    after_name = _after_name(offset, FieldSizeT)
+    zvar || return VDR{FieldSizeT}(fields..., false, Int32(0), after_name)
+    z_num_dims, pos = read_be_i(buffer, after_name, Int32)
+    return VDR{FieldSizeT}(fields..., true, z_num_dims, pos)
 end
 
 function record_sizes(vdr::VDR, cdf, ::Val{M}) where {M}
+    vdr.zvar || return _rvar_record_sizes(vdr, cdf, Val(M))
     vdr.num_dims == M ||
         throw(DimensionMismatch("variable has $(vdr.num_dims) dimensions, expected $M"))
     return read_be(parent(cdf), vdr.pos, Val(M), Int32)
 end
 
-function record_sizes(vdr::rVDR, cdf, ::Val{M}) where {M}
+function _rvar_record_sizes(vdr::VDR, cdf, ::Val{M}) where {M}
     gdr = GDR(cdf)
     buf = parent(cdf)
     sizes_pos = gdr.pos + sizeof(Int64) + 3 * sizeof(Int32)
@@ -94,8 +66,8 @@ function record_sizes(vdr::rVDR, cdf, ::Val{M}) where {M}
     return ntuple(i -> sizes[i], Val(M))
 end
 
-num_record_dims(vdr::VDR, cdf) = Int(vdr.num_dims)
-function num_record_dims(vdr::rVDR, cdf)
+function num_record_dims(vdr::VDR, cdf)
+    vdr.zvar && return Int(vdr.num_dims)
     n = 0
     for i in 1:Int(GDR(cdf).r_num_dims)
         n += read_be(parent(cdf), vdr.pos + (i - 1) * 4, Int32) != 0
@@ -104,7 +76,7 @@ function num_record_dims(vdr::rVDR, cdf)
 end
 
 
-function Base.show(io::IO, vdr::AbstractVDR)
+function Base.show(io::IO, vdr::VDR)
     print(io, "VDR: ", CDFDataType(vdr.data_type))
     is_nrv(vdr) && print(io, " [NRV]")
     is_compressed(vdr) && print(io, " [compressed]")
@@ -116,7 +88,7 @@ end
 # 1 Whether or not a pad value is specified for this variable. Set indicates that a pad value has been specified. Clear indicates that a pad value has not been specified. The PadValue field described below is only present if a pad value has been specified.
 # 2 Whether or not a compression method might be applied to this variable data. Set indicates that a compression is chosen by the user and the data might be compressed, depending on the data size and content. If the compressed data becomes larger than its uncompressed data, no compression is applied and the data are stored as uncompressed, even the compression bit is set. The compressed data is stored in Compressed Variable Value Record (CVVR) while uncompressed data go into Variable Value Record (VVR). Clear indicates that a compression will not be used. The CPRorSPRoffset field provides the offset of the Compressed Parameters Record if this compression bit is set and the compression used.
 
-function read_vvrs(vdr::AbstractVDR{FieldSizeT}, cdf) where {FieldSizeT}
+function read_vvrs(vdr::VDR{FieldSizeT}, cdf) where {FieldSizeT}
     entries = Vector{VVREntry}()
     sizehint!(entries, 1)
     collect_vxr_entries!(entries, parent(cdf), Int(vdr.vxr_head), FieldSizeT)
@@ -126,4 +98,4 @@ end
 is_record_varying(vdr) = !is_nrv(vdr)
 """Whether or not the variable is a non-record variable"""
 is_nrv(vdr) = (vdr.flags & 0x01) == 0
-is_compressed(vdr::AbstractVDR) = (vdr.flags & 0x04) != 0
+is_compressed(vdr::VDR) = (vdr.flags & 0x04) != 0

src/variable.jl

@@ -64,7 +64,7 @@ end
 Base.getindex(var::CDFVariable, name::String) = var.attrib[name]
 Base.haskey(var::CDFVariable, name::String) = haskey(var.attrib, name)
 
-attrib(var::CDFVariable) = LazyVAttrib(var.parentdataset, var.vdr.num, var.vdr isa VDR)
+attrib(var::CDFVariable) = LazyVAttrib(var.parentdataset, var.vdr.num, var.vdr.zvar)
 attrib(var::CDFVariable, name::String) = get(attrib(var), name)
 
 is_record_varying(v::CDFVariable) = is_record_varying(v.vdr)

test/runtests.jl

@@ -152,14 +152,14 @@ end
     file = data_path("ac_h0_mfi_20230102_v07.cdf")
     ds = CDFDataset(file)
     var = ds["BGSEc"]
-    @test var.vdr isa CommonDataFormat.rVDR
+    @test !var.vdr.zvar
     @test size(var) == (3, 5400)
 end
 
 @testset "r-variables 2" begin
     file = download_test_data("https://github.com/JuliaSpacePhysics/CommonDataFormat.jl/releases/download/v0.1.8/omni2_h0_mrg1hr_20150101_v01.cdf")
     ds = CDFDataset(file)
     var = ds["Epoch"]
-    @test var.vdr isa CommonDataFormat.rVDR
+    @test !var.vdr.zvar
     @test size(var) == (4344,)
 end