From b5f40f21aa72db8ee5f68453a96846fefd19d214 Mon Sep 17 00:00:00 2001 From: Beforerr Date: Mon, 5 Oct 2026 22:18:01 -0600 Subject: [PATCH 1/3] perf: share compiled read paths across r/z-variables and CDF v2 files Claude-Session: https://claude.ai/code/session_01SJ8TCKtj357ZjhLh9Y7dwQ --- src/dataset.jl | 9 +---- src/loading/attribute.jl | 3 +- src/loading/variable.jl | 2 +- src/precompile.jl | 12 +++---- src/records/vdr.jl | 72 ++++++++++++---------------------------- src/variable.jl | 2 +- test/runtests.jl | 4 +-- 7 files changed, 35 insertions(+), 69 deletions(-) diff --git a/src/dataset.jl b/src/dataset.jl index 543f832..8f06d41 100644 --- a/src/dataset.jl +++ b/src/dataset.jl @@ -68,14 +68,7 @@ majority(cdf::CDFDataset) = majority(cdf.cdr) throw(ArgumentError("Unknown property $name")) end -# Load the (z or r) VDR at a known offset -function _vdr_at(cdf::CDFDataset{FST}, offset::Int) where {FST} - buffer = parent(cdf) - record_type = read_be(buffer, offset + 1 + sizeof(FST), Int32) - @assert record_type in (8, 3) - return record_type == 8 ? VDR{FST}(buffer, offset) : - rVDR{FST}(buffer, offset) -end +_vdr_at(cdf::CDFDataset{FST}, offset::Int) where {FST} = VDR{FST}(parent(cdf), offset) # Name is the first variable-width field of a VDR; everything before it is fixed size. vdr_name(buffer, offset, ::Type{FST}) where {FST} = readname(buffer, offset + 45 + 5 * sizeof(FST)) diff --git a/src/loading/attribute.jl b/src/loading/attribute.jl index 4ebdc13..96b7539 100644 --- a/src/loading/attribute.jl +++ b/src/loading/attribute.jl @@ -114,7 +114,8 @@ Base.haskey(la::LazyVAttrib, name) = false # https://github.com/SciQLop/PyISTP/blob/0a565c39c73dd800934bc379dd7c2e00c28d23d0/pyistp/_impl.py#L16 function _get_attributes(name, value, cdf) if occursin("LABL_PTR", name) - return cdf[value][:] + # `Array` skips DiskArrays indexing + return vec(Array(cdf[value])) end return value end diff --git a/src/loading/variable.jl b/src/loading/variable.jl index 90a0786..1f03c5c 100644 --- a/src/loading/variable.jl +++ b/src/loading/variable.jl @@ -203,7 +203,7 @@ function collect_vxr_entries!(entries::Vector{VVREntry}, src, offset, ::Type{Fie return entries end -function variable_compression(vdr::AbstractVDR{FieldSizeT}, cdf) where {FieldSizeT} +function variable_compression(vdr::VDR{FieldSizeT}, cdf) where {FieldSizeT} offset_value = Int(vdr.cpr_or_spr_offset) if is_compressed(vdr) && offset_value != 0 buffer = parent(cdf) diff --git a/src/precompile.jl b/src/precompile.jl index 2ebd38e..367e53d 100644 --- a/src/precompile.jl +++ b/src/precompile.jl @@ -1,11 +1,11 @@ -# Seed one variable per common (eltype, ndims) shape; every new pair costs DiskArrays -# indexing compiles, so rarer shapes (integers, Epoch16, >3-D, extra string widths) are left out. -_workload_key(var) = (eltype(var) <: StaticString ? StaticString : eltype(var), ndims(var)) +# Seed one variable per common (eltype, ndims) shape, per CDF v2/v3 offset width. Every new combination costs DiskArrays indexing compiles, so rarer +# shapes (integers, Epoch16, >3-D, extra string widths) are left out. +_workload_key(var) = (eltype(var) <: StaticString ? StaticString : eltype(var), ndims(var), recordsize_type(var.parentdataset)) _workload_maxdims(T) = T <: AbstractFloat ? 3 : T <: StaticString ? 2 : T in (TT2000, Epoch) ? 1 : 0 PrecompileTools.@setup_workload begin - files = joinpath.(@__DIR__, "../data", ("elb_l2_epdef_20210914_v01.cdf", "a_cdf_with_compressed_vars.cdf")) - seen = Set{Tuple{Any, Int}}() + files = joinpath.(@__DIR__, "../data", ("elb_l2_epdef_20210914_v01.cdf", "a_cdf_with_compressed_vars.cdf", "ac_h2_sis_20101105_v06.cdf")) + seen = Set{Tuple{Any, Int, DataType}}() PrecompileTools.@compile_workload begin for (i, file) in enumerate(files) @@ -15,7 +15,7 @@ PrecompileTools.@setup_workload begin key = _workload_key(var) (isempty(var) || key[2] > _workload_maxdims(key[1]) || key in seen) && continue push!(seen, key) - get(var.attrib, "FILLVAL", nothing) + Dict(var.attrib) Array(var) var[1] var[ntuple(_ -> Colon(), ndims(var) - 1)..., 1:1] diff --git a/src/records/vdr.jl b/src/records/vdr.jl index 09d7ce7..5f29895 100644 --- a/src/records/vdr.jl +++ b/src/records/vdr.jl @@ -1,31 +1,9 @@ # VDR loading functionality -abstract type AbstractVDR{FST} <: Record end - -struct rVDR{FST} <: AbstractVDR{FST} - vdr_next::FST # Offset to next VDR in chain - data_type::Int32 # CDF data type - max_rec::Int32 # Maximum record number (-1 if none) - vxr_head::FST # Variable indeX Record head - vxr_tail::FST # Variable indeX Record tail - flags::Int32 # Variable flags - s_records::Int32 # Sparse records flag - rfu_b::RInt32 # Reserved field B - rfu_c::RInt32 # Reserved field C - rfu_f::RInt32 # Reserved field F - num_elems::Int32 # Number of elements (for strings) - num::Int32 # Variable number - cpr_or_spr_offset::FST # Compression/Sparseness Parameters Record offset - # blocking_factor::Int32 - # name::S # Variable name - pos::Int -end - """ -Variable Descriptor Record (VDR) - describes a single variable +Variable Descriptor Record (VDR) - describes a single r- or z-variable. """ -struct VDR{FST} <: AbstractVDR{FST} - # header::Header +struct VDR{FST} <: Record vdr_next::FST # Offset to next VDR in chain data_type::Int32 # CDF data type max_rec::Int32 # Maximum record number (-1 if none) @@ -37,14 +15,13 @@ struct VDR{FST} <: AbstractVDR{FST} rfu_c::RInt32 # Reserved field C rfu_f::RInt32 # Reserved field F num_elems::Int32 # Number of elements (for strings) - num::Int32 # Variable number + num::Int32 # Variable number (r- and z-variables are numbered independently) cpr_or_spr_offset::FST # Compression/Sparseness Parameters Record offset # blocking_factor::Int32 # name::S # Variable name - num_dims::Int32 # Number of dimensions - pos::Int - # z_dim_sizes::Tuple{Vararg{Int32}} # Dimension sizes (z-variables only) - # dim_varys::Tuple{Vararg{Int32}} # Dimension variance flags + zvar::Bool + num_dims::Int32 # zNumDims; unused for r-variables, whose dims come from the GDR + pos::Int # zDimSizes (z) / DimVarys (r) end # zNumDims (zVDRs) / DimVarys (rVDRs) start right after the fixed-width name field. @@ -53,33 +30,28 @@ _after_name(offset, ::Type{FieldSizeT}) where {FieldSizeT} = FieldSizeT == Int64 """ VDR{FieldSizeT}(buffer, offset) -Load a z-Variable Descriptor Record from the buffer at the specified offset. +Load a z- or r-Variable Descriptor Record from the buffer at the specified offset. """ @inline function VDR{FieldSizeT}(buffer::Vector{UInt8}, offset) where {FieldSizeT} - pos = check_record_type(8, buffer, offset, FieldSizeT) + record_type = get_record_type(buffer, offset, FieldSizeT) + @assert record_type in (8, 3) + zvar = record_type == 8 + pos = offset + sizeof(FieldSizeT) + 5 fields, _ = read_be_fields(buffer, pos, VDR{FieldSizeT}, Val(1:13)) - z_num_dims, pos = read_be_i(buffer, _after_name(offset, FieldSizeT), Int32) - return VDR{FieldSizeT}(fields..., z_num_dims, pos) -end - -""" - rVDR{FieldSizeT}(buffer, offset) - -Load an r-Variable Descriptor Record from the buffer at the specified offset. -""" -@inline function rVDR{FieldSizeT}(buffer::Vector{UInt8}, offset) where {FieldSizeT} - pos = check_record_type(3, buffer, offset, FieldSizeT) - fields, _ = read_be_fields(buffer, pos, rVDR{FieldSizeT}, Val(1:13)) - return rVDR{FieldSizeT}(fields..., _after_name(offset, FieldSizeT)) + after_name = _after_name(offset, FieldSizeT) + zvar || return VDR{FieldSizeT}(fields..., false, Int32(0), after_name) + z_num_dims, pos = read_be_i(buffer, after_name, Int32) + return VDR{FieldSizeT}(fields..., true, z_num_dims, pos) end function record_sizes(vdr::VDR, cdf, ::Val{M}) where {M} + vdr.zvar || return _rvar_record_sizes(vdr, cdf, Val(M)) vdr.num_dims == M || throw(DimensionMismatch("variable has $(vdr.num_dims) dimensions, expected $M")) return read_be(parent(cdf), vdr.pos, Val(M), Int32) end -function record_sizes(vdr::rVDR, cdf, ::Val{M}) where {M} +function _rvar_record_sizes(vdr::VDR, cdf, ::Val{M}) where {M} gdr = GDR(cdf) buf = parent(cdf) sizes_pos = gdr.pos + sizeof(Int64) + 3 * sizeof(Int32) @@ -94,8 +66,8 @@ function record_sizes(vdr::rVDR, cdf, ::Val{M}) where {M} return ntuple(i -> sizes[i], Val(M)) end -num_record_dims(vdr::VDR, cdf) = Int(vdr.num_dims) -function num_record_dims(vdr::rVDR, cdf) +function num_record_dims(vdr::VDR, cdf) + vdr.zvar && return Int(vdr.num_dims) n = 0 for i in 1:Int(GDR(cdf).r_num_dims) n += read_be(parent(cdf), vdr.pos + (i - 1) * 4, Int32) != 0 @@ -104,7 +76,7 @@ function num_record_dims(vdr::rVDR, cdf) end -function Base.show(io::IO, vdr::AbstractVDR) +function Base.show(io::IO, vdr::VDR) print(io, "VDR: ", CDFDataType(vdr.data_type)) is_nrv(vdr) && print(io, " [NRV]") is_compressed(vdr) && print(io, " [compressed]") @@ -116,7 +88,7 @@ end # 1 Whether or not a pad value is specified for this variable. Set indicates that a pad value has been specified. Clear indicates that a pad value has not been specified. The PadValue field described below is only present if a pad value has been specified. # 2 Whether or not a compression method might be applied to this variable data. Set indicates that a compression is chosen by the user and the data might be compressed, depending on the data size and content. If the compressed data becomes larger than its uncompressed data, no compression is applied and the data are stored as uncompressed, even the compression bit is set. The compressed data is stored in Compressed Variable Value Record (CVVR) while uncompressed data go into Variable Value Record (VVR). Clear indicates that a compression will not be used. The CPRorSPRoffset field provides the offset of the Compressed Parameters Record if this compression bit is set and the compression used. -function read_vvrs(vdr::AbstractVDR{FieldSizeT}, cdf) where {FieldSizeT} +function read_vvrs(vdr::VDR{FieldSizeT}, cdf) where {FieldSizeT} entries = Vector{VVREntry}() sizehint!(entries, 1) collect_vxr_entries!(entries, parent(cdf), Int(vdr.vxr_head), FieldSizeT) @@ -126,4 +98,4 @@ end is_record_varying(vdr) = !is_nrv(vdr) """Whether or not the variable is a non-record variable""" is_nrv(vdr) = (vdr.flags & 0x01) == 0 -is_compressed(vdr::AbstractVDR) = (vdr.flags & 0x04) != 0 +is_compressed(vdr::VDR) = (vdr.flags & 0x04) != 0 diff --git a/src/variable.jl b/src/variable.jl index 6c4f2e5..eda1227 100644 --- a/src/variable.jl +++ b/src/variable.jl @@ -64,7 +64,7 @@ end Base.getindex(var::CDFVariable, name::String) = var.attrib[name] Base.haskey(var::CDFVariable, name::String) = haskey(var.attrib, name) -attrib(var::CDFVariable) = LazyVAttrib(var.parentdataset, var.vdr.num, var.vdr isa VDR) +attrib(var::CDFVariable) = LazyVAttrib(var.parentdataset, var.vdr.num, var.vdr.zvar) attrib(var::CDFVariable, name::String) = get(attrib(var), name) is_record_varying(v::CDFVariable) = is_record_varying(v.vdr) diff --git a/test/runtests.jl b/test/runtests.jl index 403d2c6..bf563b8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -152,7 +152,7 @@ end file = data_path("ac_h0_mfi_20230102_v07.cdf") ds = CDFDataset(file) var = ds["BGSEc"] - @test var.vdr isa CommonDataFormat.rVDR + @test !var.vdr.zvar @test size(var) == (3, 5400) end @@ -160,6 +160,6 @@ end file = download_test_data("https://github.com/JuliaSpacePhysics/CommonDataFormat.jl/releases/download/v0.1.8/omni2_h0_mrg1hr_20150101_v01.cdf") ds = CDFDataset(file) var = ds["Epoch"] - @test var.vdr isa CommonDataFormat.rVDR + @test !var.vdr.zvar @test size(var) == (4344,) end From 47dea4f63b19c90c49fcd2d855b34199097c88c9 Mon Sep 17 00:00:00 2001 From: Beforerr Date: Tue, 6 Oct 2026 07:01:18 -0600 Subject: [PATCH 2/3] fix: drop NOVARY dimensions of z-variables NOVARY dimensions aren't physically stored, so counting them read past each variable's data. r-variables already dropped them; z-variables now share that path. Matches CDFpp and cdflib. --- data/make_novary_z.py | 16 ++++++++++++++ data/novary_z.cdf | Bin 0 -> 1054 bytes src/records/vdr.jl | 50 +++++++++++++++++++++++------------------- test/runtests.jl | 5 +++++ 4 files changed, 49 insertions(+), 22 deletions(-) create mode 100644 data/make_novary_z.py create mode 100644 data/novary_z.cdf diff --git a/data/make_novary_z.py b/data/make_novary_z.py new file mode 100644 index 0000000..7fb4d7c --- /dev/null +++ b/data/make_novary_z.py @@ -0,0 +1,16 @@ +# z-var "novary": declared (2,3,4) with dim 2 NOVARY, so each record stores 2x4 values. +# cdflib/CDFpp writers ignore Dim_Vary, so write (2,1,4) and patch the zVDR's dim 2 to size 3, NOVARY. +import sys, struct, numpy as np +from cdflib.cdfwrite import CDF +out = sys.argv[1] +c = CDF(out, cdf_spec={"Majority": "row_major"}, delete=True) +data = np.arange(5 * 2 * 4, dtype=np.int32).reshape(5, 2, 1, 4) +c.write_var({"Variable": "novary", "Data_Type": 4, "Num_Elements": 1, "Rec_Vary": True, + "Var_Type": "zVariable", "Dim_Sizes": [2, 1, 4]}, var_attrs={}, var_data=data) +c.close() +b = bytearray(open(out, "rb").read()) +p = b.find(b"novary".ljust(256, b"\0")) + 256 # zNumDims follows the 256-byte name +assert b.count(b"novary") == 1 and struct.unpack(">i", b[p:p + 4])[0] == 3 +b[p + 8:p + 12] = struct.pack(">i", 3) # zDimSizes[2] +b[p + 20:p + 24] = struct.pack(">i", 0) # DimVarys[2] +open(out, "wb").write(b) diff --git a/data/novary_z.cdf b/data/novary_z.cdf new file mode 100644 index 0000000000000000000000000000000000000000..4ca4cccee0f2c2e61ef85f8a2f79b13aeb2242d5 GIT binary patch literal 1054 zcmX^6nSqgk;s1XIFkrL*Qb0ZkFggGUW+3JSVm2rnNWlTqe<0v;&d<%w&r@(oEJ;*w z%P-1JEK$&Kc5&0>$|xx*D7MnqPfkhGOD|4K*2_yQPSi`!FXIX>NK8&u2*{``&P*P$?|%*iaNP9)bj!(1&nQuFPR%PxE#h)7N=?m6O3f+JQSePH zs?15uOHnW~FgG_;2n}}R(g-b1Em9~hEhxw@DzQ=k*{ch5fG$+TAt$rAq!{iJO)j`+ zC}e~H^)sQG!MFjH&msrogTf8UhK4nS$7p=C6iH47D9R1QtVn7>@c;yjB|w4$%!Cju zKnjF`fO!%U%?@FMVi*E|d?Y&=r+|#f%P#|lBuwRqq(SK%6n`L^1r`S&8W>m&Krs-= z17cn<3qpj+cXKcZeLRxI@SpcEw@_mM1H%Dz&0i}5oRqx2?_YR#Ca+>k@5JcjaS8MU3 literal 0 HcmV?d00001 diff --git a/src/records/vdr.jl b/src/records/vdr.jl index 5f29895..81f40f9 100644 --- a/src/records/vdr.jl +++ b/src/records/vdr.jl @@ -44,35 +44,41 @@ Load a z- or r-Variable Descriptor Record from the buffer at the specified offse return VDR{FieldSizeT}(fields..., true, z_num_dims, pos) end -function record_sizes(vdr::VDR, cdf, ::Val{M}) where {M} - vdr.zvar || return _rvar_record_sizes(vdr, cdf, Val(M)) - vdr.num_dims == M || - throw(DimensionMismatch("variable has $(vdr.num_dims) dimensions, expected $M")) - return read_be(parent(cdf), vdr.pos, Val(M), Int32) +# (declared dim count, position of their sizes, position of their DimVarys) +function _dim_layout(vdr::VDR, cdf) + vdr.zvar && return Int(vdr.num_dims), vdr.pos, vdr.pos + 4 * Int(vdr.num_dims) + gdr = GDR(cdf) + return Int(gdr.r_num_dims), gdr.pos + sizeof(Int64) + 3 * sizeof(Int32), vdr.pos end -function _rvar_record_sizes(vdr::VDR, cdf, ::Val{M}) where {M} - gdr = GDR(cdf) - buf = parent(cdf) - sizes_pos = gdr.pos + sizeof(Int64) + 3 * sizeof(Int32) - sizes = zeros(Int32, M) - count = 0 - for i in 1:Int(gdr.r_num_dims) - read_be(buf, vdr.pos + (i - 1) * 4, Int32) == 0 && continue - count += 1 - count <= M && (sizes[count] = read_be(buf, sizes_pos + (i - 1) * 4, Int32)) +_varies(buf, varys_pos, i) = read_be(buf, varys_pos + (i - 1) * 4, Int32) != 0 + +# Declared index of the `j`-th varying dimension +function _varying_dim(buf, varys_pos, j) + i = 0 + while j > 0 + i += 1 + j -= _varies(buf, varys_pos, i) end - count == M || throw(DimensionMismatch("variable has $count dimensions, expected $M")) - return ntuple(i -> sizes[i], Val(M)) + return i +end + +# NOVARY dimensions aren't physically stored, so they aren't array dimensions either. +# `M` must be `num_record_dims(vdr, cdf)`. +function record_sizes(vdr::VDR, cdf, ::Val{M}) where {M} + n, sizes_pos, varys_pos = _dim_layout(vdr, cdf) + buf = parent(cdf) + n == M && return read_be(buf, sizes_pos, Val(M), Int32) + return ntuple(j -> read_be(buf, sizes_pos + (_varying_dim(buf, varys_pos, j) - 1) * 4, Int32), Val(M)) end function num_record_dims(vdr::VDR, cdf) - vdr.zvar && return Int(vdr.num_dims) - n = 0 - for i in 1:Int(GDR(cdf).r_num_dims) - n += read_be(parent(cdf), vdr.pos + (i - 1) * 4, Int32) != 0 + n, _, varys_pos = _dim_layout(vdr, cdf) + count = 0 + for i in 1:n + count += _varies(parent(cdf), varys_pos, i) end - return n + return count end diff --git a/test/runtests.jl b/test/runtests.jl index bf563b8..a784074 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -120,6 +120,11 @@ end @test occursin("compressed", string(var.vdr)) end +@testset "NOVARY z-variable dimension" begin + v = CDFDataset(data_path("novary_z.cdf"))["novary"] # row-major, declared (2,3,4), dim 2 NOVARY + @test Array(v) == permutedims(reshape(Int32(0):Int32(39), 4, 2, 5), (2, 1, 3)) +end + @testset "CHECK_VARIABLES - Variable structure verification" begin file = data_path("omni_coho1hr_merged_mag_plasma_20240901_v01.cdf") ds = CDFDataset(file) From 8c11b7509c733c33edceccf9b6a95f3e34765616 Mon Sep 17 00:00:00 2001 From: Beforerr Date: Tue, 6 Oct 2026 07:32:23 -0600 Subject: [PATCH 3/3] perf: compare variable and attribute names in place Lookups by name scanned each record's name field for its terminator and compared a view element-wise; attribute lookup also parsed every ADR first. Compare with one memcmp in the buffer instead, and read an ADR's name only when it is returned. Name fields are 64 bytes in CDF v2, not 256, so a v2 name filling its field was read into the following bytes. --- src/dataset.jl | 6 +++--- src/loading/attribute.jl | 18 ++++++++---------- src/parsing.jl | 22 ++++++++++++++++------ src/records/adr.jl | 12 +++++++----- 4 files changed, 34 insertions(+), 24 deletions(-) diff --git a/src/dataset.jl b/src/dataset.jl index 8f06d41..240dbfc 100644 --- a/src/dataset.jl +++ b/src/dataset.jl @@ -71,15 +71,15 @@ end _vdr_at(cdf::CDFDataset{FST}, offset::Int) where {FST} = VDR{FST}(parent(cdf), offset) # Name is the first variable-width field of a VDR; everything before it is fixed size. -vdr_name(buffer, offset, ::Type{FST}) where {FST} = readname(buffer, offset + 45 + 5 * sizeof(FST)) +_vdr_name_pos(offset, ::Type{FST}) where {FST} = offset + 45 + 5 * sizeof(FST) +vdr_name(buffer, offset, ::Type{FST}) where {FST} = readname(buffer, _vdr_name_pos(offset, FST), FST) # r-variables are chained first, then z-variables vdr_heads(cdf::CDFDataset) = (GDR(cdf).rVDRhead, GDR(cdf).zVDRhead) function find_vdr(cdf::CDFDataset{FST}, var_name::String) where {FST} buffer = parent(cdf) - var_name_bytes = codeunits(var_name) for head in vdr_heads(cdf), offset in OffsetsIterator{FST}(buffer, head) - vdr_name(buffer, offset, FST) == var_name_bytes && return _vdr_at(cdf, offset) + name_equals(buffer, _vdr_name_pos(offset, FST), var_name, FST) && return _vdr_at(cdf, offset) end return nothing end diff --git a/src/loading/attribute.jl b/src/loading/attribute.jl index 96b7539..ac534a9 100644 --- a/src/loading/attribute.jl +++ b/src/loading/attribute.jl @@ -24,7 +24,7 @@ function attrib(cdf::CDFDataset; predicate = is_global) for offset in OffsetsIterator(cdf) adr = ADR{RecordSizeType}(buffer, offset) predicate(adr) || continue - result[String(adr.Name)] = load_attribute_entries(buffer, adr, RecordSizeType, needs_byte_swap) + result[adr_name(buffer, adr)] = load_attribute_entries(buffer, adr, RecordSizeType, needs_byte_swap) end return result end @@ -37,11 +37,9 @@ Retrieve all entries for a named attribute from the CDF file. function attrib(cdf::CDFDataset{FST}, name::String) where {FST} buffer = cdf.buffer needs_byte_swap = is_big_endian_encoding(cdf) - offsets = OffsetsIterator(cdf) - name_bytes = codeunits(name) - for offset in offsets - adr = ADR{FST}(buffer, offset) - name_bytes == adr.Name && return load_attribute_entries(buffer, adr, FST, needs_byte_swap) + for offset in OffsetsIterator(cdf) + name_equals(buffer, _adr_name_pos(offset, FST), name, FST) && + return load_attribute_entries(buffer, ADR{FST}(buffer, offset), FST, needs_byte_swap) end error("Attribute '$name' not found in CDF file") end @@ -71,7 +69,7 @@ function Base.iterate(la::LazyVAttrib, offset::Int = Int(la.cdf.gdr.ADRhead)) next_offset = Int(adr.ADRnext) found = _search_aedr_entries(buffer, _entry_head(la, adr), RecordSizeType, needs_byte_swap, la.varnum) if !isnothing(found) - name = String(adr.Name) + name = adr_name(buffer, adr) return (name => _get_attributes(name, found, la.cdf), next_offset) end offset = next_offset @@ -93,12 +91,12 @@ function Base.get(la::LazyVAttrib, name::AbstractString, default = nothing) varnum = la.varnum RecordSizeType = recordsize_type(cdf) buffer = cdf.buffer - name_bytes = codeunits(name) + name_str = String(name) needs_byte_swap = is_big_endian_encoding(cdf) for offset in OffsetsIterator(cdf) is_global(buffer, offset, RecordSizeType) && continue + name_equals(buffer, _adr_name_pos(offset, RecordSizeType), name_str, RecordSizeType) || continue adr = ADR{RecordSizeType}(buffer, offset) - adr.Name != name_bytes && continue found = _search_aedr_entries(buffer, _entry_head(la, adr), RecordSizeType, needs_byte_swap, varnum) return isnothing(found) ? default : _get_attributes(name, found, cdf) end @@ -144,7 +142,7 @@ function attribnames(cdf::CDFDataset{FST}; predicate = is_global) where {FST} buffer = cdf.buffer for offset in OffsetsIterator(cdf) adr = ADR{FST}(buffer, offset) - predicate(adr) && push!(names, String(adr.Name)) + predicate(adr) && push!(names, adr_name(buffer, adr)) end return names end diff --git a/src/parsing.jl b/src/parsing.jl index 3453843..7bfa1ae 100644 --- a/src/parsing.jl +++ b/src/parsing.jl @@ -59,13 +59,23 @@ end @inline write_be(v::Vector{UInt8}, i, ::RInt32) = i + _sizeof(RInt32) -function readname(buf::Vector{UInt8}, offset::Int) - for i in offset:(offset + 255) - if buf[i] == 0x00 - return @views buf[offset:(i - 1)] - end +# Name fields are NUL-padded to a fixed width: 256 bytes in CDF v3, 64 in v2. +_name_width(::Type{FST}) where {FST} = FST == Int64 ? 256 : 64 + +function readname(buf::Vector{UInt8}, offset::Int, ::Type{FST}) where {FST} + last = offset + _name_width(FST) - 1 + for i in offset:last + buf[i] == 0x00 && return @views buf[offset:(i - 1)] end - return @views buf[offset:(offset + 255)] + return @views buf[offset:last] +end + +function name_equals(buf::Vector{UInt8}, offset::Int, name::String, ::Type{FST}) where {FST} + n = ncodeunits(name) + w = _name_width(FST) + n <= w || return false + (n == w || buf[offset + n] == 0x00) || return false + return GC.@preserve buf name ccall(:memcmp, Cint, (Ptr{UInt8}, Ptr{UInt8}, Csize_t), pointer(buf, offset), pointer(name), n) == 0 end is_cdf_v3(magic_bytes) = magic_bytes == 0xCDF30001 diff --git a/src/records/adr.jl b/src/records/adr.jl index ca4b9ce..16b615e 100644 --- a/src/records/adr.jl +++ b/src/records/adr.jl @@ -3,7 +3,7 @@ Attribute Descriptor Record (ADR) Contains a description of an attribute in a CDF. There will be one ADR per attribute. The ADRhead field of the ADR contains the file offset of the first ADR. """ -struct ADR{FSZ, S} <: Record +struct ADR{FSZ} <: Record # header::Header ADRnext::FSZ # Offset to next ADR in chain AgrEDRhead::FSZ # The offset of the first Attribute g/rEntry Descriptor Record (AgrEDR) for this attribute. @@ -16,18 +16,20 @@ struct ADR{FSZ, S} <: Record NzEntries::Int32 # Number of z-variables MAXzEntry::Int32 # Number of z-entries rfuE::RInt32 # Reserved field E - Name::S + name_pos::Int # Name is read on demand (`adr_name`); lookups only compare it in place end is_global(adr) = adr.Scope == 1 # Scope, read without parsing the rest of the ADR: header + ADRnext + AgrEDRhead is_global(buffer, offset, ::Type{FST}) where {FST} = read_be(buffer, offset + 3 * sizeof(FST) + 5, Int32) == 1 +_adr_name_pos(offset, ::Type{FST}) where {FST} = offset + 37 + 4 * sizeof(FST) @inline function ADR{FST}(buffer::Vector{UInt8}, offset) where {FST} pos = check_record_type(4, buffer, offset, FST) # Read ADR fields - fields, pos = read_be_fields(buffer, pos, ADR{FST, String}, Val(1:11)) - name = readname(buffer, pos) - return ADR(fields..., name) + fields, pos = read_be_fields(buffer, pos, ADR{FST}, Val(1:11)) + return ADR{FST}(fields..., pos) end + +adr_name(buffer, adr::ADR{FST}) where {FST} = String(readname(buffer, adr.name_pos, FST))