diff --git a/data/make_novary_z.py b/data/make_novary_z.py new file mode 100644 index 0000000..7fb4d7c --- /dev/null +++ b/data/make_novary_z.py @@ -0,0 +1,16 @@ +# z-var "novary": declared (2,3,4) with dim 2 NOVARY, so each record stores 2x4 values. +# cdflib/CDFpp writers ignore Dim_Vary, so write (2,1,4) and patch the zVDR's dim 2 to size 3, NOVARY. +import sys, struct, numpy as np +from cdflib.cdfwrite import CDF +out = sys.argv[1] +c = CDF(out, cdf_spec={"Majority": "row_major"}, delete=True) +data = np.arange(5 * 2 * 4, dtype=np.int32).reshape(5, 2, 1, 4) +c.write_var({"Variable": "novary", "Data_Type": 4, "Num_Elements": 1, "Rec_Vary": True, + "Var_Type": "zVariable", "Dim_Sizes": [2, 1, 4]}, var_attrs={}, var_data=data) +c.close() +b = bytearray(open(out, "rb").read()) +p = b.find(b"novary".ljust(256, b"\0")) + 256 # zNumDims follows the 256-byte name +assert b.count(b"novary") == 1 and struct.unpack(">i", b[p:p + 4])[0] == 3 +b[p + 8:p + 12] = struct.pack(">i", 3) # zDimSizes[2] +b[p + 20:p + 24] = struct.pack(">i", 0) # DimVarys[2] +open(out, "wb").write(b) diff --git a/data/novary_z.cdf b/data/novary_z.cdf new file mode 100644 index 0000000..4ca4ccc Binary files /dev/null and b/data/novary_z.cdf differ diff --git a/src/dataset.jl b/src/dataset.jl index 543f832..240dbfc 100644 --- a/src/dataset.jl +++ b/src/dataset.jl @@ -68,25 +68,18 @@ majority(cdf::CDFDataset) = majority(cdf.cdr) throw(ArgumentError("Unknown property $name")) end -# Load the (z or r) VDR at a known offset -function _vdr_at(cdf::CDFDataset{FST}, offset::Int) where {FST} - buffer = parent(cdf) - record_type = read_be(buffer, offset + 1 + sizeof(FST), Int32) - @assert record_type in (8, 3) - return record_type == 8 ? VDR{FST}(buffer, offset) : - rVDR{FST}(buffer, offset) -end +_vdr_at(cdf::CDFDataset{FST}, offset::Int) where {FST} = VDR{FST}(parent(cdf), offset) # Name is the first variable-width field of a VDR; everything before it is fixed size. -vdr_name(buffer, offset, ::Type{FST}) where {FST} = readname(buffer, offset + 45 + 5 * sizeof(FST)) +_vdr_name_pos(offset, ::Type{FST}) where {FST} = offset + 45 + 5 * sizeof(FST) +vdr_name(buffer, offset, ::Type{FST}) where {FST} = readname(buffer, _vdr_name_pos(offset, FST), FST) # r-variables are chained first, then z-variables vdr_heads(cdf::CDFDataset) = (GDR(cdf).rVDRhead, GDR(cdf).zVDRhead) function find_vdr(cdf::CDFDataset{FST}, var_name::String) where {FST} buffer = parent(cdf) - var_name_bytes = codeunits(var_name) for head in vdr_heads(cdf), offset in OffsetsIterator{FST}(buffer, head) - vdr_name(buffer, offset, FST) == var_name_bytes && return _vdr_at(cdf, offset) + name_equals(buffer, _vdr_name_pos(offset, FST), var_name, FST) && return _vdr_at(cdf, offset) end return nothing end diff --git a/src/loading/attribute.jl b/src/loading/attribute.jl index 4ebdc13..ac534a9 100644 --- a/src/loading/attribute.jl +++ b/src/loading/attribute.jl @@ -24,7 +24,7 @@ function attrib(cdf::CDFDataset; predicate = is_global) for offset in OffsetsIterator(cdf) adr = ADR{RecordSizeType}(buffer, offset) predicate(adr) || continue - result[String(adr.Name)] = load_attribute_entries(buffer, adr, RecordSizeType, needs_byte_swap) + result[adr_name(buffer, adr)] = load_attribute_entries(buffer, adr, RecordSizeType, needs_byte_swap) end return result end @@ -37,11 +37,9 @@ Retrieve all entries for a named attribute from the CDF file. function attrib(cdf::CDFDataset{FST}, name::String) where {FST} buffer = cdf.buffer needs_byte_swap = is_big_endian_encoding(cdf) - offsets = OffsetsIterator(cdf) - name_bytes = codeunits(name) - for offset in offsets - adr = ADR{FST}(buffer, offset) - name_bytes == adr.Name && return load_attribute_entries(buffer, adr, FST, needs_byte_swap) + for offset in OffsetsIterator(cdf) + name_equals(buffer, _adr_name_pos(offset, FST), name, FST) && + return load_attribute_entries(buffer, ADR{FST}(buffer, offset), FST, needs_byte_swap) end error("Attribute '$name' not found in CDF file") end @@ -71,7 +69,7 @@ function Base.iterate(la::LazyVAttrib, offset::Int = Int(la.cdf.gdr.ADRhead)) next_offset = Int(adr.ADRnext) found = _search_aedr_entries(buffer, _entry_head(la, adr), RecordSizeType, needs_byte_swap, la.varnum) if !isnothing(found) - name = String(adr.Name) + name = adr_name(buffer, adr) return (name => _get_attributes(name, found, la.cdf), next_offset) end offset = next_offset @@ -93,12 +91,12 @@ function Base.get(la::LazyVAttrib, name::AbstractString, default = nothing) varnum = la.varnum RecordSizeType = recordsize_type(cdf) buffer = cdf.buffer - name_bytes = codeunits(name) + name_str = String(name) needs_byte_swap = is_big_endian_encoding(cdf) for offset in OffsetsIterator(cdf) is_global(buffer, offset, RecordSizeType) && continue + name_equals(buffer, _adr_name_pos(offset, RecordSizeType), name_str, RecordSizeType) || continue adr = ADR{RecordSizeType}(buffer, offset) - adr.Name != name_bytes && continue found = _search_aedr_entries(buffer, _entry_head(la, adr), RecordSizeType, needs_byte_swap, varnum) return isnothing(found) ? default : _get_attributes(name, found, cdf) end @@ -114,7 +112,8 @@ Base.haskey(la::LazyVAttrib, name) = false # https://github.com/SciQLop/PyISTP/blob/0a565c39c73dd800934bc379dd7c2e00c28d23d0/pyistp/_impl.py#L16 function _get_attributes(name, value, cdf) if occursin("LABL_PTR", name) - return cdf[value][:] + # `Array` skips DiskArrays indexing + return vec(Array(cdf[value])) end return value end @@ -143,7 +142,7 @@ function attribnames(cdf::CDFDataset{FST}; predicate = is_global) where {FST} buffer = cdf.buffer for offset in OffsetsIterator(cdf) adr = ADR{FST}(buffer, offset) - predicate(adr) && push!(names, String(adr.Name)) + predicate(adr) && push!(names, adr_name(buffer, adr)) end return names end diff --git a/src/loading/variable.jl b/src/loading/variable.jl index 90a0786..1f03c5c 100644 --- a/src/loading/variable.jl +++ b/src/loading/variable.jl @@ -203,7 +203,7 @@ function collect_vxr_entries!(entries::Vector{VVREntry}, src, offset, ::Type{Fie return entries end -function variable_compression(vdr::AbstractVDR{FieldSizeT}, cdf) where {FieldSizeT} +function variable_compression(vdr::VDR{FieldSizeT}, cdf) where {FieldSizeT} offset_value = Int(vdr.cpr_or_spr_offset) if is_compressed(vdr) && offset_value != 0 buffer = parent(cdf) diff --git a/src/parsing.jl b/src/parsing.jl index 3453843..7bfa1ae 100644 --- a/src/parsing.jl +++ b/src/parsing.jl @@ -59,13 +59,23 @@ end @inline write_be(v::Vector{UInt8}, i, ::RInt32) = i + _sizeof(RInt32) -function readname(buf::Vector{UInt8}, offset::Int) - for i in offset:(offset + 255) - if buf[i] == 0x00 - return @views buf[offset:(i - 1)] - end +# Name fields are NUL-padded to a fixed width: 256 bytes in CDF v3, 64 in v2. +_name_width(::Type{FST}) where {FST} = FST == Int64 ? 256 : 64 + +function readname(buf::Vector{UInt8}, offset::Int, ::Type{FST}) where {FST} + last = offset + _name_width(FST) - 1 + for i in offset:last + buf[i] == 0x00 && return @views buf[offset:(i - 1)] end - return @views buf[offset:(offset + 255)] + return @views buf[offset:last] +end + +function name_equals(buf::Vector{UInt8}, offset::Int, name::String, ::Type{FST}) where {FST} + n = ncodeunits(name) + w = _name_width(FST) + n <= w || return false + (n == w || buf[offset + n] == 0x00) || return false + return GC.@preserve buf name ccall(:memcmp, Cint, (Ptr{UInt8}, Ptr{UInt8}, Csize_t), pointer(buf, offset), pointer(name), n) == 0 end is_cdf_v3(magic_bytes) = magic_bytes == 0xCDF30001 diff --git a/src/precompile.jl b/src/precompile.jl index 2ebd38e..367e53d 100644 --- a/src/precompile.jl +++ b/src/precompile.jl @@ -1,11 +1,11 @@ -# Seed one variable per common (eltype, ndims) shape; every new pair costs DiskArrays -# indexing compiles, so rarer shapes (integers, Epoch16, >3-D, extra string widths) are left out. -_workload_key(var) = (eltype(var) <: StaticString ? StaticString : eltype(var), ndims(var)) +# Seed one variable per common (eltype, ndims) shape, per CDF v2/v3 offset width. Every new combination costs DiskArrays indexing compiles, so rarer +# shapes (integers, Epoch16, >3-D, extra string widths) are left out. +_workload_key(var) = (eltype(var) <: StaticString ? StaticString : eltype(var), ndims(var), recordsize_type(var.parentdataset)) _workload_maxdims(T) = T <: AbstractFloat ? 3 : T <: StaticString ? 2 : T in (TT2000, Epoch) ? 1 : 0 PrecompileTools.@setup_workload begin - files = joinpath.(@__DIR__, "../data", ("elb_l2_epdef_20210914_v01.cdf", "a_cdf_with_compressed_vars.cdf")) - seen = Set{Tuple{Any, Int}}() + files = joinpath.(@__DIR__, "../data", ("elb_l2_epdef_20210914_v01.cdf", "a_cdf_with_compressed_vars.cdf", "ac_h2_sis_20101105_v06.cdf")) + seen = Set{Tuple{Any, Int, DataType}}() PrecompileTools.@compile_workload begin for (i, file) in enumerate(files) @@ -15,7 +15,7 @@ PrecompileTools.@setup_workload begin key = _workload_key(var) (isempty(var) || key[2] > _workload_maxdims(key[1]) || key in seen) && continue push!(seen, key) - get(var.attrib, "FILLVAL", nothing) + Dict(var.attrib) Array(var) var[1] var[ntuple(_ -> Colon(), ndims(var) - 1)..., 1:1] diff --git a/src/records/adr.jl b/src/records/adr.jl index ca4b9ce..16b615e 100644 --- a/src/records/adr.jl +++ b/src/records/adr.jl @@ -3,7 +3,7 @@ Attribute Descriptor Record (ADR) Contains a description of an attribute in a CDF. There will be one ADR per attribute. The ADRhead field of the ADR contains the file offset of the first ADR. """ -struct ADR{FSZ, S} <: Record +struct ADR{FSZ} <: Record # header::Header ADRnext::FSZ # Offset to next ADR in chain AgrEDRhead::FSZ # The offset of the first Attribute g/rEntry Descriptor Record (AgrEDR) for this attribute. @@ -16,18 +16,20 @@ struct ADR{FSZ, S} <: Record NzEntries::Int32 # Number of z-variables MAXzEntry::Int32 # Number of z-entries rfuE::RInt32 # Reserved field E - Name::S + name_pos::Int # Name is read on demand (`adr_name`); lookups only compare it in place end is_global(adr) = adr.Scope == 1 # Scope, read without parsing the rest of the ADR: header + ADRnext + AgrEDRhead is_global(buffer, offset, ::Type{FST}) where {FST} = read_be(buffer, offset + 3 * sizeof(FST) + 5, Int32) == 1 +_adr_name_pos(offset, ::Type{FST}) where {FST} = offset + 37 + 4 * sizeof(FST) @inline function ADR{FST}(buffer::Vector{UInt8}, offset) where {FST} pos = check_record_type(4, buffer, offset, FST) # Read ADR fields - fields, pos = read_be_fields(buffer, pos, ADR{FST, String}, Val(1:11)) - name = readname(buffer, pos) - return ADR(fields..., name) + fields, pos = read_be_fields(buffer, pos, ADR{FST}, Val(1:11)) + return ADR{FST}(fields..., pos) end + +adr_name(buffer, adr::ADR{FST}) where {FST} = String(readname(buffer, adr.name_pos, FST)) diff --git a/src/records/vdr.jl b/src/records/vdr.jl index 09d7ce7..81f40f9 100644 --- a/src/records/vdr.jl +++ b/src/records/vdr.jl @@ -1,31 +1,9 @@ # VDR loading functionality -abstract type AbstractVDR{FST} <: Record end - -struct rVDR{FST} <: AbstractVDR{FST} - vdr_next::FST # Offset to next VDR in chain - data_type::Int32 # CDF data type - max_rec::Int32 # Maximum record number (-1 if none) - vxr_head::FST # Variable indeX Record head - vxr_tail::FST # Variable indeX Record tail - flags::Int32 # Variable flags - s_records::Int32 # Sparse records flag - rfu_b::RInt32 # Reserved field B - rfu_c::RInt32 # Reserved field C - rfu_f::RInt32 # Reserved field F - num_elems::Int32 # Number of elements (for strings) - num::Int32 # Variable number - cpr_or_spr_offset::FST # Compression/Sparseness Parameters Record offset - # blocking_factor::Int32 - # name::S # Variable name - pos::Int -end - """ -Variable Descriptor Record (VDR) - describes a single variable +Variable Descriptor Record (VDR) - describes a single r- or z-variable. """ -struct VDR{FST} <: AbstractVDR{FST} - # header::Header +struct VDR{FST} <: Record vdr_next::FST # Offset to next VDR in chain data_type::Int32 # CDF data type max_rec::Int32 # Maximum record number (-1 if none) @@ -37,14 +15,13 @@ struct VDR{FST} <: AbstractVDR{FST} rfu_c::RInt32 # Reserved field C rfu_f::RInt32 # Reserved field F num_elems::Int32 # Number of elements (for strings) - num::Int32 # Variable number + num::Int32 # Variable number (r- and z-variables are numbered independently) cpr_or_spr_offset::FST # Compression/Sparseness Parameters Record offset # blocking_factor::Int32 # name::S # Variable name - num_dims::Int32 # Number of dimensions - pos::Int - # z_dim_sizes::Tuple{Vararg{Int32}} # Dimension sizes (z-variables only) - # dim_varys::Tuple{Vararg{Int32}} # Dimension variance flags + zvar::Bool + num_dims::Int32 # zNumDims; unused for r-variables, whose dims come from the GDR + pos::Int # zDimSizes (z) / DimVarys (r) end # zNumDims (zVDRs) / DimVarys (rVDRs) start right after the fixed-width name field. @@ -53,58 +30,59 @@ _after_name(offset, ::Type{FieldSizeT}) where {FieldSizeT} = FieldSizeT == Int64 """ VDR{FieldSizeT}(buffer, offset) -Load a z-Variable Descriptor Record from the buffer at the specified offset. +Load a z- or r-Variable Descriptor Record from the buffer at the specified offset. """ @inline function VDR{FieldSizeT}(buffer::Vector{UInt8}, offset) where {FieldSizeT} - pos = check_record_type(8, buffer, offset, FieldSizeT) + record_type = get_record_type(buffer, offset, FieldSizeT) + @assert record_type in (8, 3) + zvar = record_type == 8 + pos = offset + sizeof(FieldSizeT) + 5 fields, _ = read_be_fields(buffer, pos, VDR{FieldSizeT}, Val(1:13)) - z_num_dims, pos = read_be_i(buffer, _after_name(offset, FieldSizeT), Int32) - return VDR{FieldSizeT}(fields..., z_num_dims, pos) + after_name = _after_name(offset, FieldSizeT) + zvar || return VDR{FieldSizeT}(fields..., false, Int32(0), after_name) + z_num_dims, pos = read_be_i(buffer, after_name, Int32) + return VDR{FieldSizeT}(fields..., true, z_num_dims, pos) end -""" - rVDR{FieldSizeT}(buffer, offset) - -Load an r-Variable Descriptor Record from the buffer at the specified offset. -""" -@inline function rVDR{FieldSizeT}(buffer::Vector{UInt8}, offset) where {FieldSizeT} - pos = check_record_type(3, buffer, offset, FieldSizeT) - fields, _ = read_be_fields(buffer, pos, rVDR{FieldSizeT}, Val(1:13)) - return rVDR{FieldSizeT}(fields..., _after_name(offset, FieldSizeT)) +# (declared dim count, position of their sizes, position of their DimVarys) +function _dim_layout(vdr::VDR, cdf) + vdr.zvar && return Int(vdr.num_dims), vdr.pos, vdr.pos + 4 * Int(vdr.num_dims) + gdr = GDR(cdf) + return Int(gdr.r_num_dims), gdr.pos + sizeof(Int64) + 3 * sizeof(Int32), vdr.pos end -function record_sizes(vdr::VDR, cdf, ::Val{M}) where {M} - vdr.num_dims == M || - throw(DimensionMismatch("variable has $(vdr.num_dims) dimensions, expected $M")) - return read_be(parent(cdf), vdr.pos, Val(M), Int32) +_varies(buf, varys_pos, i) = read_be(buf, varys_pos + (i - 1) * 4, Int32) != 0 + +# Declared index of the `j`-th varying dimension +function _varying_dim(buf, varys_pos, j) + i = 0 + while j > 0 + i += 1 + j -= _varies(buf, varys_pos, i) + end + return i end -function record_sizes(vdr::rVDR, cdf, ::Val{M}) where {M} - gdr = GDR(cdf) +# NOVARY dimensions aren't physically stored, so they aren't array dimensions either. +# `M` must be `num_record_dims(vdr, cdf)`. +function record_sizes(vdr::VDR, cdf, ::Val{M}) where {M} + n, sizes_pos, varys_pos = _dim_layout(vdr, cdf) buf = parent(cdf) - sizes_pos = gdr.pos + sizeof(Int64) + 3 * sizeof(Int32) - sizes = zeros(Int32, M) - count = 0 - for i in 1:Int(gdr.r_num_dims) - read_be(buf, vdr.pos + (i - 1) * 4, Int32) == 0 && continue - count += 1 - count <= M && (sizes[count] = read_be(buf, sizes_pos + (i - 1) * 4, Int32)) - end - count == M || throw(DimensionMismatch("variable has $count dimensions, expected $M")) - return ntuple(i -> sizes[i], Val(M)) + n == M && return read_be(buf, sizes_pos, Val(M), Int32) + return ntuple(j -> read_be(buf, sizes_pos + (_varying_dim(buf, varys_pos, j) - 1) * 4, Int32), Val(M)) end -num_record_dims(vdr::VDR, cdf) = Int(vdr.num_dims) -function num_record_dims(vdr::rVDR, cdf) - n = 0 - for i in 1:Int(GDR(cdf).r_num_dims) - n += read_be(parent(cdf), vdr.pos + (i - 1) * 4, Int32) != 0 +function num_record_dims(vdr::VDR, cdf) + n, _, varys_pos = _dim_layout(vdr, cdf) + count = 0 + for i in 1:n + count += _varies(parent(cdf), varys_pos, i) end - return n + return count end -function Base.show(io::IO, vdr::AbstractVDR) +function Base.show(io::IO, vdr::VDR) print(io, "VDR: ", CDFDataType(vdr.data_type)) is_nrv(vdr) && print(io, " [NRV]") is_compressed(vdr) && print(io, " [compressed]") @@ -116,7 +94,7 @@ end # 1 Whether or not a pad value is specified for this variable. Set indicates that a pad value has been specified. Clear indicates that a pad value has not been specified. The PadValue field described below is only present if a pad value has been specified. # 2 Whether or not a compression method might be applied to this variable data. Set indicates that a compression is chosen by the user and the data might be compressed, depending on the data size and content. If the compressed data becomes larger than its uncompressed data, no compression is applied and the data are stored as uncompressed, even the compression bit is set. The compressed data is stored in Compressed Variable Value Record (CVVR) while uncompressed data go into Variable Value Record (VVR). Clear indicates that a compression will not be used. The CPRorSPRoffset field provides the offset of the Compressed Parameters Record if this compression bit is set and the compression used. -function read_vvrs(vdr::AbstractVDR{FieldSizeT}, cdf) where {FieldSizeT} +function read_vvrs(vdr::VDR{FieldSizeT}, cdf) where {FieldSizeT} entries = Vector{VVREntry}() sizehint!(entries, 1) collect_vxr_entries!(entries, parent(cdf), Int(vdr.vxr_head), FieldSizeT) @@ -126,4 +104,4 @@ end is_record_varying(vdr) = !is_nrv(vdr) """Whether or not the variable is a non-record variable""" is_nrv(vdr) = (vdr.flags & 0x01) == 0 -is_compressed(vdr::AbstractVDR) = (vdr.flags & 0x04) != 0 +is_compressed(vdr::VDR) = (vdr.flags & 0x04) != 0 diff --git a/src/variable.jl b/src/variable.jl index 6c4f2e5..eda1227 100644 --- a/src/variable.jl +++ b/src/variable.jl @@ -64,7 +64,7 @@ end Base.getindex(var::CDFVariable, name::String) = var.attrib[name] Base.haskey(var::CDFVariable, name::String) = haskey(var.attrib, name) -attrib(var::CDFVariable) = LazyVAttrib(var.parentdataset, var.vdr.num, var.vdr isa VDR) +attrib(var::CDFVariable) = LazyVAttrib(var.parentdataset, var.vdr.num, var.vdr.zvar) attrib(var::CDFVariable, name::String) = get(attrib(var), name) is_record_varying(v::CDFVariable) = is_record_varying(v.vdr) diff --git a/test/runtests.jl b/test/runtests.jl index 403d2c6..a784074 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -120,6 +120,11 @@ end @test occursin("compressed", string(var.vdr)) end +@testset "NOVARY z-variable dimension" begin + v = CDFDataset(data_path("novary_z.cdf"))["novary"] # row-major, declared (2,3,4), dim 2 NOVARY + @test Array(v) == permutedims(reshape(Int32(0):Int32(39), 4, 2, 5), (2, 1, 3)) +end + @testset "CHECK_VARIABLES - Variable structure verification" begin file = data_path("omni_coho1hr_merged_mag_plasma_20240901_v01.cdf") ds = CDFDataset(file) @@ -152,7 +157,7 @@ end file = data_path("ac_h0_mfi_20230102_v07.cdf") ds = CDFDataset(file) var = ds["BGSEc"] - @test var.vdr isa CommonDataFormat.rVDR + @test !var.vdr.zvar @test size(var) == (3, 5400) end @@ -160,6 +165,6 @@ end file = download_test_data("https://github.com/JuliaSpacePhysics/CommonDataFormat.jl/releases/download/v0.1.8/omni2_h0_mrg1hr_20150101_v01.cdf") ds = CDFDataset(file) var = ds["Epoch"] - @test var.vdr isa CommonDataFormat.rVDR + @test !var.vdr.zvar @test size(var) == (4344,) end