refactor: consolidate record header access, drop dead code

.github/workflows/CI.yml

@@ -13,7 +13,7 @@ concurrency:
   cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
 jobs:
   test:
-    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }}
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     permissions: # needed to allow julia-actions/cache to proactively delete old caches that it has created
@@ -25,17 +25,13 @@ jobs:
         version:
           - '1'
           - 'lts'
-          - 'pre'
         os:
           - ubuntu-latest
-        arch:
-          - x64
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: julia-actions/setup-julia@v3
         with:
           version: ${{ matrix.version }}
-          arch: ${{ matrix.arch }}
       - uses: julia-actions/cache@v3
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1

ext/CommonDataFormatCommonDataModelExt.jl

@@ -15,10 +15,7 @@ const SymbolOrString = Union{Symbol, AbstractString}
 CDM.path(ds::CDFDataset) = CDF.filename(ds)
 CDM.varnames(ds::CDFDataset) = keys(ds)
 
-function CDM.variable(ds::CDFDataset, name::SymbolOrString)
-    return CDF.variable(ds, String(name))
-end
-
+CDM.variable(ds::CDFDataset, name::SymbolOrString) = ds[String(name)]
 CDM.attribnames(ds::CDFDataset) = CDF.attribnames(ds)
 CDM.attrib(ds::CDFDataset, args...) = CDF.attrib(ds, args...)
 
@@ -31,15 +28,8 @@ CDM.attrib(var::CDFVariable, args...) = CDF.attrib(var, args...)
 function CDM.dimnames(var::CDFVariable, i)
     N = ndims(var)
     @assert i <= N
-    key = if i == N
-        "DEPEND_0"
-    elseif i == 1
-        "DEPEND_1"
-    elseif i == 2
-        "DEPEND_2"
-    else
-        "DEPEND_$i"
-    end
+    # The record dimension is last in Julia but is DEPEND_0 in the ISTP conventions.
+    key = i == N ? "DEPEND_0" : "DEPEND_$i"
     return CDF.attrib(var, key)::Union{String, Nothing}
 end
 

src/CommonDataFormat.jl

@@ -14,6 +14,27 @@ export Epoch, Epoch16, TT2000
 export CDF_EPOCH, CDF_EPOCH16, CDF_TIME_TT2000, CDF_CHAR, CDF_UCHAR
 export is_record_varying
 
+@static if isdefined(Base, :OncePerProcess)
+    const decompressors = Base.OncePerProcess{Channel{Decompressor}}() do
+        n_ch = nthreads()
+        chnl = Channel{Decompressor}(n_ch)
+        foreach(i -> put!(chnl, Decompressor()), 1:n_ch)
+        return chnl
+    end
+else
+    const _decompressors = Ref{Union{Channel{Decompressor},Nothing}}(nothing)
+    function decompressors()
+        if _decompressors[] === nothing
+            n_ch = nthreads()
+            chnl = Channel{Decompressor}(n_ch)
+            foreach(i -> put!(chnl, Decompressor()), 1:n_ch)
+            _decompressors[] = chnl
+        end
+        return _decompressors[]
+    end
+    __init__() = _decompressors[] = nothing
+end
+
 include("epochs.jl")
 include("enums.jl")
 include("types.jl")

src/dataset.jl

@@ -77,64 +77,80 @@ function _vdr_at(cdf::CDFDataset{FST}, offset::Int) where {FST}
         rVDR{FST}(buffer, offset)
 end
 
-function find_vdr(cdf::CDFDataset, var_name::String)
-    gdr = GDR(cdf)
-    RecordSizeType = recordsize_type(cdf)
-    buffer = cdf.buffer
+# Name is the first variable-width field of a VDR; everything before it is fixed size.
+vdr_name(buffer, offset, ::Type{FST}) where {FST} = readname(buffer, offset + 45 + 5 * sizeof(FST))
+# r-variables are chained first, then z-variables
+vdr_heads(cdf::CDFDataset) = (GDR(cdf).rVDRhead, GDR(cdf).zVDRhead)
+
+function find_vdr(cdf::CDFDataset{FST}, var_name::String) where {FST}
+    buffer = parent(cdf)
     var_name_bytes = codeunits(var_name)
-    vdr_name_offset = 45 + 5 * sizeof(RecordSizeType)
-    for current_offset in (gdr.rVDRhead, gdr.zVDRhead)
-        while current_offset != 0
-            if readname(buffer, current_offset + vdr_name_offset) == var_name_bytes
-                return _vdr_at(cdf, Int(current_offset))
-            end
-            current_offset = read_be(buffer, current_offset + 5 + sizeof(RecordSizeType), RecordSizeType)
-        end
+    for head in vdr_heads(cdf), offset in OffsetsIterator{FST}(buffer, head)
+        vdr_name(buffer, offset, FST) == var_name_bytes && return _vdr_at(cdf, offset)
     end
     return nothing
 end
 
-# Direct variable access via indexing
-function Base.getindex(cdf::CDFDataset, var_name::String)
-    return variable(cdf, var_name)
+function Base.getindex(cdf::CDFDataset, name::String)
+    vdr = find_vdr(cdf, name)
+    isnothing(vdr) && throw(KeyError(name))
+    return _variable(cdf, name, vdr)
 end
 
+# Branch over dimension count so each leaf builds dims tuple at compile time statically
+function _variable(cdf, name, vdr)
+    M = num_record_dims(vdr, cdf)
+    return Base.Cartesian.@nif 12 d -> (M == d - 1) d -> (
+        d == 12 ? throw(ArgumentError("variable has $M dimensions; the CDF format allows at most 10")) :
+        _variable(cdf, name, vdr, Val(d - 1))
+    )
+end
+
+function _variable(cdf, name, vdr, ::Val{M}) where {M}
+    dims = (map(Int, record_sizes(vdr, cdf, Val(M)))..., Int(vdr.max_rec) + 1)
+    code = Int(vdr.data_type)
+    if code == CDF_CHAR || code == CDF_UCHAR # eltype depends on runtime num_elems
+        T = StaticString{Int(vdr.num_elems),UInt8}
+        return CDFVariable{T,M + 1,typeof(vdr),typeof(cdf)}(name, vdr, cdf, dims)
+    end
+    # Branch to static constructor per element type
+    return Base.Cartesian.@nif(
+        16,
+        d -> code == CODE_TYPE_PAIRS[d][1],
+        d -> _construct(cdf, name, vdr, dims, CODE_TYPE_PAIRS[d][2]),
+        d -> throw(ArgumentError("unsupported CDF data type $code"))
+    )
+end
+
+@inline _construct(cdf, name, vdr, dims::NTuple{N,Int}, ::Type{T}) where {N,T} =
+    CDFVariable{T,N,typeof(vdr),typeof(cdf)}(name, vdr, cdf, dims)
+
 Base.length(cdf::CDFDataset) = Int(GDR(cdf).NrVars + GDR(cdf).NzVars)
 
-function Base.keys(cdf::CDFDataset)
-    RecordSizeType = recordsize_type(cdf)
-    gdr = cdf.gdr
-    source = parent(cdf)
-    varnames = Vector{String}(undef, gdr.NrVars + gdr.NzVars)
+function Base.keys(cdf::CDFDataset{FST}) where {FST}
+    buffer = parent(cdf)
+    varnames = Vector{String}(undef, length(cdf))
     i = 1
-    vdr_name_offset = 45 + 5 * sizeof(RecordSizeType)
-    for current_offset in (gdr.rVDRhead, gdr.zVDRhead)
-        while current_offset != 0
-            vdr_next = read_be(source, current_offset + 5 + sizeof(RecordSizeType), RecordSizeType)
-            varnames[i] = String(readname(source, current_offset + vdr_name_offset))
-            i += 1
-            current_offset = vdr_next
-        end
+    for head in vdr_heads(cdf), offset in OffsetsIterator{FST}(buffer, head)
+        varnames[i] = String(vdr_name(buffer, offset, FST))
+        i += 1
     end
     return varnames
 end
 
 Base.haskey(cdf::CDFDataset, var_name::String) = !isnothing(find_vdr(cdf, var_name))
 
 # Walk the rVDR then zVDR chain directly; state = (offset, still_in_r_chain)
-function Base.iterate(cdf::CDFDataset, state = (Int(GDR(cdf).rVDRhead), true))
+function Base.iterate(cdf::CDFDataset{FST}, state = (Int(GDR(cdf).rVDRhead), true)) where {FST}
     offset, in_rchain = state
     if offset == 0
         in_rchain || return nothing
         offset, in_rchain = Int(GDR(cdf).zVDRhead), false
         offset == 0 && return nothing
     end
-    RecordSizeType = recordsize_type(cdf)
     buffer = parent(cdf)
-    name = String(readname(buffer, offset + 45 + 5 * sizeof(RecordSizeType)))
-    var = _variable(cdf, name, _vdr_at(cdf, offset))
-    next_offset = Int(read_be(buffer, offset + 5 + sizeof(RecordSizeType), RecordSizeType))
-    return (var, (next_offset, in_rchain))
+    var = _variable(cdf, String(vdr_name(buffer, offset, FST)), _vdr_at(cdf, offset))
+    return (var, (next_record_offset(buffer, offset, FST), in_rchain))
 end
 
 function Base.show(io::IO, m::MIME"text/plain", cdf::CDFDataset)

src/decompress.jl

@@ -9,32 +9,27 @@ function decompress_bytes(buffer, ::Type{RecordSizeType}) where {RecordSizeType}
     payload = data_view(ccr, buffer)
     expected = Int(ccr.uncompressed_size)
     decompressed = decompress_bytes(payload, compression; expected_bytes = expected)
-    new_size = 8 + length(decompressed)
-    new_buffer = Vector{UInt8}(undef, new_size)
+    new_buffer = Vector{UInt8}(undef, 8 + length(decompressed))
     copyto!(new_buffer, 1, buffer, 1, 4)
-    new_buffer[5] = 0x00
-    new_buffer[6] = 0x00
-    new_buffer[7] = 0xFF
-    new_buffer[8] = 0xFF
+    write_be(new_buffer, 5, 0x0000FFFF)  # the uncompressed marker replaces the CCR magic
     copyto!(new_buffer, 9, decompressed, 1, length(decompressed))
     return new_buffer, compression
 end
 
 function decompress_bytes(data, compression::CompressionType; expected_bytes::Union{Nothing, Int} = nothing)
     compression == NoCompression && return data
-    compression in (GzipCompression, RLECompression) ||
-        throw(ArgumentError("unsupported compression: $compression"))
     result = if compression == GzipCompression
-        decompressor = Decompressor()
         input = convert(Vector{UInt8}, data)
         max_size = isnothing(expected_bytes) ? length(input) * 10 : expected_bytes
         output = Vector{UInt8}(undef, max_size)
-        decomp_result = gzip_decompress!(decompressor, output, input)
+        decomp_result = gzip_decompress!(Decompressor(), output, input)
         resize!(output, decomp_result.len)
         output
-    else
+    elseif compression == RLECompression
         isnothing(expected_bytes) && throw(ArgumentError("RLE decompression requires expected size"))
         _rle_decompress(data, expected_bytes)
+    else
+        throw(ArgumentError("unsupported compression: $compression"))
     end
     if !isnothing(expected_bytes) && length(result) != expected_bytes
         throw(ArgumentError("Decompressed payload size mismatch (expected $(expected_bytes), got $(length(result)))"))

src/enums.jl

@@ -53,13 +53,6 @@ const CODE_TYPE_PAIRS = (
     (11, UInt8), (12, UInt16), (14, UInt32), (41, Int8),
 )
 
-const type_map = Dict{CDFDataType, Type}(
-    Dict(CDFDataType(c) => T for (c, T) in CODE_TYPE_PAIRS)...,
-    CDF_CHAR => UInt8,
-    CDF_UCHAR => UInt8,
-)
+const type_map = Dict{CDFDataType, Type}(CDFDataType(c) => T for (c, T) in CODE_TYPE_PAIRS)
 
-function julia_type(cdf_type, num_elems)
-    cdf_type = CDFDataType(cdf_type)
-    return cdf_type in (CDF_CHAR, CDF_UCHAR) ? StaticString{Int(num_elems), UInt8} : type_map[cdf_type]
-end
+julia_type(cdf_type) = type_map[CDFDataType(cdf_type)]

src/loading/attribute.jl

@@ -59,7 +59,7 @@ function Base.iterate(la::LazyVAttrib, offset::Int = Int(la.cdf.gdr.ADRhead))
     while offset != 0
         # cheap scope check before parsing the full ADR (avoids Name string allocation for globals)
         if is_global(buffer, offset, RecordSizeType)
-            offset = Int(read_be(buffer, offset + 1 + sizeof(RecordSizeType) + 4, RecordSizeType))
+            offset = next_record_offset(buffer, offset, RecordSizeType)
             continue
         end
         adr = ADR{RecordSizeType}(buffer, offset)
@@ -125,13 +125,12 @@ end
     aedr_head == 0 && return nothing
     offset = Int(aedr_head)
     _num_offset = 13 + 2 * sizeof(RecordSizeType)
-    _next_offset = 5 + sizeof(RecordSizeType)
     while offset != 0
         num = read_be(source, offset + _num_offset, Int32)
         if num == target_varnum
             return load_aedr_data(source, offset, RecordSizeType, needs_byte_swap)
         end
-        offset = Int(read_be(source, offset + _next_offset, RecordSizeType))
+        offset = next_record_offset(source, offset, RecordSizeType)
     end
     return nothing
 end

src/loading/variable.jl

@@ -1,24 +1,3 @@
-@static if isdefined(Base, :OncePerProcess)
-    const decompressors = Base.OncePerProcess{Channel{Decompressor}}() do
-        n_ch = nthreads()
-        chnl = Channel{Decompressor}(n_ch)
-        foreach(i -> put!(chnl, Decompressor()), 1:n_ch)
-        return chnl
-    end
-else
-    const _decompressors = Ref{Union{Channel{Decompressor},Nothing}}(nothing)
-    function decompressors()
-        if _decompressors[] === nothing
-            n_ch = nthreads()
-            chnl = Channel{Decompressor}(n_ch)
-            foreach(i -> put!(chnl, Decompressor()), 1:n_ch)
-            _decompressors[] = chnl
-        end
-        return _decompressors[]
-    end
-    __init__() = _decompressors[] = nothing
-end
-
 # ---- Type-erased block moves: dims/strides are runtime vectors ----
 # Element movers, chosen once per block by element size; `ByteMove` covers odd sizes (CHAR data).
 struct WordMove{U} end
@@ -115,40 +94,6 @@ function _copy_subblock!(dst::Ptr{UInt8}, src::Ptr{UInt8}, esz::Int, rdims::Vect
     return
 end
 
-function variable(cdf::CDFDataset, name)
-    vdr = find_vdr(cdf, name)
-    isnothing(vdr) && throw(KeyError(name))
-    return _variable(cdf, name, vdr)
-end
-
-# Branch over dimension count so each leaf builds dims tuple at compile time statically
-function _variable(cdf, name, vdr)
-    M = num_record_dims(vdr, cdf)
-    return Base.Cartesian.@nif 12 d -> (M == d - 1) d -> (
-        d == 12 ? throw(ArgumentError("variable has $M dimensions; the CDF format allows at most 10")) :
-        _variable(cdf, name, vdr, Val(d - 1))
-    )
-end
-
-function _variable(cdf, name, vdr, ::Val{M}) where {M}
-    dims = (map(Int, record_sizes(vdr, cdf, Val(M)))..., Int(vdr.max_rec) + 1)
-    code = Int(vdr.data_type)
-    if code == 51 || code == 52 # CHAR/UCHAR: eltype depends on runtime num_elems
-        T = StaticString{Int(vdr.num_elems),UInt8}
-        return CDFVariable{T,M + 1,typeof(vdr),typeof(cdf)}(name, vdr, cdf, dims)
-    end
-    # Branch to static constructor per element type
-    return Base.Cartesian.@nif(
-        16,
-        d -> code == CODE_TYPE_PAIRS[d][1],
-        d -> _construct(cdf, name, vdr, dims, CODE_TYPE_PAIRS[d][2]),
-        d -> throw(ArgumentError("unsupported CDF data type $code"))
-    )
-end
-
-@inline _construct(cdf, name, vdr, dims::NTuple{N,Int}, ::Type{T}) where {N,T} =
-    CDFVariable{T,N,typeof(vdr),typeof(cdf)}(name, vdr, cdf, dims)
-
 function DiskArrays.readblock!(var::CDFVariable{T,N}, dest::AbstractArray{T}, ranges::Vararg{AbstractUnitRange{<:Integer},N}) where {T,N}
     N > 0 && @boundscheck checkbounds(var, ranges...)
     isempty(dest) && return dest

src/parsing.jl

@@ -16,11 +16,6 @@ end
     return ntoh(unsafe_load(p + (i - 1) * sizeof(T)))
 end
 
-@inline function read_be(v::Vector{UInt8}, i, n, T)
-    S = sizeof(T)
-    return ntuple(j -> read_be(v, i + (j - 1) * S, T), n)
-end
-
 @inline function read_be(v::Vector{UInt8}, i, ::Val{M}, T) where {M}
     S = sizeof(T)
     return ntuple(j -> read_be(v, i + (j - 1) * S, T), Val(M))
@@ -30,11 +25,6 @@ end
     return read_be(v, i, T), i + _sizeof(T)
 end
 
-@inline function read_be_i(v::Vector{UInt8}, i, n::Integer, T)
-    S = sizeof(T)
-    return ntuple(j -> read_be(v, i + (j - 1) * S, T), n), i + n * S
-end
-
 function field_layout(SType, indxs)
     offsets = Int[]
     types = Any[]
@@ -69,21 +59,6 @@ end
 
 @inline write_be(v::Vector{UInt8}, i, ::RInt32) = i + _sizeof(RInt32)
 
-function flatten_field_types(mod, args)
-    types = Any[]
-    for arg in args
-        if arg isa Expr && arg.head === :...
-            vals = Base.eval(mod, arg.args[1])
-            for T in vals
-                push!(types, Meta.quot(T))
-            end
-        else
-            push!(types, arg)
-        end
-    end
-    return types
-end
-
 function readname(buf::Vector{UInt8}, offset::Int)
     for i in offset:(offset + 255)
         if buf[i] == 0x00