From 32869e21a3788b3a96d8e8c43ea0baabbfea1635 Mon Sep 17 00:00:00 2001 From: JohnLCaron Date: Sat, 26 Jul 2025 10:39:52 -0600 Subject: [PATCH] Fix BTree1data bug. cleanup IntellJ Inspections in core:netchdf --- .../kotlin/com/sunya/netchdf/NetchdfFile.kt | 4 +- .../com/sunya/netchdf/NetchdfFileFormat.kt | 4 +- .../com/sunya/netchdf/hdf4/DataReaders.kt | 1 - .../com/sunya/netchdf/hdf4/H4builder.kt | 6 +- .../com/sunya/netchdf/hdf4/H4chunkIterator.kt | 1 - .../com/sunya/netchdf/hdf4/H4tiledData.kt | 2 +- .../com/sunya/netchdf/hdf4/ODLparser.kt | 6 +- .../kotlin/com/sunya/netchdf/hdf4/Tag.kt | 2 +- .../kotlin/com/sunya/netchdf/hdf4/TagEnum.kt | 2 +- .../kotlin/com/sunya/netchdf/hdf4/Vinfo.kt | 2 +- .../kotlin/com/sunya/netchdf/hdf5/BTree1.kt | 144 --------------- .../com/sunya/netchdf/hdf5/BTree1data.kt | 33 ++-- .../com/sunya/netchdf/hdf5/BTree2data.kt | 1 - .../netchdf/hdf5/ExtensibleArrayIndex.kt | 1 + .../com/sunya/netchdf/hdf5/FractalHeapJ.kt | 1 + .../sunya/netchdf/hdf5/GroupSymbolTable.kt | 3 +- .../com/sunya/netchdf/hdf5/H5TiledData1.kt | 107 ----------- .../com/sunya/netchdf/hdf5/H5TypeInfo.kt | 2 +- .../com/sunya/netchdf/hdf5/H5builder.kt | 35 +--- .../com/sunya/netchdf/hdf5/H5cdmBuilder.kt | 4 +- .../com/sunya/netchdf/hdf5/H5chunkIterator.kt | 88 --------- .../com/sunya/netchdf/hdf5/H5dataReader.kt | 2 +- .../kotlin/com/sunya/netchdf/hdf5/H5group.kt | 20 +-- .../kotlin/com/sunya/netchdf/hdf5/Heap.kt | 4 +- .../sunya/netchdf/hdf5/MessageDataLayout.kt | 2 +- .../com/sunya/netchdf/hdf5/StructDsl.kt | 4 +- .../com/sunya/netchdf/netcdf3/N3builder.kt | 8 +- .../com/sunya/netchdf/netcdf3/Netcdf3File.kt | 6 +- .../com/sunya/netchdf/NetchdfClibExtra.kt | 2 +- .../com/sunya/netchdf/NetchdfClibTest.kt | 168 +++++------------- 30 files changed, 105 insertions(+), 560 deletions(-) delete mode 100644 core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1.kt delete mode 100644 core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5TiledData1.kt delete mode 100644 core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkIterator.kt diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/NetchdfFile.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/NetchdfFile.kt index bfebe4e5..690bf872 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/NetchdfFile.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/NetchdfFile.kt @@ -22,7 +22,7 @@ fun openNetchdfFile(filename : String, strict : Boolean = false) : Netchdf? { NetchdfFileFormat.NC_FORMAT_NETCDF4_CLASSIC -> Hdf5File(useFilename, strict) NetchdfFileFormat.HDF5 -> Hdf5File(useFilename, strict) NetchdfFileFormat.HDF4 -> Hdf4File(useFilename) - else -> null // throw RuntimeException(" unsupported NetcdfFileFormat $format") + else -> null } } } @@ -38,7 +38,7 @@ fun openNetchdfFileWithFormat(filename : String, format : NetchdfFileFormat) : N NetchdfFileFormat.NC_FORMAT_NETCDF4_CLASSIC -> Hdf5File(useFilename, false) NetchdfFileFormat.HDF5 -> Hdf5File(useFilename, false) NetchdfFileFormat.HDF4 -> Hdf4File(useFilename) - else -> null // throw RuntimeException(" unsupported NetcdfFileFormat $format") + else -> null } } } diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/NetchdfFileFormat.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/NetchdfFileFormat.kt index 2d6c3c33..76dddd33 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/NetchdfFileFormat.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/NetchdfFileFormat.kt @@ -185,7 +185,7 @@ enum class NetchdfFileFormat(private val version: Int, private val formatName: S val h5type = searchForwardHdf5(raf, magic) if (h5type != null) return h5type - val h4type = searchForwardHdf4(raf, magic) + val h4type = searchForwardHdf4(raf) return h4type ?: INVALID } @@ -284,7 +284,7 @@ enum class NetchdfFileFormat(private val version: Int, private val formatName: S return null } - private fun searchForwardHdf4(raf: OpenFileIF, want: ByteArray): NetchdfFileFormat? { + private fun searchForwardHdf4(raf: OpenFileIF): NetchdfFileFormat? { val size: Long = raf.size() val state = OpenFileState(0L, true) var startPos = 0L diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/DataReaders.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/DataReaders.kt index 9f010ea1..5f76f432 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/DataReaders.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/DataReaders.kt @@ -79,7 +79,6 @@ internal class LinkedByteSource(val h4 : H4builder, private val state : OpenFileState private var segno = -1 private var segSize = 0 - private var buffer = ByteArray(0) private var exhausted = false val totalSize: Int diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/H4builder.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/H4builder.kt index ca0708e0..7ef20832 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/H4builder.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/H4builder.kt @@ -65,7 +65,7 @@ class H4builder(val raf: OpenFileIF, val valueCharset: Charset) { val ndd: Int = raf.readShort(state).toUShort().toInt() // number of DD blocks link = raf.readInt(state).toUInt().toLong() // point to the next DDH; link == 0 means no more var pos = state.pos - for (i in 0 until ndd) { + repeat (ndd) { val tag: Tag = readTag(raf, state) pos += 12 state.pos = pos // tag usually changed the file pointer @@ -425,7 +425,7 @@ class H4builder(val raf: OpenFileIF, val valueCharset: Charset) { } // compute the vb and add to the group - val vb = SDread(tagNDG!!, vgroup.name, group, dims) ?: return + val vb = SDread(tagNDG, vgroup.name, group, dims) ?: return // tagVH's on the group tag (TagVGroup) might be attributes tagVHs.forEach { @@ -558,7 +558,7 @@ class H4builder(val raf: OpenFileIF, val valueCharset: Charset) { println(" **** NO dimensions found for SD ${dataGroup.refCode()}") return null } - val dimSDD = dimSDDout!! + val dimSDD = dimSDDout val nt: TagNT = tagidMap[tagid(dimSDD.data_nt_ref, TagEnum.NT.code)] as TagNT? ?: throw IllegalStateException(" **** NO nt tag found for SD ${dataGroup.refCode()}") diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/H4chunkIterator.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/H4chunkIterator.kt index 667d0455..e221e88d 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/H4chunkIterator.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/H4chunkIterator.kt @@ -14,7 +14,6 @@ internal class H4chunkIterator(h4 : H4builder, val v2: Variable<*>, val wantS private val vinfo = v2.spObject as Vinfo private val elemSize = vinfo.elemSize - private val datatype = v2.datatype private val wantSpace = IndexSpace(wantSection) private val tiledData : H4tiledData = H4tiledData(h4, v2.shape, vinfo.chunkLengths, vinfo.chunks!!) diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/H4tiledData.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/H4tiledData.kt index 13a4230f..5786df72 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/H4tiledData.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/H4tiledData.kt @@ -43,7 +43,7 @@ internal class H4tiledData(val h4 : H4builder, varShape : LongArray, chunk : Int internal class H4CompressedDataChunk( val h4 : H4builder, - val offsets: IntArray, // offset index of this chunk, reletive to entire array + val offsets: IntArray, // offset index of this chunk, relative to entire array private val compress: SpecialComp? ) { fun isMissing() = (compress == null) diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/ODLparser.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/ODLparser.kt index b0ee43b7..c61a68e9 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/ODLparser.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/ODLparser.kt @@ -126,7 +126,7 @@ private fun transform(org: ODLgroup, trans: ODLgroup) { } if (wantName != null) { - val nestedTrans = ODLgroup(wantName!!, trans) + val nestedTrans = ODLgroup(wantName, trans) trans.nested.add(nestedTrans) if (org.variables.isNotEmpty()) { nestedTrans.variables.add(transformVariables(org)) @@ -234,7 +234,7 @@ internal fun ODLparseFromString(text: String): ODLgroup { } else if (line.startsWith("END")) { // noop } else if (currentObject != null) { - addFieldToObject(currentObject as ODLobject, line) + addFieldToObject(currentObject, line) } else { addFieldToGroup(currentStruct, line) } @@ -399,7 +399,7 @@ class ODLparser(val rootGroup: Group.Builder, val show : Boolean = false) { return false } } catch (ex : Exception) { - logger.warn{" *** ODL cant parse dimension ${att.component1()} length ${att.component2()}"} + logger.warn{" *** ODL cant parse dimension ${att.component1()} length ${att.component2()} message=${ex.message}"} return false } } diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/Tag.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/Tag.kt index cc23d685..6fad8716 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/Tag.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/Tag.kt @@ -657,7 +657,7 @@ internal class TagVH(icode: Int, refno: Int, offset : Long, length : Int) : Tag( fld_isize = IntArray(nfields) { h4.raf.readShort(state).toUShort().toInt() } fld_offset = IntArray(nfields) { h4.raf.readShort(state).toUShort().toInt() } fld_nelems = IntArray(nfields) { h4.raf.readShort(state).toUShort().toInt() } // "Order of the nth field of the Vdata (16-bit integer)" - for (i in 0 until nfields) { + repeat (nfields) { val slen = h4.raf.readShort(state).toInt() fld_name.add(h4.raf.readString(state, slen)) } diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/TagEnum.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/TagEnum.kt index 26671916..5f5c3eac 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/TagEnum.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/TagEnum.kt @@ -102,7 +102,7 @@ enum class TagEnum(val desc: String, val code: Int) { fun byCode(code: Int): TagEnum { if (hashCodes == null) { // LOOK may fail when multithreading val mapit = mutableMapOf() - values().forEach { mapit[it.code] = it} + entries.forEach { mapit[it.code] = it} hashCodes = mapit } val te = hashCodes!![code] diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/Vinfo.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/Vinfo.kt index c283a254..d3c1831f 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/Vinfo.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/Vinfo.kt @@ -75,7 +75,7 @@ internal class Vinfo(val refno: Int) : Comparable { return this } - // make sure needed info is present : call this when variable needs to be read + // make sure that the needed info is present: call this when variable needs to be read // this allows us to defer getting layout info until then fun setLayoutInfo(header: H4builder) { if (tagData == null) return diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1.kt deleted file mode 100644 index 5bd1f9aa..00000000 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1.kt +++ /dev/null @@ -1,144 +0,0 @@ -@file:OptIn(InternalLibraryApi::class) - -package com.sunya.netchdf.hdf5 - -import com.sunya.cdm.iosp.OpenFileState -import com.sunya.cdm.layout.Tiling -import com.sunya.cdm.util.InternalLibraryApi - -/** B-tree, version 1, used for data (node type 1) -internal class BTree1( - val h5: H5builder, - val rootNodeAddress: Long, - val nodeType : Int, // 0 = group/symbol table, 1 = raw data chunks - val ndimStorage: Int? = null // TODO allowed to be null ?? -) { - - fun rootNodeAddress() = rootNodeAddress - - fun readNode(address: Long, parent: BTree1.Node?): Node = - Node(address, parent as Node?) - - fun makeMissingDataChunkEntry(rootNode: Node, wantKey: LongArray) : DataChunkIF = - DataChunkEntry1(0, rootNode, -1, DataChunkKey(-1, 0, wantKey), -1L) - - fun readGroupEntries() : Iterator { - require(nodeType == 0) - val root = Node(rootNodeAddress, null) - return if (root.level == 0) { - root.groupEntries.iterator() - } else { - val result = mutableListOf() - for (entry in root.groupEntries) { - readAllEntries(entry, root, result) - } - result.iterator() - } - } - - private fun readAllEntries(entry : GroupEntry, parent : Node, list : MutableList) { - val node = Node(entry.childAddress, parent) - if (node.level == 0) { - list.addAll(node.groupEntries) - } else { - for (nested in node.groupEntries) { - readAllEntries(nested, node, list) - } - } - } - - // here both internal and leaf are the same structure - // Btree nodes Level 1A1 - Version 1 B-trees - inner class Node(val address: Long, val parent: Node?) { - val level: Int - val nentries: Int - private val leftAddress: Long - private val rightAddress: Long - - // type 0 - val groupEntries = mutableListOf() - - // type 1 - val dataChunkEntries = mutableListOf() - - init { - val state = OpenFileState(h5.getFileOffset(address), false) - val magic: String = h5.raf.readString(state, 4) - check(magic == "TREE") { "DataBTree doesnt start with TREE" } - - val type: Int = h5.raf.readByte(state).toInt() - check(type == nodeType) { "DataBTree must be type $nodeType" } - - level = h5.raf.readByte(state).toInt() // leaf nodes are level 0 - nentries = h5.raf.readShort(state).toInt() // number of children to which this node points - leftAddress = h5.readOffset(state) - rightAddress = h5.readOffset(state) - - for (idx in 0 until nentries) { - if (type == 0) { - val key = h5.readLength(state) // 4 or 8 bytes - val address = h5.readOffset(state) // 4 or 8 bytes - if (address > 0) groupEntries.add(GroupEntry(key, address)) - } else { - val chunkSize = h5.raf.readInt(state) - val filterMask = h5.raf.readInt(state) - val inner = LongArray(ndimStorage!!) { j -> h5.raf.readLong(state) } - val key = DataChunkKey(chunkSize, filterMask, inner) - val childPointer = h5.readAddress(state) // 4 or 8 bytes, then add fileOffset - dataChunkEntries.add(DataChunkEntry1(level, this, idx, key, childPointer)) - } - } - - // note there may be unused entries, "All nodes of a particular type of tree have the same maximum degree, - // but most nodes will point to less than that number of children"" - } - - fun isLeaf() = (level == 0) - - fun nentries() = nentries - - fun dataChunkEntryAt(idx: Int) = dataChunkEntries[idx] - } - - /** @param key the byte offset into the local heap for the first object name in the subtree which that key describes. */ - data class GroupEntry(val key : Long, val childAddress : Long) - - data class DataChunkKey(val chunkSize: Int, val filterMask : Int, val offsets: LongArray) { - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (other !is DataChunkKey) return false - - if (!offsets.contentEquals(other.offsets)) return false - - return true - } - - override fun hashCode(): Int { - return offsets.contentHashCode() - } - } - - // childAddress = data chunk (level 1) else a child node - data class DataChunkEntry1(val level : Int, val parent : Node, val idx : Int, val key : DataChunkKey, val childAddress : Long) : DataChunkIF { - override fun childAddress() = childAddress - override fun offsets() = key.offsets - override fun isMissing() = (childAddress == -1L) - override fun chunkSize() = key.chunkSize - override fun filterMask() = key.filterMask - - override fun show(tiling : Tiling) : String = "chunkSize=${key.chunkSize}, chunkStart=${key.offsets.contentToString()}" + - ", tile= ${tiling.tile(key.offsets).contentToString()} idx=$idx" - } - -} */ - - -interface DataChunkIF { - fun childAddress(): Long - fun offsets(): LongArray - fun isMissing(): Boolean - fun chunkSize(): Int - fun filterMask(): Int? - - fun show(tiling : Tiling): String -} diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1data.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1data.kt index 6d22d00c..96edb572 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1data.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1data.kt @@ -71,28 +71,19 @@ internal class BTree1data( keyValues.add(Pair(order, DataChunk(key, childPointer))) lastOrder = order } else { - children.add(BTreeNode(childPointer, this)) + children.add( BTreeNode(childPointer, this) ) } } + if (children.isNotEmpty()) { + lastOrder = children.last().lastOrder + } } // note there may be unused entries, "All nodes of a particular type of tree have the same maximum degree, // but most nodes will point to less than that number of children"" } - // this does not have missing data. Use iterator on the Btree1data class - /* return only the leaf nodes, in depth-first order - fun asSequence(): Sequence> = sequence { - // Handle child nodes recursively (in-order traversal) - if (children.isNotEmpty()) { - children.forEachIndexed { index, childNode -> - yieldAll(childNode.asSequence()) // Yield all elements from the child - } - } else { // If it's a leaf node (no children) - keyValues.forEach { yield(it) } - } - } */ - + // uses a tree search = O(log n) fun findDataChunk(wantOrder: Int): DataChunk? { if (children.isNotEmpty()) { // search tree; assumes that chunks are ordered children.forEach { childNode -> @@ -106,6 +97,10 @@ internal class BTree1data( return null } + override fun toString(): String { + return "BTreeNode(address=$address, level=$level, nentries=$nentries, lastOrder=$lastOrder)" + } + } data class DataChunkKey(val order: Int, val chunkSize: Int, val filterMask : Int) @@ -129,3 +124,13 @@ internal class BTree1data( } } +interface DataChunkIF { + fun childAddress(): Long + fun offsets(): LongArray + fun isMissing(): Boolean + fun chunkSize(): Int + fun filterMask(): Int? + + fun show(tiling : Tiling): String +} + diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2data.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2data.kt index 269c4669..bd016e31 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2data.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2data.kt @@ -311,7 +311,6 @@ internal class BTree2data(private val h5: H5builder, owner: String, address: Lon private inner class ChunkIterator : AbstractIterator() { var count = 0 - var first = true override fun computeNext() { if (count >= records.size) { diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/ExtensibleArrayIndex.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/ExtensibleArrayIndex.kt index 12782aaf..f0074d79 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/ExtensibleArrayIndex.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/ExtensibleArrayIndex.kt @@ -7,6 +7,7 @@ import com.sunya.cdm.iosp.OpenFileState import com.sunya.cdm.util.InternalLibraryApi import java.util.* +// DataLayoutExtensibleArray4 class ExtensibleArrayIndex(val h5: H5builder, address: Long, datasetDimensions: IntArray, chunkDimensions: IntArray) { val raf = h5.raf diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FractalHeapJ.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FractalHeapJ.kt index a53dd663..b986ee39 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FractalHeapJ.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FractalHeapJ.kt @@ -11,6 +11,7 @@ import kotlin.math.min val UNDEFINED_ADDRESS = -1L +// the start of a port of jfhf FractalHeap. May not need it. class FractalHeapJ(val h5: H5builder, forWho: String, val fractalHeapAddress: Long) { val raf = h5.raf diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/GroupSymbolTable.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/GroupSymbolTable.kt index fd9ffbcd..a5c299b2 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/GroupSymbolTable.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/GroupSymbolTable.kt @@ -28,7 +28,7 @@ internal class GroupSymbolTable(val btreeAddress : Long) { val nentries = h5.raf.readShort(state) var posEntry = state.pos - for (i in 0 until nentries) { + repeat (nentries.toInt()) { val entry = h5.readSymbolTable(state) posEntry += entry.dataSize if (entry.objectHeaderAddress != 0L) { // skip zeroes, probably a bug in HDF5 file format or docs, or me @@ -58,7 +58,6 @@ internal fun H5builder.readSymbolTable(state : OpenFileState) : SymbolTableEntry } if (debugGroup) rootEntry.show() - // may be btree or symbolic link var btreeAddress : Long? = null var nameHeapAddress : Long? = null var linkOffset : Int? = null diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5TiledData1.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5TiledData1.kt deleted file mode 100644 index c750adc1..00000000 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5TiledData1.kt +++ /dev/null @@ -1,107 +0,0 @@ -package com.sunya.netchdf.hdf5 - -import com.sunya.cdm.layout.IndexSpace -import com.sunya.cdm.layout.IndexND -import com.sunya.cdm.layout.Tiling - -/** wraps BTree1 to handle iterating through tiled data (aka chunked data) -internal class H5TiledData1(val btree : BTree1, val varShape: LongArray, val chunkShape: LongArray) { - private val check = true - private val debug = false - private val debugMissing = false - - val tiling = Tiling(varShape, chunkShape) - val rootNode : BTree1.Node - - // keep track of nodes so we only read once - private val nodeCache = mutableMapOf() - private var readHit = 0 - private var readMiss = 0 - - init { - rootNode = readNode(btree.rootNodeAddress(), null) - } - - // node reading goes through here for caching - private fun readNode(address : Long, parent : BTree1.Node?) : BTree1.Node { - if (nodeCache[address] != null) { - readHit++ - return nodeCache[address]!! - } - readMiss++ - val node = btree.readNode(address, parent) - nodeCache[address] = node - - /* - if (check) { - if (debug) println("node = $address, level = ${node.level} nentries = ${node.nentries}") - for (idx in 0 until node.nentries) { - val thisEntry = node.dataChunkEntries[idx] - val key = thisEntry.key.offsets - if (debug) println(" $idx = ${key.contentToString()} tile = ${tiling.tile(key).contentToString()}") - if (idx < node.nentries - 1) { - val nextEntry = node.dataChunkEntries[idx + 1] - require(tiling.compare(key, nextEntry.key.offsets) < 0) - } - } - } - */ - return node - } - - fun dataChunks(wantSpace : IndexSpace) = Iterable { DataChunkIterator(wantSpace) } - - private inner class DataChunkIterator(wantSpace : IndexSpace) : AbstractIterator() { - val tileIterator : Iterator - - init { - val tileSection = tiling.section(wantSpace) // section in tiles that we want - tileIterator = IndexND(tileSection, tiling.tileShape).iterator() // iterate over tiles we want - } - - override fun computeNext() { - if (!tileIterator.hasNext()) { - return done() - } else { - val wantTile = tileIterator.next() - val wantKey = tiling.index(wantTile) // convert to index "keys" - val haveEntry = findEntryContainingKey(rootNode, wantKey) - val useEntry = haveEntry ?: btree.makeMissingDataChunkEntry(rootNode, wantKey) - setNext(useEntry) - } - } - } - - // TODO optimize. Might be easier to read in all the Nodes. - private fun findEntryContainingKey(parent : BTree1.Node, key : LongArray) : DataChunkIF? { - var foundEntry : DataChunkIF? = null - for (idx in 0 until parent.nentries()) { - foundEntry = parent.dataChunkEntryAt(idx) - if (idx < parent.nentries() - 1) { - val nextEntry = parent.dataChunkEntryAt(idx + 1) // look at the next one - if (tiling.compare(key, nextEntry.offsets()) < 0) { - break - } - } - } - if (foundEntry == null) { - if (parent.isLeaf()) { - if (debugMissing) println("H5TiledData findEntryContainingKey missing key ${key.contentToString()}") - return null - } - throw RuntimeException("H5TiledData findEntryContainingKey cant find key ${key.contentToString()}") - } - if (parent.isLeaf()) { - return if (tiling.compare(key, foundEntry.offsets()) == 0L) foundEntry else null - } - - // if not a leaf, keep descending into the tree - val node= readNode(foundEntry.childAddress(), parent) - return findEntryContainingKey(node, key) - } - - override fun toString(): String { - return "TiledData(chunk=${chunkShape.contentToString()}, readHit=$readHit, readMiss=$readMiss)" - } - -} */ \ No newline at end of file diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5TypeInfo.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5TypeInfo.kt index 476b49b7..a9b94099 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5TypeInfo.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5TypeInfo.kt @@ -42,7 +42,7 @@ internal fun H5builder.makeH5TypeInfo(mdt: DatatypeMessage, typedef : Typedef? = // all the info you need to create a CDM Datatype internal data class H5TypeInfo(val isVlenString: Boolean, val isRefObject : Boolean, val datatype5 : Datatype5, val elemSize : Int, val signed : Boolean, val isBE : Boolean, val mdtAddress : Long, val mdtHash : Int, - val base : H5TypeInfo? = null, val typedef : Typedef? = null, val dims : IntArray? = null) { + val base : H5TypeInfo? = null, val typedef : Typedef? = null) { fun datatype(): Datatype<*> { return when (datatype5) { diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5builder.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5builder.kt index 5d3bb96f..0687ea23 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5builder.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5builder.kt @@ -232,13 +232,8 @@ class H5builder( ////////////////////////////////////////////////////////////// // Internal organization of Data Objects - fun convertReferenceToDataObjectName(reference: Long): String { - val name = getDataObjectName(reference) - return name ?: reference.toString() // LOOK - } - fun convertReferencesToDataObjectName(refArray: Array): List { - return refArray.map { convertReferenceToDataObjectName(it) } + return refArray.map { getDataObjectName(it) } } /** @@ -246,24 +241,11 @@ class H5builder( * * @param objId address of the data object * @return String the data object's name, or null if not found - * @throws IOException on read error */ fun getDataObjectName(objId: Long): String { return getDataObject(objId, null)?.name ?: "unknown" } - /** - * Get a data object's name, using the objectId you get from a reference (aka hard link). - * - * @param objId address of the data object - * @return String the data object's name, or null if not found - * @throws IOException on read error - * - @Throws(IOException::class) - fun getDataObjectName(objId: Long): String { - return getDataObject(objId, null)?.name ?: "unknown" - } */ - /** * All access to data objects come through here, so we can cache. * Look in cache first; read if not in cache. @@ -370,10 +352,6 @@ class H5builder( return result } - fun readAddress(state : OpenFileState): Long { - return getFileOffset(readOffset(state)) - } - // size of data depends on "maximum possible number" fun getNumBytesFromMax(maxNumber: Long): Int { var maxn = maxNumber @@ -457,7 +435,6 @@ class H5builder( 0x1a, '\n'.code.toByte() ) - private val magicString = makeString(magicHeader) private const val transformReference = true @@ -469,11 +446,11 @@ class H5builder( * any field called filePos or dataPos is a byte offset within the file. * * it appears theres no sure fire way to tell if the file was written by netcdf4 library - * 1) if one of the the NETCF4-XXX atts are set - * 2) dimension scales: - * 1) all dimensions have a dimension scale - * 2) they all have the same length as the dimension - * 3) all variables' dimensions have a dimension scale + * 1) if one of the the NETCF4-XXX atts are set + * 2) dimension scales: + * 1) all dimensions have a dimension scale + * 2) they all have the same length as the dimension + * 3) all variables' dimensions have a dimension scale */ private const val KNOWN_FILTERS = 3 diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5cdmBuilder.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5cdmBuilder.kt index 071edcd4..842f3fcf 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5cdmBuilder.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5cdmBuilder.kt @@ -224,7 +224,7 @@ internal class DataContainerVariable( else -> -1 // LOOK compact? } // deal with unallocated data - fillValue = getFillValue(h5, v5, h5type) + fillValue = getFillValue(v5, h5type) onlyFillValue = (dataPos == -1L) when (mdl) { @@ -287,7 +287,7 @@ internal class DataContainerVariable( } -internal fun getFillValue(h5 : H5builder, v5 : H5Variable, h5type: H5TypeInfo): ByteArray { +internal fun getFillValue(v5 : H5Variable, h5type: H5TypeInfo): ByteArray { // look for fill value message var fillValue : ByteArray? = null for (mess in v5.dataObject.messages) { diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkIterator.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkIterator.kt deleted file mode 100644 index 44e6def2..00000000 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkIterator.kt +++ /dev/null @@ -1,88 +0,0 @@ -package com.sunya.netchdf.hdf5 - -import com.sunya.cdm.api.* -import com.sunya.cdm.array.ArrayTyped -import com.sunya.cdm.iosp.OpenFileState -import com.sunya.cdm.layout.Chunker -import com.sunya.cdm.layout.IndexSpace -import com.sunya.cdm.layout.transferMissingNelems -import com.sunya.cdm.util.InternalLibraryApi - -/* (to be removed) -@OptIn(InternalLibraryApi::class) -internal class H5chunkIterator(val h5 : H5builder, val v2: Variable, val wantSection : Section) : AbstractIterator>() { - private val debugChunking = false - - val vinfo : DataContainerVariable = v2.spObject as DataContainerVariable - val h5type : H5TypeInfo - val elemSize : Int - val datatype : Datatype<*> - val tiledData : H5TiledData1 - val filters : FilterPipeline - val state : OpenFileState - - private val wantSpace : IndexSpace - private val chunkIterator : Iterator - - init { - h5type = vinfo.h5type - elemSize = vinfo.storageDims[vinfo.storageDims.size - 1].toInt() // last one is always the elements size - datatype = h5type.datatype() - - val btree1 = BTree1(h5, vinfo.dataPos, 1, vinfo.storageDims.size) - tiledData = H5TiledData1(btree1, v2.shape, vinfo.storageDims) - filters = FilterPipeline(v2.name, vinfo.mfp, h5type.isBE) - if (debugChunking) println(" H5chunkIterator tiles=${tiledData.tiling}") - - state = OpenFileState(0L, h5type.isBE) - wantSpace = IndexSpace(wantSection) - chunkIterator = tiledData.dataChunks(wantSpace).iterator() - } - - override fun computeNext() { - if (chunkIterator.hasNext()) { - setNext( getaPair(chunkIterator.next()) ) - } else { - done() - } - } - - private fun getaPair(dataChunk : DataChunkIF) : ArraySection { - val dataSpace = IndexSpace(v2.rank, dataChunk.offsets(), vinfo.storageDims) - - // TODO we need to intersect the dataChunk with the wanted section. - // optionally, we could make a view of the array, rather than copying the data. - val useEntireChunk = wantSpace.contains(dataSpace) - val intersectSpace = if (useEntireChunk) dataSpace else wantSpace.intersect(dataSpace) - - val ba = if (dataChunk.isMissing()) { - if (debugChunking) println(" missing ${dataChunk.show(tiledData.tiling)}") - val sizeBytes = intersectSpace.totalElements * elemSize - val bbmissing = ByteArray(sizeBytes.toInt()) - transferMissingNelems(vinfo.fillValue, intersectSpace.totalElements.toInt(), bbmissing, 0) - if (debugChunking) println(" missing transfer ${intersectSpace.totalElements} fillValue=${vinfo.fillValue}") - bbmissing - } else { - if (debugChunking) println(" chunkIterator=${dataChunk.show(tiledData.tiling)}") - state.pos = dataChunk.childAddress() - val rawdata = h5.raf.readByteArray(state, dataChunk.chunkSize()) - val filteredData = if (dataChunk.filterMask() == null) rawdata else filters.apply(rawdata, dataChunk.filterMask()!!) - if (useEntireChunk) { - filteredData - } else { - val chunker = Chunker(dataSpace, wantSpace) // each DataChunkEntry has its own Chunker iteration - chunker.copyOut(filteredData, 0, elemSize, intersectSpace.totalElements.toInt()) - } - } - - val array = if (h5type.datatype5 == Datatype5.Vlen) { - // internal fun H5builder.processVlenIntoArray(h5type: H5TypeInfo, shape: IntArray, ba: ByteArray, nelems: Int, elemSize : Int): ArrayTyped { - h5.processVlenIntoArray(h5type, intersectSpace.shape.toIntArray(), ba, intersectSpace.totalElements.toInt(), elemSize) - } else { - h5.processDataIntoArray(ba, h5type.isBE, datatype, intersectSpace.shape.toIntArray(), h5type, elemSize) as ArrayTyped - } - - return ArraySection(array, intersectSpace.section(v2.shape)) // LOOK use space instead of Section ?? - } -} */ - diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5dataReader.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5dataReader.kt index b1405bb6..36fcbd04 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5dataReader.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5dataReader.kt @@ -111,7 +111,7 @@ internal fun H5builder.processCompoundData(sdataArray : ArrayStructureData, isBE sdataArray.putVlensOnHeap { member, offset -> val listOfArrays = mutableListOf>() - for (i in 0 until member.nelems) { + repeat (member.nelems) { val heapId = h5heap.readHeapIdentifier(sdataArray.ba, offset) val vlenArray = h5heap.getHeapDataArray(heapId, member.datatype, isBE) listOfArrays.add(vlenArray) diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt index 9a2cb9d4..6c7d8536 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt @@ -16,7 +16,7 @@ internal fun H5builder.readH5Group(facade: DataObjectFacade): H5GroupBuilder? { val dataObject = facade.dataObject!! val result = H5GroupBuilder(this, facade.parent, facade.name, dataObject, nestedObjects) - // if has a "group message", then its an "old group" + // if dataObject has a "group message", then its an "old group" if (dataObject.groupMessage != null) { val groupMessage = dataObject.groupMessage!! // check for hard links @@ -96,18 +96,8 @@ internal fun H5builder.readGroupNew( internal fun H5builder.readGroupOld(groupb: H5GroupBuilder, btreeAddress: Long, nameHeapAddress: Long) { // track by address for hard links hashGroups[btreeAddress] = groupb - val nameHeap = LocalHeap(this, nameHeapAddress) - /* println("readGroupOld Btree1 name = '${groupb.name}' btreeAddress = $btreeAddress nameHeapAddress = $nameHeapAddress") - var count = 0 - val btree = Btree1(this, btreeAddress, groupb.name) - for (s in btree.symbolTableEntries) { - println("$count ${ nameHeap.getStringAt(s.nameOffset.toInt())}") - count++ - } */ - - // println("readGroupOld GroupSymbolTable ") val symbolTable = GroupSymbolTable(btreeAddress) for (s in symbolTable.symbolTableEntries(this)) { val sname: String = nameHeap.getStringAt(s.nameOffset.toInt()) @@ -126,7 +116,7 @@ internal fun H5builder.readGroupOld(groupb: H5GroupBuilder, btreeAddress: Long, } } -// I think this is preventing cycles in the graph, ie turning it into a tree +// I think this is preventing cycles in the graph, i.e., turning it into a tree internal fun H5builder.replaceSymbolicLinks(groupb: H5GroupBuilder) { val objList = groupb.nestedObjects var count = 0 @@ -204,7 +194,7 @@ internal class DataObjectFacade(val parent : H5GroupBuilder?, val name: String) } else if ((local.mdt != null) and (local.mdl != null)) { // has a Datatype and a DataLayout message isVariable = true - // if its an enum or compound, could be a non-shared typedef. Found in non-netcdf4 hdf5 files. + // if DataObject is an enum or compound, could be a non-shared typedef. Found in non-netcdf4 hdf5 files. if (!local.mdt!!.isShared) { if ((local.mdt.type == Datatype5.Enumerated) or (local.mdt.type == Datatype5.Compound)) { isTypedef = true @@ -234,7 +224,7 @@ internal class H5GroupBuilder( ) { val nestedGroupsBuilders = mutableListOf() - // is this a child of that ? + // is this a child of that? fun isChildOf(that: H5GroupBuilder): Boolean { if (this == that) return true if (parent == null) return false @@ -285,7 +275,7 @@ internal class H5GroupBuilder( // gather the H5typedef found in DataObjects if (nested.isTypedef) { val mdt = nested.dataObject!!.mdt!! - // if its a typedef but not a variable, promote to shared + // if DataObject is a typedef but not a variable, promote to shared if (!nested.isVariable) mdt.isShared = true val typename = if (mdt.isShared) nested.dataObject!!.name else null val typedef = H5typedef(typename, mdt) diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Heap.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Heap.kt index 0a35634f..4d6f47c2 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Heap.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Heap.kt @@ -59,7 +59,7 @@ internal class H5heap(val header: H5builder) { Datatype.ULONG, Datatype.ENUM8 -> raf.readArrayOfULong(state, heapId.nelems) Datatype.COMPOUND -> { val members = (datatype.typedef as CompoundTypedef).members - val recsize = members.map { it.nelems * it.datatype.size }.sum() + val recsize = members.sumOf { it.nelems * it.datatype.size } val ba = raf.readByteArray(state, heapId.nelems * recsize) // class ArrayStructureData(shape : IntArray, val ba : ByteArray, val isBE: Boolean, val recsize : Int, val members : List>) val asd = ArrayStructureData(intArrayOf(heapId.nelems), ba, isBE, recsize, members) @@ -112,7 +112,7 @@ internal class H5heap(val header: H5builder) { return HeapIdentifier(globalHeapIdAddress) } - // the heap id is has already been read into a byte array at given pos + // the heap id has already been read into a byte array at given pos fun readHeapIdentifier(bb: ByteArray, pos: Int): HeapIdentifier { return HeapIdentifier(bb, pos) } diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageDataLayout.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageDataLayout.kt index f01f7c3b..25585378 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageDataLayout.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageDataLayout.kt @@ -152,7 +152,7 @@ internal val dataLayoutNames = listOf( // debugging internal fun getDataLayoutCounts() : MutableMap { - return dataLayoutNames.map { Pair(it!!, 0) }.toMap().toMutableMap() + return dataLayoutNames.associate { Pair(it!!, 0) }.toMutableMap() } //////////////////////////////////////////////////////////////////////////////////////// diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/StructDsl.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/StructDsl.kt index ac2117cc..26a62f1a 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/StructDsl.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/StructDsl.kt @@ -56,7 +56,7 @@ internal class StructDsl(val name : String, val ba : ByteArray, val isBE: Boolea fun getByte(fldName : String) : Byte { val fld = fldm[fldName] ?: throw IllegalArgumentException("StructDsl $name has no fld '$fldName'") require(fld.elemSize == 1) { fldName } - return ba.get(fld.pos) + return ba[fld.pos] } fun getShort(fldName : String) : Short { val fld = fldm[fldName] ?: throw IllegalArgumentException("StructDsl $name has no fld '$fldName'") @@ -129,7 +129,7 @@ internal class StructDslBuilder(val name : String, val raf: OpenFileIF, val stat val tstate = state.copy(pos = startPos + from.pos) val ba = raf.readByteArray(tstate, from.elemSize) return when (from.elemSize) { - 1 -> ba.get(0).toInt() + 1 -> ba[0].toInt() 2 -> convertToShort(ba, 0, state.isBE).toInt() 4 -> convertToInt(ba, 0, state.isBE) 8 -> convertToInt(ba, 0, state.isBE) // ignore extra bytes diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/netcdf3/N3builder.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/netcdf3/N3builder.kt index 10e990de..879ee6c5 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/netcdf3/N3builder.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/netcdf3/N3builder.kt @@ -137,7 +137,7 @@ internal class N3header(rafOrg: OpenFileIF, val root: Group.Builder) { } // Must keep dimensions in strict order - for (i in 0 until numdims) { + repeat (numdims) { val name = readString(raf)!! val len = if (isPnetcdf) raf.readLong(filePos) else raf.readInt(filePos).toLong() @@ -165,7 +165,7 @@ internal class N3header(rafOrg: OpenFileIF, val root: Group.Builder) { } // loop over variables - for (i in 0 until nvars) { + repeat (nvars) { val name = readString(raf)!! if (debug) println(" reading variable $name pos=${filePos.pos}") @@ -176,7 +176,7 @@ internal class N3header(rafOrg: OpenFileIF, val root: Group.Builder) { val dims = mutableListOf() val dimIdx = mutableListOf() val dimLengths = mutableListOf() - for (j in 0 until rank) { + repeat (rank) { val dimIndex: Int = if (isPnetcdf) raf.readLong(filePos).toInt() else raf.readInt(filePos) val dim: Dimension = root.dimensions[dimIndex] if (dim == unlimitedDimension) { @@ -227,7 +227,7 @@ internal class N3header(rafOrg: OpenFileIF, val root: Group.Builder) { if (isPnetcdf) raf.readLong(filePos).toInt() else raf.readInt(filePos) } - for (i in 0 until natts) { + repeat (natts) { val name = readString(raf)!! val type: Int = raf.readInt(filePos) val att = if (type == 2) { // CHAR converted to String for Attributes diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/netcdf3/Netcdf3File.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/netcdf3/Netcdf3File.kt index 31f27944..78d341d9 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/netcdf3/Netcdf3File.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/netcdf3/Netcdf3File.kt @@ -47,12 +47,12 @@ internal class Netcdf3File(val filename : String) : Netchdf { return this.dimensions.find { it == header.unlimitedDimension } != null } - override fun chunkIterator(v2: Variable, section: SectionPartial?, maxElements : Int?): Iterator> { + override fun chunkIterator(v2: Variable, wantSection: SectionPartial?, maxElements : Int?): Iterator> { if (v2.nelems == 0L) { return listOf>().iterator() } - val wantSection = SectionPartial.fill(section, v2.shape) - return NCmaxIterator(v2, wantSection, maxElements ?: 100_000) + val section = SectionPartial.fill(wantSection, v2.shape) + return NCmaxIterator(v2, section, maxElements ?: 100_000) } private inner class NCmaxIterator(val v2: Variable, wantSection : Section, maxElems: Int) : AbstractIterator>() { diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibExtra.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibExtra.kt index 8e29580c..79dcdbfb 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibExtra.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibExtra.kt @@ -92,7 +92,7 @@ class NetchdfClibExtra { fun testFilesAfter() { var skip = true files().forEach { filename -> - if (filename.equals(testData + "netchdf/martaan/RADNL_TEST_R___25PCPRR_L3__20090305T120000_20090305T120500_0001.nc")) skip = false + if (filename.equals(testData + "netchdf/bird/watlev_NOAA.F.C_IKE_VIMS_3D_WITHWAVE.nc")) skip = false if (!skip) compareDataWithClib(filename) } } diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibTest.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibTest.kt index 8c77e006..fdf0149f 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibTest.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibTest.kt @@ -27,15 +27,11 @@ class NetchdfClibTest { .flatten().iterator() } - fun beforeAll() { - Stats.clear() // problem with concurrent tests - } - fun afterAll() { if (versions.size > 0) { val sversions = versions.toSortedMap() sversions.keys.forEach{ println("$it = ${sversions[it]!!.size } files") } - val total = sversions.keys.map{ sversions[it]!!.size }.sum() + val total = sversions.keys.sumOf { sversions[it]!!.size } println("total # files = $total") } Stats.show() @@ -43,7 +39,7 @@ class NetchdfClibTest { private val versions = mutableMapOf>() - var compareMiddleSection = false + var compareMiddleSection = true var showDataRead = false var showData = false var showFailedData = false @@ -143,7 +139,7 @@ class NetchdfClibTest { println("netch section $section data=$mysdata") Hdf5ClibFile(filename).use { hcfile -> - println("Hdf5ClibFile ${hcfile!!.type()}\n${hcfile.cdl()}") + println("Hdf5ClibFile ${hcfile.type()}\n${hcfile.cdl()}") val v = hcfile.rootGroup().allVariables().find { it.fullname() == "/SomaticMutation/SomCall" } val ncdata = hcfile.readArrayData(v!!, null) assertTrue (ncdata.equals(mydata)) @@ -216,13 +212,6 @@ isThreadsafe = 0 = false compareDataWithClib(filename, "/GeometricParameters/Data_Fields/SolarAzimuth") } - @Test - fun testFailDataCompare4() { - val filename = testData + "/devcdm/netcdf4/tst_opaque_data.nc4" - CompareCdmWithClib(filename, true) - compareDataWithClib(filename) - } - ////////////////////////////////////////////////////////////////////////////////////////////////////// @Test @@ -292,6 +281,13 @@ isThreadsafe = 0 = false } } + @Test + fun testFailDataCompare() { + val filename = "/home/all/testdata/netchdf/bird/watlev_NOAA.F.C_IKE_VIMS_3D_WITHWAVE.nc" + compareDataWithClib(filename, varname = "time") + } + + @Test fun testCompareDataWithClib() { files().forEach { filename -> @@ -302,8 +298,8 @@ isThreadsafe = 0 = false // @Test fun testFilesAfter() { var skip = true - NetchdfClibExtra.Companion.files().forEach { filename -> - if (filename.equals(testData + "netchdf/martaan/RADNL_TEST_R___25PCPRR_L3__20090305T120000_20090305T120500_0001.nc")) skip = false + files().forEach { filename -> + if (filename.equals(testData + "netchdf/bird/watlev_NOAA.F.C_IKE_VIMS_3D_WITHWAVE.nc")) skip = false if (!skip) compareDataWithClib(filename) } } @@ -350,12 +346,6 @@ fun readNetchdfData(filename: String, varname: String? = null, section: SectionP } } -fun readNcData(filename: String, varname: String? = null, section: SectionPartial? = null, showCdl : Boolean = false) { - NClibFile(filename).use { ncfile -> - readMyData(ncfile, varname, section, showCdl) - } -} - fun compareDataWithClib(filename: String, varname: String? = null, section: SectionPartial? = null, showCdl: Boolean = false) { println("=============================================================") openNetchdfFile(filename).use { netchdf -> @@ -532,7 +522,6 @@ fun compareNetcdfData(myfile: Netchdf, cfile: Netchdf, varname: String?, section } } - fun compareSelectedDataWithClib(filename: String, wanted : (Variable<*>) -> Boolean) { println("=============================================================") openNetchdfFile(filename).use { netchdf -> @@ -590,29 +579,26 @@ fun compareOneVar(myvar: Variable<*>, myfile: Netchdf, cvar : Variable<*>, cfile val totalElems = mydata.shape.computeSize() println(" ${myvar.datatype} ${myvar.fullname()}[${filledSection}] = $totalElems elems" ) - //if (myvar.datatype == Datatype.CHAR) { - // compareCharData(myvar.fullname(), mydata, ncdata) - //} else { - if (!ncdata.equals(mydata)) { - println(" *** FAIL comparing data for variable = ${cvar.datatype} ${cvar.fullname()} ${cvar.dimensions.map { it.name }}") - if (NetchdfClibTest.showFailedData) { - println("\n mydata = $mydata") - println(" cdata = $ncdata") - } - val countDiffs = countArrayDiffs(ncdata, mydata, 10) - println(" *** count values differ = $countDiffs same = ${totalElems - countDiffs}") - assertEquals(0, countDiffs) - // assertEquals(ncdata, mydata, "variable ${myvar.fullname()}") - return - } else { - if (NetchdfClibTest.showData) { - print(" ${cvar.cdl()}, ") - print("\n mydata = $mydata") - print(" cdata = $ncdata") - } + if (!ncdata.equals(mydata)) { + println(" *** FAIL comparing data for variable = ${cvar.datatype} ${cvar.fullname()} ${cvar.dimensions.map { it.name }}") + if (NetchdfClibTest.showFailedData) { + println("\n mydata = $mydata") + println(" cdata = $ncdata") } - // } + val countDiffs = countArrayDiffs(ncdata, mydata, 10) + println(" *** count values differ = $countDiffs same = ${totalElems - countDiffs}") + assertEquals(0, countDiffs) + // assertEquals(ncdata, mydata, "variable ${myvar.fullname()}") + return + } else { + if (NetchdfClibTest.showData) { + print(" ${cvar.cdl()}, ") + print("\n mydata = $mydata") + print(" cdata = $ncdata") + } + } } + if (NetchdfClibTest.compareMiddleSection && cvar.nelems > 8 && cvar.datatype != Datatype.CHAR) { compareMiddleSection(myfile, myvar, cfile, cvar, cvar.shape) } @@ -642,64 +628,19 @@ fun compareMiddleSection(myfile: Netchdf, myvar: Variable<*>, cfile: Netchdf, cv } println(" ${myvar.fullname()}[$middleSection] = ${mydata.shape.contentToString()} ${mydata.shape.computeSize()} elems") - //if (myvar.datatype == Datatype.CHAR) { - // compareCharData(myvar.fullname(), mydata, ncdata) - //} else { - if (!ncdata.equals(mydata)) { - println(" *** FAIL comparing middle section variable = ${cvar.nameAndShape()}") - if (NetchdfClibTest.showFailedData) { - println(" mydata = $mydata") - println(" cdata = $ncdata") - } else { - println("\n countDifferences = ${countArrayDiffs(ncdata, mydata, 10)}") - } - assertTrue(false, "variable ${myvar.name}") - return - } - //} -} -fun compareCharDataOld(name : String, mydata: ArrayTyped<*>, ncdata: ArrayTyped<*>) { - if (!ArrayTyped.valuesEqual(ncdata, mydata)) { - println(" *** FAIL comparing char variable = ${name}") - print(" ncdata = $ncdata") - print(" mydata = $mydata") - assertTrue(false, "variable $name") - } -} - -////////////////////////////////////////////////////////////////////////////////////////////////////// -// just read data from myfile with iterator - -/* -fun readDataIterate(myfile: Netchdf, varname: String? = null, section: SectionPartial? = null, showCdl : Boolean = false) { - - if (showCdl) { - println(myfile.cdl()) - } - if (varname != null) { - val myvar = myfile.rootGroup().allVariables().find { it.fullname() == varname } - if (myvar == null) { - println("cant find $varname") - return - } - readOneVarIterate(myvar, myfile, section) - } else { - myfile.rootGroup().allVariables().forEach { it -> - readOneVarIterate(it, myfile, null) + if (!ncdata.equals(mydata)) { + println(" *** FAIL comparing middle section variable = ${cvar.nameAndShape()}") + if (NetchdfClibTest.showFailedData) { + println(" mydata = $mydata") + println(" cdata = $ncdata") + } else { + println("\n countDifferences = ${countArrayDiffs(ncdata, mydata, 10)}") } + assertTrue(false, "variable ${myvar.name}") + return } } -fun readOneVarIterate(myvar: Variable<*>, myfile: Netchdf, section: SectionPartial?) { - val chunkIter = myfile.chunkIterator(myvar, section, maxBytes) - val sum = AtomicDouble(0.0) - for (pair in chunkIter) { - sumValues(pair.array, sum) - } -} - - */ - ////////////////////////////////////////////////////////////////////////////////////// // compare reading data chunkIterate API with two Netchdf @@ -736,7 +677,7 @@ fun compareOneVarIterate(myvar: Variable<*>, myfile: Netchdf, cvar : Variable<*> var sum1 = 0.0 var countChunks = 0 val time1 = measureNanoTime { - val chunkIter = myfile.chunkIterator(myvar) + val chunkIter = myfile.chunkIterator(myvar, wantSection = section) for (pair in chunkIter) { if (debugIter) println(" compareOneVarIterate myvar=${myvar.name} ${pair.chunkSection} = ${pair.array.shape.contentToString()}") sum1 += sumValues(pair.array) @@ -748,7 +689,7 @@ fun compareOneVarIterate(myvar: Variable<*>, myfile: Netchdf, cvar : Variable<*> var sum2 = 0.0 countChunks = 0 val time2 = measureNanoTime { - val chunkIter = cfile.chunkIterator(cvar) + val chunkIter = cfile.chunkIterator(cvar, wantSection = section) for (pair in chunkIter) { if (debugIter) println(" compareOneVarIterate cvar=${cvar.name} ${pair.chunkSection} = ${pair.array.shape.contentToString()}") sum2 += sumValues(pair.array) @@ -759,36 +700,9 @@ fun compareOneVarIterate(myvar: Variable<*>, myfile: Netchdf, cvar : Variable<*> if (sum1.isFinite() && sum2.isFinite()) { assertTrue(nearlyEquals(sum1, sum2), "$sum1 != $sum2 sum2") - // println("sum = $sum1") } } -/////////////////////////////////////////////////////////// - -/* fun sumValues(array : ArrayTyped<*>) { - if (array is ArraySingle || array is ArrayEmpty) { - return // test fillValue the same ?? - } - // cant cast unsigned to Numbers - val useArray = when (array.datatype) { - Datatype.UBYTE -> ArrayByte(array.shape, (array as ArrayUByte).bb) - Datatype.USHORT -> ArrayShort(array.shape, (array as ArrayUShort).bb) - Datatype.UINT -> ArrayInt(array.shape, (array as ArrayUInt).bb) - Datatype.ULONG -> ArrayLong(array.shape, (array as ArrayULong).bb) - else -> array - } - - if (useArray.datatype.isNumber) { - for (value in useArray) { - val number = (value as Number) - val numberd: Double = number.toDouble() - if (numberd.isFinite()) { - sum.getAndAdd(numberd) - } - } - } -} */ - private fun sumValues(array : ArrayTyped<*>): Double { var result = 0.0 if (array is ArraySingle || array is ArrayEmpty) {