From 053a088012684b47648bc964a32cc0bd15f28bdd Mon Sep 17 00:00:00 2001 From: Bharat Kaurav <117659673+TheNetherWatcher@users.noreply.github.com> Date: Sat, 31 Oct 2026 22:50:35 +0530 Subject: [PATCH] Pack format 16, 24 and 32 samples without per-byte arrays wr_dat_file split every sample into separate int64 arrays for each byte, then interleaved and cast them, using about 4 bytes of memory per byte written. Convert to little-endian 16 or 32 bit integers and view the result as bytes instead (dropping the top byte for format 24), as format 80 already does. The input is no longer copied up front. Formats 80 and 212 now compute their offsets out of place, so the caller's array is still not modified. Closes #464. --- docs/changes.rst | 3 ++ src/wfdb/io/_signal.py | 71 ++++++++-------------------------------- tests/test_record.py | 74 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 58 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 8c63c074..a33a0b21 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -29,6 +29,9 @@ Unreleased **Fix parameter handling in csv_to_wfdb** ``csv_to_wfdb`` now builds the same ``Record`` whether it writes files or returns the record (``record_only=True``), so all header parameters (such as ``dat_file_name``, ``counter_freq``, ``base_counter``, ``skew``, ``byte_offset``, ``adc_res`` and ``adc_zero``) are written instead of being ignored. Single values of any numeric type (including NumPy scalars and floats) are used for all channels, and an explicit ``0`` is no longer replaced by the default. The initial value and checksum are computed from the digital samples, and the default ADC resolution now follows ``fmt`` in both modes. The ``samps_per_frame``, ``init_value`` and ``checksum`` arguments are deprecated and ignored. The parameter descriptions in the docstring have been corrected. +**Reduce memory use when writing format 16, 24 and 32 signals** + ``wrsamp`` and ``wr_dat_file`` now convert signals in formats 16, 24 and 32 to bytes directly, instead of splitting every sample into separate byte arrays. For a 4-hour, 8-channel record at 500 Hz, the peak memory used to pack a format 16 signal drops from about 2 GB to 110 MB, and packing is about 13 times faster. The written files are unchanged. + **Use the last RR interval in compute_hr** ``compute_hr`` now uses the interval between the last two beats for the heart rate after the last beat, instead of repeating the previous rate. With exactly two beats it now returns their rate after the second beat instead of only NaN. diff --git a/src/wfdb/io/_signal.py b/src/wfdb/io/_signal.py index 7bdc5b60..09633f76 100644 --- a/src/wfdb/io/_signal.py +++ b/src/wfdb/io/_signal.py @@ -2320,23 +2320,14 @@ def wr_dat_file( d_signal[:, expand_ch] = e_d_signal[ch][framenum::spf] expand_ch = expand_ch + 1 else: - # Create a copy to prevent overwrite - d_signal = d_signal.copy() - # Non-expanded format always has 1 sample per frame n_sig = d_signal.shape[1] samps_per_frame = [1] * n_sig - # Total number of samples per frame (equal to number of signals if - # expanded=False, but may be greater for expanded=True) - tsamps_per_frame = d_signal.shape[1] - if fmt == "80": - # convert to 8 bit offset binary form - d_signal += 128 - - # Convert to unsigned 8 bit dtype to write (and flatten if necessary) - b_write = d_signal.astype("uint8").reshape(-1) + # Convert to 8 bit offset binary form, and to unsigned 8 bit dtype to + # write (and flatten if necessary) + b_write = (d_signal + 128).astype("uint8").reshape(-1) elif fmt == "212": # Each sample is represented by a 12 bit two's complement @@ -2348,8 +2339,8 @@ def wr_dat_file( # the remaining 8 bits of the second sample). The process is # repeated for each successive pair of samples. - # convert to 12 bit two's complement - d_signal[d_signal < 0] += 4096 + # convert to 12 bit two's complement (as a copy, to keep the input) + d_signal = np.where(d_signal < 0, d_signal + 4096, d_signal) # Concatenate into 1D d_signal = d_signal.reshape(-1) @@ -2382,53 +2373,17 @@ def wr_dat_file( b_write = b_write[:-1] elif fmt == "16": - # convert to 16 bit two's complement - d_signal = d_signal.astype(np.uint16) - - # Split samples into separate bytes using binary masks - b1 = d_signal & [255] * tsamps_per_frame - b2 = (d_signal & [65280] * tsamps_per_frame) >> 8 - # Interweave the bytes so that the same samples' bytes are consecutive - b1 = b1.reshape((-1, 1)) - b2 = b2.reshape((-1, 1)) - b_write = np.concatenate((b1, b2), axis=1) - b_write = b_write.reshape((1, -1))[0] - # Convert to un_signed 8 bit dtype to write - b_write = b_write.astype("uint8") + # Little-endian 16 bit two's complement, written byte by byte + b_write = d_signal.astype("> 8 - b3 = (d_signal & [16711680] * tsamps_per_frame) >> 16 - # Interweave the bytes so that the same samples' bytes are consecutive - b1 = b1.reshape((-1, 1)) - b2 = b2.reshape((-1, 1)) - b3 = b3.reshape((-1, 1)) - b_write = np.concatenate((b1, b2, b3), axis=1) - b_write = b_write.reshape((1, -1))[0] - # Convert to un_signed 8 bit dtype to write - b_write = b_write.astype("uint8") + # Little-endian 32 bit two's complement (as int24 is not an option), + # without the most significant byte of each sample + b_write = d_signal.astype("> 8 - b3 = (d_signal & [16711680] * tsamps_per_frame) >> 16 - b4 = (d_signal & [4278190080] * tsamps_per_frame) >> 24 - # Interweave the bytes so that the same samples' bytes are consecutive - b1 = b1.reshape((-1, 1)) - b2 = b2.reshape((-1, 1)) - b3 = b3.reshape((-1, 1)) - b4 = b4.reshape((-1, 1)) - b_write = np.concatenate((b1, b2, b3, b4), axis=1) - b_write = b_write.reshape((1, -1))[0] - # Convert to un_signed 8 bit dtype to write - b_write = b_write.astype("uint8") + # Little-endian 32 bit two's complement, written byte by byte + b_write = d_signal.astype("> 8) | ((b >> 8) << 4), b & 0xFF]) + if len(samples) % 2: + out = out[:-1] + return bytes(out) + size = {"16": 2, "24": 3, "32": 4}[fmt] + return b"".join(v.to_bytes(size, "little", signed=True) for v in samples) + + +class TestWrDatFile: + """Tests for the bytes written by wr_dat_file.""" + + @pytest.mark.parametrize("fmt", ["80", "212", "16", "24", "32"]) + @pytest.mark.parametrize("sig_len", [1, 2, 7]) + def test_bytes(self, fmt, sig_len, tmp_path): + """Each format is written little-endian, sample by sample.""" + dmin, dmax = wfdb.io._signal._digi_bounds(fmt) + rng = np.random.default_rng(0) + d_signal = rng.integers(dmin, dmax + 1, size=(sig_len, 3)) + d_signal[0] = [dmin, dmax, -1] + original = d_signal.copy() + + wfdb.io._signal.wr_dat_file( + "x.dat", fmt, d_signal, byte_offset=0, write_dir=str(tmp_path) + ) + + written = (tmp_path / "x.dat").read_bytes() + assert written == _pack_reference(fmt, d_signal.reshape(-1)) + np.testing.assert_array_equal(d_signal, original) + + @pytest.mark.parametrize("fmt", ["80", "212", "16", "24", "32"]) + def test_non_contiguous_input(self, fmt, tmp_path): + """Fortran-ordered and sliced signals are written in sample order.""" + dmin, dmax = wfdb.io._signal._digi_bounds(fmt) + rng = np.random.default_rng(1) + full = rng.integers(dmin, dmax + 1, size=(9, 4)) + d_signal = np.asfortranarray(full[:, ::2]) + + wfdb.io._signal.wr_dat_file( + "x.dat", fmt, d_signal, byte_offset=0, write_dir=str(tmp_path) + ) + + written = (tmp_path / "x.dat").read_bytes() + assert written == _pack_reference(fmt, full[:, ::2].reshape(-1)) + + @pytest.mark.parametrize("fmt", ["80", "212", "16", "24", "32"]) + def test_expanded(self, fmt, tmp_path): + """Multi-frequency signals are written frame by frame.""" + e_d_signal = [np.array([1, -2, 3, -4]), np.array([5, -6])] + + wfdb.io._signal.wr_dat_file( + "x.dat", + fmt, + None, + byte_offset=0, + expanded=True, + e_d_signal=e_d_signal, + samps_per_frame=[2, 1], + write_dir=str(tmp_path), + ) + + written = (tmp_path / "x.dat").read_bytes() + assert written == _pack_reference(fmt, [1, -2, 5, 3, -4, -6])