diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml new file mode 100644 index 0000000..b93d9ef --- /dev/null +++ b/.github/workflows/build-binaries.yml @@ -0,0 +1,215 @@ +name: Build binaries from NDI-compress + +# Builds the NDI compression executables from Waltham-Data-Science/NDI-compress +# on Linux, macOS (universal: arm64 + x86_64) and Windows, tests this package +# with them on each platform, and commits them into src/ndicompress/bin/ on the +# branch the workflow ran on. +# +# Which NDI-compress commit to build is recorded in ndi-compress-ref.txt: +# - push a change to that file (on any branch) to rebuild from that commit +# (a change to this workflow rebuilds too), or +# - run this workflow by hand (Actions > Build binaries from NDI-compress > +# Run workflow), optionally naming a branch, tag or commit; the file is +# updated to the commit that was built. +# +# NDI-compress is private, and a workflow's own GITHUB_TOKEN can read only this +# repository, so the build reads NDI-compress with a read-only deploy key: the +# public key is added to NDI-compress (Settings > Deploy keys, write access +# off) and the private key is this repository's secret +# NDI_COMPRESS_DEPLOY_KEY. A deploy key does not expire, belongs to no person, +# and can read that one repository only. +# +# The commit is made with GITHUB_TOKEN, so it does not trigger other workflows +# (test.yml): the tests that matter run here, on the new binaries, before +# anything is committed. + +on: + workflow_dispatch: + inputs: + ndi_compress_ref: + description: "NDI-compress branch, tag or commit to build (blank: the one in ndi-compress-ref.txt)" + required: false + default: "" + push: + paths: + - ndi-compress-ref.txt + - .github/workflows/build-binaries.yml + +# Read-only by default; only the commit job can push. Third-party actions are +# pinned to commits, because whatever runs here ends up in executables that +# every NDI-python install runs. +permissions: + contents: read + +concurrency: + group: build-binaries-${{ github.ref }} + cancel-in-progress: false + +jobs: + resolve: + name: Resolve the NDI-compress commit + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.sha.outputs.sha }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - id: ref + shell: bash + env: + INPUT_REF: ${{ github.event.inputs.ndi_compress_ref }} + HAS_KEY: ${{ secrets.NDI_COMPRESS_DEPLOY_KEY != '' }} + run: | + if [[ "$HAS_KEY" != "true" ]]; then + echo "::error::The repository secret NDI_COMPRESS_DEPLOY_KEY is not set. NDI-compress is private: add a read-only deploy key to it and put the private key in this secret (see the comment at the top of this workflow)." + exit 1 + fi + ref="${INPUT_REF:-$(tr -d '[:space:]' < ndi-compress-ref.txt)}" + echo "ref=$ref" >> "$GITHUB_OUTPUT" + - name: Check out NDI-compress at that ref + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: Waltham-Data-Science/NDI-compress + ref: ${{ steps.ref.outputs.ref }} + ssh-key: ${{ secrets.NDI_COMPRESS_DEPLOY_KEY }} + persist-credentials: false + path: ndi-compress + - id: sha + shell: bash + run: | + sha=$(git -C ndi-compress rev-parse HEAD) + echo "Building NDI-compress ${{ steps.ref.outputs.ref }} = $sha" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + + build: + name: Build and test on ${{ matrix.os }} + needs: resolve + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + include: + # 22.04 rather than latest: the Linux binaries need the glibc of the + # system they are built on, so an older one runs on more systems. + - os: ubuntu-22.04 + bindir: linux + exe: "" + - os: macos-latest + bindir: macos + exe: "" + - os: windows-latest + bindir: windows + exe: ".exe" + defaults: + run: + shell: bash + steps: + - name: Check out this package + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + path: pkg + + - name: Check out NDI-compress + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: Waltham-Data-Science/NDI-compress + ref: ${{ needs.resolve.outputs.sha }} + ssh-key: ${{ secrets.NDI_COMPRESS_DEPLOY_KEY }} + persist-credentials: false + path: ndi-compress + + - uses: jwlawson/actions-setup-cmake@0d6a7d60b009d01c9e7523be22153ff8f19460d3 # v2.2.0 + + - name: Configure (Windows, MinGW) + if: runner.os == 'Windows' + run: cmake -S ndi-compress/C -B build -G "MinGW Makefiles" + + - name: Configure (Linux, macOS) + if: runner.os != 'Windows' + run: cmake -S ndi-compress/C -B build + + - name: Build + run: cmake --build build + + - name: C tests + run: ctest --test-dir build --output-on-failure + + - name: Put the executables in the package + run: | + dest="pkg/src/ndicompress/bin/${{ matrix.bindir }}" + mkdir -p "$dest" + for name in ndi_compress_digital ndi_compress_ephys ndi_compress_time \ + ndi_compress_metadata ndi_compress_eventmarktext \ + ndi_uncompress_digital ndi_uncompress_ephys ndi_uncompress_time \ + ndi_uncompress_metadata ndi_uncompress_eventmarktext \ + compare_json compare_double; do + cp "build/${name}${{ matrix.exe }}" "$dest/" + done + chmod +x "$dest"/* || true + ls -l "$dest" + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.10" + + - name: Test the package with the new executables + if: runner.os != 'Windows' + working-directory: pkg + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + pytest + + # The tools call tar. In Git Bash (this job's default shell) that is + # Git's GNU tar, which reads "C:\..." as a remote host; users, and + # test.yml, get Windows' own tar from PowerShell, so test there. + - name: Test the package with the new executables (Windows) + if: runner.os == 'Windows' + working-directory: pkg + shell: pwsh + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + pytest + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: bin-${{ matrix.bindir }} + path: pkg/src/ndicompress/bin/${{ matrix.bindir }} + if-no-files-found: error + + commit: + name: Commit the executables + needs: [resolve, build] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: bin-* + path: artifacts + + - name: Replace src/ndicompress/bin + run: | + for dir in linux macos windows; do + rm -rf "src/ndicompress/bin/$dir" + mkdir -p "src/ndicompress/bin/$dir" + cp artifacts/bin-$dir/* "src/ndicompress/bin/$dir/" + done + chmod +x src/ndicompress/bin/linux/* src/ndicompress/bin/macos/* + echo "${{ needs.resolve.outputs.sha }}" > ndi-compress-ref.txt + + - name: Commit and push + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add src/ndicompress/bin ndi-compress-ref.txt + if git diff --cached --quiet; then + echo "The binaries are already those of NDI-compress ${{ needs.resolve.outputs.sha }}." + exit 0 + fi + git commit -m "Binaries from NDI-compress ${{ needs.resolve.outputs.sha }}" \ + -m "Built and tested on Linux, macOS and Windows by the build-binaries workflow (run ${{ github.run_id }})." + git push origin "HEAD:${{ github.ref_name }}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6a39f9e..a2f3b0c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,6 +5,9 @@ on: branches: [ main, master ] pull_request: branches: [ main, master ] + # Run by hand on any branch, e.g. after build-binaries commits new + # executables (a commit made by a workflow starts no other workflow). + workflow_dispatch: jobs: test: diff --git a/README.md b/README.md index bc30af0..4b28cae 100644 --- a/README.md +++ b/README.md @@ -34,3 +34,26 @@ ndi_compress.compress_digital(data, "my_data") # Expand data_out, ratio, output_files = ndi_compress.expand_digital("my_data.nbf.tgz") ``` + +## The bundled executables + +The compression tools are C programs from +[NDI-compress](https://github.com/Waltham-Data-Science/NDI-compress). This +package ships them prebuilt in `src/ndicompress/bin/{linux,macos,windows}`, so +nothing is compiled on install. `ndi-compress-ref.txt` names the NDI-compress +commit they were built from. + +To update them after NDI-compress changes, either change +`ndi-compress-ref.txt` to the new commit (or to a branch or tag name) and +push, or run the **Build binaries +from NDI-compress** workflow (Actions tab), optionally naming a branch, tag or +commit. The workflow builds the tools on Linux, macOS (universal) and Windows, +runs this package's tests with them on each, and commits them, with the updated +`ndi-compress-ref.txt`, to the branch it ran on. A commit made by a workflow starts no +other workflow, so afterwards run **Test NDI Compress** on that branch from the +Actions tab (or push anything) to test the new head. + +NDI-compress is private, so the workflow reads it with a read-only deploy key: +add the public key to NDI-compress (Settings > Deploy keys, without write +access) and store the private key as this repository's Actions secret +`NDI_COMPRESS_DEPLOY_KEY`. diff --git a/ndi-compress-ref.txt b/ndi-compress-ref.txt new file mode 100644 index 0000000..5622f0f --- /dev/null +++ b/ndi-compress-ref.txt @@ -0,0 +1 @@ +650b3080165fb7c17683e2c646915731e9acf0cf diff --git a/src/ndicompress/bin/linux/compare_double b/src/ndicompress/bin/linux/compare_double index fa0aac1..701a5f4 100755 Binary files a/src/ndicompress/bin/linux/compare_double and b/src/ndicompress/bin/linux/compare_double differ diff --git a/src/ndicompress/bin/linux/compare_json b/src/ndicompress/bin/linux/compare_json index 2f4b08d..78961fe 100755 Binary files a/src/ndicompress/bin/linux/compare_json and b/src/ndicompress/bin/linux/compare_json differ diff --git a/src/ndicompress/bin/linux/ndi_compress_digital b/src/ndicompress/bin/linux/ndi_compress_digital index c50fe45..570d218 100755 Binary files a/src/ndicompress/bin/linux/ndi_compress_digital and b/src/ndicompress/bin/linux/ndi_compress_digital differ diff --git a/src/ndicompress/bin/linux/ndi_compress_ephys b/src/ndicompress/bin/linux/ndi_compress_ephys index e568463..9651a37 100755 Binary files a/src/ndicompress/bin/linux/ndi_compress_ephys and b/src/ndicompress/bin/linux/ndi_compress_ephys differ diff --git a/src/ndicompress/bin/linux/ndi_compress_eventmarktext b/src/ndicompress/bin/linux/ndi_compress_eventmarktext index 2aa060c..187e19c 100755 Binary files a/src/ndicompress/bin/linux/ndi_compress_eventmarktext and b/src/ndicompress/bin/linux/ndi_compress_eventmarktext differ diff --git a/src/ndicompress/bin/linux/ndi_compress_metadata b/src/ndicompress/bin/linux/ndi_compress_metadata index 43833ff..e09153e 100755 Binary files a/src/ndicompress/bin/linux/ndi_compress_metadata and b/src/ndicompress/bin/linux/ndi_compress_metadata differ diff --git a/src/ndicompress/bin/linux/ndi_compress_time b/src/ndicompress/bin/linux/ndi_compress_time index 25a5a12..69fd7c5 100755 Binary files a/src/ndicompress/bin/linux/ndi_compress_time and b/src/ndicompress/bin/linux/ndi_compress_time differ diff --git a/src/ndicompress/bin/linux/ndi_uncompress_digital b/src/ndicompress/bin/linux/ndi_uncompress_digital index f6e2fc7..061578f 100755 Binary files a/src/ndicompress/bin/linux/ndi_uncompress_digital and b/src/ndicompress/bin/linux/ndi_uncompress_digital differ diff --git a/src/ndicompress/bin/linux/ndi_uncompress_ephys b/src/ndicompress/bin/linux/ndi_uncompress_ephys index be9941d..efa46c9 100755 Binary files a/src/ndicompress/bin/linux/ndi_uncompress_ephys and b/src/ndicompress/bin/linux/ndi_uncompress_ephys differ diff --git a/src/ndicompress/bin/linux/ndi_uncompress_eventmarktext b/src/ndicompress/bin/linux/ndi_uncompress_eventmarktext index 1320b80..0f99ffc 100755 Binary files a/src/ndicompress/bin/linux/ndi_uncompress_eventmarktext and b/src/ndicompress/bin/linux/ndi_uncompress_eventmarktext differ diff --git a/src/ndicompress/bin/linux/ndi_uncompress_metadata b/src/ndicompress/bin/linux/ndi_uncompress_metadata index 59c1a39..ce915c5 100755 Binary files a/src/ndicompress/bin/linux/ndi_uncompress_metadata and b/src/ndicompress/bin/linux/ndi_uncompress_metadata differ diff --git a/src/ndicompress/bin/linux/ndi_uncompress_time b/src/ndicompress/bin/linux/ndi_uncompress_time index e2d9c38..53be843 100755 Binary files a/src/ndicompress/bin/linux/ndi_uncompress_time and b/src/ndicompress/bin/linux/ndi_uncompress_time differ diff --git a/src/ndicompress/bin/macos/compare_double b/src/ndicompress/bin/macos/compare_double index a34bb74..9bf44c5 100755 Binary files a/src/ndicompress/bin/macos/compare_double and b/src/ndicompress/bin/macos/compare_double differ diff --git a/src/ndicompress/bin/macos/compare_json b/src/ndicompress/bin/macos/compare_json index e733e55..b8112ff 100755 Binary files a/src/ndicompress/bin/macos/compare_json and b/src/ndicompress/bin/macos/compare_json differ diff --git a/src/ndicompress/bin/macos/ndi_compress_digital b/src/ndicompress/bin/macos/ndi_compress_digital index 8a75cd4..c40a8ed 100755 Binary files a/src/ndicompress/bin/macos/ndi_compress_digital and b/src/ndicompress/bin/macos/ndi_compress_digital differ diff --git a/src/ndicompress/bin/macos/ndi_compress_ephys b/src/ndicompress/bin/macos/ndi_compress_ephys index c3a3354..00ce8c0 100755 Binary files a/src/ndicompress/bin/macos/ndi_compress_ephys and b/src/ndicompress/bin/macos/ndi_compress_ephys differ diff --git a/src/ndicompress/bin/macos/ndi_compress_eventmarktext b/src/ndicompress/bin/macos/ndi_compress_eventmarktext index c54563e..734960f 100755 Binary files a/src/ndicompress/bin/macos/ndi_compress_eventmarktext and b/src/ndicompress/bin/macos/ndi_compress_eventmarktext differ diff --git a/src/ndicompress/bin/macos/ndi_compress_metadata b/src/ndicompress/bin/macos/ndi_compress_metadata index f1c4a06..393b563 100755 Binary files a/src/ndicompress/bin/macos/ndi_compress_metadata and b/src/ndicompress/bin/macos/ndi_compress_metadata differ diff --git a/src/ndicompress/bin/macos/ndi_compress_time b/src/ndicompress/bin/macos/ndi_compress_time index 6c87fce..3d6409e 100755 Binary files a/src/ndicompress/bin/macos/ndi_compress_time and b/src/ndicompress/bin/macos/ndi_compress_time differ diff --git a/src/ndicompress/bin/macos/ndi_uncompress_digital b/src/ndicompress/bin/macos/ndi_uncompress_digital index 34b1a13..9c64412 100755 Binary files a/src/ndicompress/bin/macos/ndi_uncompress_digital and b/src/ndicompress/bin/macos/ndi_uncompress_digital differ diff --git a/src/ndicompress/bin/macos/ndi_uncompress_ephys b/src/ndicompress/bin/macos/ndi_uncompress_ephys index 8dd473a..49dc90b 100755 Binary files a/src/ndicompress/bin/macos/ndi_uncompress_ephys and b/src/ndicompress/bin/macos/ndi_uncompress_ephys differ diff --git a/src/ndicompress/bin/macos/ndi_uncompress_eventmarktext b/src/ndicompress/bin/macos/ndi_uncompress_eventmarktext index 19aa0af..344b0f3 100755 Binary files a/src/ndicompress/bin/macos/ndi_uncompress_eventmarktext and b/src/ndicompress/bin/macos/ndi_uncompress_eventmarktext differ diff --git a/src/ndicompress/bin/macos/ndi_uncompress_metadata b/src/ndicompress/bin/macos/ndi_uncompress_metadata index c768ed1..e1049fe 100755 Binary files a/src/ndicompress/bin/macos/ndi_uncompress_metadata and b/src/ndicompress/bin/macos/ndi_uncompress_metadata differ diff --git a/src/ndicompress/bin/macos/ndi_uncompress_time b/src/ndicompress/bin/macos/ndi_uncompress_time index 2f8ac89..e63dc42 100755 Binary files a/src/ndicompress/bin/macos/ndi_uncompress_time and b/src/ndicompress/bin/macos/ndi_uncompress_time differ diff --git a/src/ndicompress/bin/windows/compare_double.exe b/src/ndicompress/bin/windows/compare_double.exe index a8362a3..cbe0606 100644 Binary files a/src/ndicompress/bin/windows/compare_double.exe and b/src/ndicompress/bin/windows/compare_double.exe differ diff --git a/src/ndicompress/bin/windows/compare_json.exe b/src/ndicompress/bin/windows/compare_json.exe index d7e28cc..8cf0254 100644 Binary files a/src/ndicompress/bin/windows/compare_json.exe and b/src/ndicompress/bin/windows/compare_json.exe differ diff --git a/src/ndicompress/bin/windows/ndi_compress_digital.exe b/src/ndicompress/bin/windows/ndi_compress_digital.exe index 2ea91f0..1bb3b0d 100644 Binary files a/src/ndicompress/bin/windows/ndi_compress_digital.exe and b/src/ndicompress/bin/windows/ndi_compress_digital.exe differ diff --git a/src/ndicompress/bin/windows/ndi_compress_ephys.exe b/src/ndicompress/bin/windows/ndi_compress_ephys.exe index f329606..c4306dc 100644 Binary files a/src/ndicompress/bin/windows/ndi_compress_ephys.exe and b/src/ndicompress/bin/windows/ndi_compress_ephys.exe differ diff --git a/src/ndicompress/bin/windows/ndi_compress_eventmarktext.exe b/src/ndicompress/bin/windows/ndi_compress_eventmarktext.exe index b7e62e4..c1198c6 100644 Binary files a/src/ndicompress/bin/windows/ndi_compress_eventmarktext.exe and b/src/ndicompress/bin/windows/ndi_compress_eventmarktext.exe differ diff --git a/src/ndicompress/bin/windows/ndi_compress_metadata.exe b/src/ndicompress/bin/windows/ndi_compress_metadata.exe index fb3b044..a2ff309 100644 Binary files a/src/ndicompress/bin/windows/ndi_compress_metadata.exe and b/src/ndicompress/bin/windows/ndi_compress_metadata.exe differ diff --git a/src/ndicompress/bin/windows/ndi_compress_time.exe b/src/ndicompress/bin/windows/ndi_compress_time.exe index b0d0d5b..b78da5b 100644 Binary files a/src/ndicompress/bin/windows/ndi_compress_time.exe and b/src/ndicompress/bin/windows/ndi_compress_time.exe differ diff --git a/src/ndicompress/bin/windows/ndi_uncompress_digital.exe b/src/ndicompress/bin/windows/ndi_uncompress_digital.exe index 2f51121..9af3499 100644 Binary files a/src/ndicompress/bin/windows/ndi_uncompress_digital.exe and b/src/ndicompress/bin/windows/ndi_uncompress_digital.exe differ diff --git a/src/ndicompress/bin/windows/ndi_uncompress_ephys.exe b/src/ndicompress/bin/windows/ndi_uncompress_ephys.exe index d50a1d1..fd3d698 100644 Binary files a/src/ndicompress/bin/windows/ndi_uncompress_ephys.exe and b/src/ndicompress/bin/windows/ndi_uncompress_ephys.exe differ diff --git a/src/ndicompress/bin/windows/ndi_uncompress_eventmarktext.exe b/src/ndicompress/bin/windows/ndi_uncompress_eventmarktext.exe index 15dc86c..df282b3 100644 Binary files a/src/ndicompress/bin/windows/ndi_uncompress_eventmarktext.exe and b/src/ndicompress/bin/windows/ndi_uncompress_eventmarktext.exe differ diff --git a/src/ndicompress/bin/windows/ndi_uncompress_metadata.exe b/src/ndicompress/bin/windows/ndi_uncompress_metadata.exe index b273071..7d408b5 100644 Binary files a/src/ndicompress/bin/windows/ndi_uncompress_metadata.exe and b/src/ndicompress/bin/windows/ndi_uncompress_metadata.exe differ diff --git a/src/ndicompress/bin/windows/ndi_uncompress_time.exe b/src/ndicompress/bin/windows/ndi_uncompress_time.exe index 1f5516a..9e41b97 100644 Binary files a/src/ndicompress/bin/windows/ndi_uncompress_time.exe and b/src/ndicompress/bin/windows/ndi_uncompress_time.exe differ diff --git a/src/ndicompress/compress.py b/src/ndicompress/compress.py index 149f99b..d3ae990 100644 --- a/src/ndicompress/compress.py +++ b/src/ndicompress/compress.py @@ -165,16 +165,31 @@ def expand_digital(fullfilename): return data, None, params -def compress_ephys(data, fullfilename): +def compress_ephys(data, fullfilename, *, output_size_bits=None, + data_is_unsigned=None, data_format=None): """ Compress ephys data. + Mirrors MATLAB's ndi.compress.compress_ephys(D, OUTPUT_SIZE_BITS, + FULLFILENAME, 'data_is_unsigned', ..., 'data_format', ...). + Parameters ---------- data : numpy.ndarray - Shape (S, C). Should be convertible to int16. + Shape (S, C), where S is samples and C is channels. fullfilename : str Output filename base. + output_size_bits : int, optional + Bits per sample recorded as the original sample size (MATLAB's + OUTPUT_SIZE_BITS). Defaults to the size of the input samples. + data_is_unsigned : bool, optional + Whether the samples are unsigned. Defaults to the signedness of an + integer dtype. + data_format : {'integer', 'float'}, optional + 'integer' predicts and stores integer samples; 'float' stores 64-bit + floats as MATLAB's float path does. Defaults to 'integer' for integer + dtypes. When no format is given and ``data`` is not an integer array, + it is converted to int16, as this function always did. Returns ------- @@ -186,20 +201,84 @@ def compress_ephys(data, fullfilename): if not isinstance(data, np.ndarray): data = np.array(data) - # C executable expects int16 binary input - data_int16 = data.astype(np.int16) + if data_format is None: + if np.issubdtype(data.dtype, np.integer): + data_format = 'integer' + else: + # what this function always did with non-integer input + data = data.astype(np.int16) + data_format = 'integer' + if data_format not in ('integer', 'float'): + raise ValueError("data_format must be 'integer' or 'float'") + + if data_format == 'float': + if data.dtype not in (np.float32, np.float64): + data = data.astype(np.float64) + if data_is_unsigned is None: + data_is_unsigned = False + else: + if not np.issubdtype(data.dtype, np.integer): + raise TypeError("data_format 'integer' requires an integer array; " + "convert scaled data first (see scaled2underlying)") + if data_is_unsigned is None: + data_is_unsigned = np.issubdtype(data.dtype, np.unsignedinteger) + + data_bits = data.dtype.itemsize * 8 + if output_size_bits is None: + output_size_bits = data_bits + in_size = data.nbytes with tempfile.TemporaryDirectory() as temp_dir: - bin_path = _write_temp_bin_json(data_int16, temp_dir) + bin_path = _write_temp_bin_json(data, temp_dir) + json_path = os.path.join(temp_dir, "input.json") + with open(json_path) as f: + header = json.load(f) + header.update({ + "data_format": data_format, + "data_is_unsigned": bool(data_is_unsigned), + "data_bits_per_sample": int(data_bits), + "output_size_bits": int(output_size_bits), + }) + with open(json_path, 'w') as f: + json.dump(header, f) _call_c_exec("ndi_compress_ephys", [bin_path, fullfilename]) outfile = fullfilename + ".nbf.tgz" + _check_ephys_header(outfile, data_format, data_is_unsigned, output_size_bits) out_size = os.path.getsize(outfile) if os.path.exists(outfile) else 0 ratio = float(out_size) / float(in_size) if in_size > 0 else 0.0 return ratio, None, None, None + +def _check_ephys_header(outfile, data_format, data_is_unsigned, output_size_bits): + """Raise if the encoder did not record the requested format. + + An ndi_compress_ephys built before the data-format keys existed ignores + them and stores the samples as signed 16-bit integers; this catches a + stale binary instead of letting it write wrong data. + """ + with tempfile.TemporaryDirectory() as temp_dir: + with tarfile.open(outfile, "r:gz") as tar: + member = next((m for m in tar.getmembers() if m.name.endswith('.nbh') + and not os.path.basename(m.name).startswith('._')), None) + if member is None: + raise ValueError("No .nbh file found") + tar.extract(member, path=temp_dir) + params = read_ndi_header(os.path.join(temp_dir, member.name)) + expected_format = 2 if data_format == 'float' else 1 + got = (params.get('original_format'), params.get('original_isunsigned'), + params.get('original_bits_per_sample')) + want = (expected_format, int(bool(data_is_unsigned)), int(output_size_bits)) + if tuple(int(g) for g in got) != want: + os.remove(outfile) + raise RuntimeError( + "ndi_compress_ephys wrote (original_format, original_isunsigned, " + f"original_bits_per_sample) = {got}, not the requested {want}; " + "the ndi_compress_ephys binary predates the data format options " + "and must be updated.") + def expand_ephys(fullfilename): """ Expand ephys data. diff --git a/tests/test_ndi_compress.py b/tests/test_ndi_compress.py index 32b03dd..807b9e4 100644 --- a/tests/test_ndi_compress.py +++ b/tests/test_ndi_compress.py @@ -57,6 +57,51 @@ def test_ephys(self): self.assertLess(max_diff, 1e-7) self.assertEqual(data_out.shape, (S, C)) + def _ephys_round_trip(self, data, name, **kwargs): + filename = name + self.files_to_remove.append(filename + ".nbf.tgz") + ndi_compress.compress_ephys(data, filename, **kwargs) + data_out, _ = ndi_compress.expand_ephys(filename + ".nbf.tgz") + self.assertEqual(data_out.shape, data.shape) + return data_out + + def test_ephys_dtypes_round_trip(self): + """Integer dtypes keep their signedness and size, as in MATLAB.""" + S, C = 500, 3 + rng = np.random.default_rng(0) + for dtype in (np.int8, np.uint8, np.int16, np.uint16, np.int32): + info = np.iinfo(dtype) + data = rng.integers(info.min, int(info.max) + 1, size=(S, C)).astype(dtype) + data_out = self._ephys_round_trip(data, "test_ephys_" + np.dtype(dtype).name) + np.testing.assert_array_equal(data_out, data.astype(np.float64), + err_msg=np.dtype(dtype).name) + + def test_ephys_uint16_header(self): + """uint16 samples (e.g. Intan amplifier data) are stored as unsigned 16-bit.""" + data = np.array([[0, 65535], [32768, 1], [100, 65000]], dtype=np.uint16) + filename = "test_ephys_uint16_header" + self._ephys_round_trip(data, filename) + import tarfile, tempfile + from ndicompress.header import read_ndi_header + with tempfile.TemporaryDirectory() as d: + with tarfile.open(filename + ".nbf.tgz", "r:gz") as tar: + m = [x for x in tar.getmembers() if x.name.endswith('.nbh')][0] + tar.extract(m, path=d) + params = read_ndi_header(os.path.join(d, m.name)) + self.assertEqual(params['original_isunsigned'], 1) + self.assertEqual(params['original_bits_per_sample'], 16) + + def test_ephys_float(self): + data = np.random.default_rng(1).standard_normal((200, 2)) + data_out = self._ephys_round_trip(data, "test_ephys_float", data_format='float') + np.testing.assert_array_equal(data_out, data) + + def test_ephys_float_without_format_is_int16(self): + """Non-integer data with no data_format is converted to int16, as before.""" + data = np.array([[1.4, -2.6], [3.0, 4.2]]) + data_out = self._ephys_round_trip(data, "test_ephys_legacy") + np.testing.assert_array_equal(data_out, data.astype(np.int16).astype(np.float64)) + def test_time(self): S, C = 100, 1 t = np.linspace(0, 10, S)