compress_ephys: pass the data format (unsigned, 8/32/64-bit, float)

.github/workflows/build-binaries.yml

@@ -0,0 +1,215 @@
+name: Build binaries from NDI-compress
+
+# Builds the NDI compression executables from Waltham-Data-Science/NDI-compress
+# on Linux, macOS (universal: arm64 + x86_64) and Windows, tests this package
+# with them on each platform, and commits them into src/ndicompress/bin/ on the
+# branch the workflow ran on.
+#
+# Which NDI-compress commit to build is recorded in ndi-compress-ref.txt:
+#   - push a change to that file (on any branch) to rebuild from that commit
+#     (a change to this workflow rebuilds too), or
+#   - run this workflow by hand (Actions > Build binaries from NDI-compress >
+#     Run workflow), optionally naming a branch, tag or commit; the file is
+#     updated to the commit that was built.
+#
+# NDI-compress is private, and a workflow's own GITHUB_TOKEN can read only this
+# repository, so the build reads NDI-compress with a read-only deploy key: the
+# public key is added to NDI-compress (Settings > Deploy keys, write access
+# off) and the private key is this repository's secret
+# NDI_COMPRESS_DEPLOY_KEY. A deploy key does not expire, belongs to no person,
+# and can read that one repository only.
+#
+# The commit is made with GITHUB_TOKEN, so it does not trigger other workflows
+# (test.yml): the tests that matter run here, on the new binaries, before
+# anything is committed.
+
+on:
+  workflow_dispatch:
+    inputs:
+      ndi_compress_ref:
+        description: "NDI-compress branch, tag or commit to build (blank: the one in ndi-compress-ref.txt)"
+        required: false
+        default: ""
+  push:
+    paths:
+      - ndi-compress-ref.txt
+      - .github/workflows/build-binaries.yml
+
+# Read-only by default; only the commit job can push. Third-party actions are
+# pinned to commits, because whatever runs here ends up in executables that
+# every NDI-python install runs.
+permissions:
+  contents: read
+
+concurrency:
+  group: build-binaries-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  resolve:
+    name: Resolve the NDI-compress commit
+    runs-on: ubuntu-latest
+    outputs:
+      sha: ${{ steps.sha.outputs.sha }}
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - id: ref
+        shell: bash
+        env:
+          INPUT_REF: ${{ github.event.inputs.ndi_compress_ref }}
+          HAS_KEY: ${{ secrets.NDI_COMPRESS_DEPLOY_KEY != '' }}
+        run: |
+          if [[ "$HAS_KEY" != "true" ]]; then
+            echo "::error::The repository secret NDI_COMPRESS_DEPLOY_KEY is not set. NDI-compress is private: add a read-only deploy key to it and put the private key in this secret (see the comment at the top of this workflow)."
+            exit 1
+          fi
+          ref="${INPUT_REF:-$(tr -d '[:space:]' < ndi-compress-ref.txt)}"
+          echo "ref=$ref" >> "$GITHUB_OUTPUT"
+      - name: Check out NDI-compress at that ref
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          repository: Waltham-Data-Science/NDI-compress
+          ref: ${{ steps.ref.outputs.ref }}
+          ssh-key: ${{ secrets.NDI_COMPRESS_DEPLOY_KEY }}
+          persist-credentials: false
+          path: ndi-compress
+      - id: sha
+        shell: bash
+        run: |
+          sha=$(git -C ndi-compress rev-parse HEAD)
+          echo "Building NDI-compress ${{ steps.ref.outputs.ref }} = $sha"
+          echo "sha=$sha" >> "$GITHUB_OUTPUT"
+
+  build:
+    name: Build and test on ${{ matrix.os }}
+    needs: resolve
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          # 22.04 rather than latest: the Linux binaries need the glibc of the
+          # system they are built on, so an older one runs on more systems.
+          - os: ubuntu-22.04
+            bindir: linux
+            exe: ""
+          - os: macos-latest
+            bindir: macos
+            exe: ""
+          - os: windows-latest
+            bindir: windows
+            exe: ".exe"
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Check out this package
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          path: pkg
+
+      - name: Check out NDI-compress
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          repository: Waltham-Data-Science/NDI-compress
+          ref: ${{ needs.resolve.outputs.sha }}
+          ssh-key: ${{ secrets.NDI_COMPRESS_DEPLOY_KEY }}
+          persist-credentials: false
+          path: ndi-compress
+
+      - uses: jwlawson/actions-setup-cmake@0d6a7d60b009d01c9e7523be22153ff8f19460d3 # v2.2.0
+
+      - name: Configure (Windows, MinGW)
+        if: runner.os == 'Windows'
+        run: cmake -S ndi-compress/C -B build -G "MinGW Makefiles"
+
+      - name: Configure (Linux, macOS)
+        if: runner.os != 'Windows'
+        run: cmake -S ndi-compress/C -B build
+
+      - name: Build
+        run: cmake --build build
+
+      - name: C tests
+        run: ctest --test-dir build --output-on-failure
+
+      - name: Put the executables in the package
+        run: |
+          dest="pkg/src/ndicompress/bin/${{ matrix.bindir }}"
+          mkdir -p "$dest"
+          for name in ndi_compress_digital ndi_compress_ephys ndi_compress_time \
+                      ndi_compress_metadata ndi_compress_eventmarktext \
+                      ndi_uncompress_digital ndi_uncompress_ephys ndi_uncompress_time \
+                      ndi_uncompress_metadata ndi_uncompress_eventmarktext \
+                      compare_json compare_double; do
+            cp "build/${name}${{ matrix.exe }}" "$dest/"
+          done
+          chmod +x "$dest"/* || true
+          ls -l "$dest"
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.10"
+
+      - name: Test the package with the new executables
+        if: runner.os != 'Windows'
+        working-directory: pkg
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+          pytest
+
+      # The tools call tar. In Git Bash (this job's default shell) that is
+      # Git's GNU tar, which reads "C:\..." as a remote host; users, and
+      # test.yml, get Windows' own tar from PowerShell, so test there.
+      - name: Test the package with the new executables (Windows)
+        if: runner.os == 'Windows'
+        working-directory: pkg
+        shell: pwsh
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+          pytest
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: bin-${{ matrix.bindir }}
+          path: pkg/src/ndicompress/bin/${{ matrix.bindir }}
+          if-no-files-found: error
+
+  commit:
+    name: Commit the executables
+    needs: [resolve, build]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          pattern: bin-*
+          path: artifacts
+
+      - name: Replace src/ndicompress/bin
+        run: |
+          for dir in linux macos windows; do
+            rm -rf "src/ndicompress/bin/$dir"
+            mkdir -p "src/ndicompress/bin/$dir"
+            cp artifacts/bin-$dir/* "src/ndicompress/bin/$dir/"
+          done
+          chmod +x src/ndicompress/bin/linux/* src/ndicompress/bin/macos/*
+          echo "${{ needs.resolve.outputs.sha }}" > ndi-compress-ref.txt
+
+      - name: Commit and push
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add src/ndicompress/bin ndi-compress-ref.txt
+          if git diff --cached --quiet; then
+            echo "The binaries are already those of NDI-compress ${{ needs.resolve.outputs.sha }}."
+            exit 0
+          fi
+          git commit -m "Binaries from NDI-compress ${{ needs.resolve.outputs.sha }}" \
+                     -m "Built and tested on Linux, macOS and Windows by the build-binaries workflow (run ${{ github.run_id }})."
+          git push origin "HEAD:${{ github.ref_name }}"

.github/workflows/test.yml

@@ -5,6 +5,9 @@ on:
     branches: [ main, master ]
   pull_request:
     branches: [ main, master ]
+  # Run by hand on any branch, e.g. after build-binaries commits new
+  # executables (a commit made by a workflow starts no other workflow).
+  workflow_dispatch:
 
 jobs:
   test:

README.md

@@ -34,3 +34,26 @@ ndi_compress.compress_digital(data, "my_data")
 # Expand
 data_out, ratio, output_files = ndi_compress.expand_digital("my_data.nbf.tgz")
 ```
+
+## The bundled executables
+
+The compression tools are C programs from
+[NDI-compress](https://github.com/Waltham-Data-Science/NDI-compress). This
+package ships them prebuilt in `src/ndicompress/bin/{linux,macos,windows}`, so
+nothing is compiled on install. `ndi-compress-ref.txt` names the NDI-compress
+commit they were built from.
+
+To update them after NDI-compress changes, either change
+`ndi-compress-ref.txt` to the new commit (or to a branch or tag name) and
+push, or run the **Build binaries
+from NDI-compress** workflow (Actions tab), optionally naming a branch, tag or
+commit. The workflow builds the tools on Linux, macOS (universal) and Windows,
+runs this package's tests with them on each, and commits them, with the updated
+`ndi-compress-ref.txt`, to the branch it ran on. A commit made by a workflow starts no
+other workflow, so afterwards run **Test NDI Compress** on that branch from the
+Actions tab (or push anything) to test the new head.
+
+NDI-compress is private, so the workflow reads it with a read-only deploy key:
+add the public key to NDI-compress (Settings > Deploy keys, without write
+access) and store the private key as this repository's Actions secret
+`NDI_COMPRESS_DEPLOY_KEY`.

ndi-compress-ref.txt

@@ -0,0 +1 @@
+650b3080165fb7c17683e2c646915731e9acf0cf

src/ndicompress/compress.py

@@ -165,16 +165,31 @@ def expand_digital(fullfilename):
 
         return data, None, params
 
-def compress_ephys(data, fullfilename):
+def compress_ephys(data, fullfilename, *, output_size_bits=None,
+                   data_is_unsigned=None, data_format=None):
     """
     Compress ephys data.
 
+    Mirrors MATLAB's ndi.compress.compress_ephys(D, OUTPUT_SIZE_BITS,
+    FULLFILENAME, 'data_is_unsigned', ..., 'data_format', ...).
+
     Parameters
     ----------
     data : numpy.ndarray
-        Shape (S, C). Should be convertible to int16.
+        Shape (S, C), where S is samples and C is channels.
     fullfilename : str
         Output filename base.
+    output_size_bits : int, optional
+        Bits per sample recorded as the original sample size (MATLAB's
+        OUTPUT_SIZE_BITS). Defaults to the size of the input samples.
+    data_is_unsigned : bool, optional
+        Whether the samples are unsigned. Defaults to the signedness of an
+        integer dtype.
+    data_format : {'integer', 'float'}, optional
+        'integer' predicts and stores integer samples; 'float' stores 64-bit
+        floats as MATLAB's float path does. Defaults to 'integer' for integer
+        dtypes. When no format is given and ``data`` is not an integer array,
+        it is converted to int16, as this function always did.
 
     Returns
     -------
@@ -186,20 +201,84 @@ def compress_ephys(data, fullfilename):
     if not isinstance(data, np.ndarray):
         data = np.array(data)
 
-    # C executable expects int16 binary input
-    data_int16 = data.astype(np.int16)
+    if data_format is None:
+        if np.issubdtype(data.dtype, np.integer):
+            data_format = 'integer'
+        else:
+            # what this function always did with non-integer input
+            data = data.astype(np.int16)
+            data_format = 'integer'
+    if data_format not in ('integer', 'float'):
+        raise ValueError("data_format must be 'integer' or 'float'")
+
+    if data_format == 'float':
+        if data.dtype not in (np.float32, np.float64):
+            data = data.astype(np.float64)
+        if data_is_unsigned is None:
+            data_is_unsigned = False
+    else:
+        if not np.issubdtype(data.dtype, np.integer):
+            raise TypeError("data_format 'integer' requires an integer array; "
+                            "convert scaled data first (see scaled2underlying)")
+        if data_is_unsigned is None:
+            data_is_unsigned = np.issubdtype(data.dtype, np.unsignedinteger)
+
+    data_bits = data.dtype.itemsize * 8
+    if output_size_bits is None:
+        output_size_bits = data_bits
+
     in_size = data.nbytes
 
     with tempfile.TemporaryDirectory() as temp_dir:
-        bin_path = _write_temp_bin_json(data_int16, temp_dir)
+        bin_path = _write_temp_bin_json(data, temp_dir)
+        json_path = os.path.join(temp_dir, "input.json")
+        with open(json_path) as f:
+            header = json.load(f)
+        header.update({
+            "data_format": data_format,
+            "data_is_unsigned": bool(data_is_unsigned),
+            "data_bits_per_sample": int(data_bits),
+            "output_size_bits": int(output_size_bits),
+        })
+        with open(json_path, 'w') as f:
+            json.dump(header, f)
         _call_c_exec("ndi_compress_ephys", [bin_path, fullfilename])
 
     outfile = fullfilename + ".nbf.tgz"
+    _check_ephys_header(outfile, data_format, data_is_unsigned, output_size_bits)
     out_size = os.path.getsize(outfile) if os.path.exists(outfile) else 0
     ratio = float(out_size) / float(in_size) if in_size > 0 else 0.0
 
     return ratio, None, None, None
 
+
+def _check_ephys_header(outfile, data_format, data_is_unsigned, output_size_bits):
+    """Raise if the encoder did not record the requested format.
+
+    An ndi_compress_ephys built before the data-format keys existed ignores
+    them and stores the samples as signed 16-bit integers; this catches a
+    stale binary instead of letting it write wrong data.
+    """
+    with tempfile.TemporaryDirectory() as temp_dir:
+        with tarfile.open(outfile, "r:gz") as tar:
+            member = next((m for m in tar.getmembers() if m.name.endswith('.nbh')
+                           and not os.path.basename(m.name).startswith('._')), None)
+            if member is None:
+                raise ValueError("No .nbh file found")
+            tar.extract(member, path=temp_dir)
+        params = read_ndi_header(os.path.join(temp_dir, member.name))
+    expected_format = 2 if data_format == 'float' else 1
+    got = (params.get('original_format'), params.get('original_isunsigned'),
+           params.get('original_bits_per_sample'))
+    want = (expected_format, int(bool(data_is_unsigned)), int(output_size_bits))
+    if tuple(int(g) for g in got) != want:
+        os.remove(outfile)
+        raise RuntimeError(
+            "ndi_compress_ephys wrote (original_format, original_isunsigned, "
+            f"original_bits_per_sample) = {got}, not the requested {want}; "
+            "the ndi_compress_ephys binary predates the data format options "
+            "and must be updated.")
+
 def expand_ephys(fullfilename):
     """
     Expand ephys data.