diff --git a/docs/PORT_STATUS.md b/docs/PORT_STATUS.md index 5ee263a..9ef4939 100644 --- a/docs/PORT_STATUS.md +++ b/docs/PORT_STATUS.md @@ -45,6 +45,7 @@ applying the **Mirror Rule**: | `ndr.reader.tiffstack` | `ndr.reader.tiffstack` | `ndr_reader_tiffstack` | | `ndr.reader.prairieview` | `ndr.reader.prairieview` | `ndr_reader_prairieview` | | `ndr.reader.vld` | `ndr.reader.vld` | `ndr_reader_vld` | +| `ndr.reader.video` | `ndr.reader.video` | `ndr_reader_video` | | `ndr.reader.somecompany_someformat` | `ndr.reader.somecompany_someformat` | `ndr_reader_somecompany__someformat` | ## Reader Status @@ -111,6 +112,7 @@ The top-level `ndr_reader` class (`reader_wrapper.py`) wraps any format-specific | `neo` | ndr\_reader\_ced\_\_smr, ndr\_reader\_neo | Read CED SMR/SON and Blackrock files | | `pyabf` | ndr\_reader\_axon\_\_abf | Read Axon Binary Format files | | `tifffile` | ndr\_reader\_tiffstack, ndr\_reader\_prairieview | Read TIFF image stacks | +| `av` (PyAV) | ndr\_reader\_video | Decode video files through its bundled FFmpeg | | `numpy` | All readers | Array operations | ## Imaging Support and Remaining Gaps @@ -151,6 +153,7 @@ Python callers that need the numpy dtype use |---|---|---| | `ndr.reader.tiffstack` | **Ported** | Native multipage-TIFF reader, built on `tifffile`. Handles single multipage files, directory epochs, anchor/marker files, and `frametimes.txt` sidecars. | | `ndr.reader.prairieview` | **Ported** | Native legacy Prairie View reader. Groups channels from `Cycle`/`Ch` file-name markers, reads real per-frame timestamps from the config, and reports raster-scan metadata. Reads legacy `.pcf`, modern PVScan XML, and legacy MM-era XML configs. No NANSEN dependency. | +| `ndr.reader.video` | **Ported** | Native video reader (`.mp4`/`.avi`/`.mov`/...), built on PyAV where MATLAB uses `VideoReader`. One video file per epoch (companion files ignored), measured per-frame presentation times (correct for variable frame rate), `t0_t1` without decoding the whole file, and seek-based random access. | | `ndr.reader.imagestack` | Not ported | Thin wrapper over NANSEN's `nansen.stack.ImageStack` — see the dependency note below. | ### Other readers and formats @@ -167,7 +170,7 @@ Python callers that need the numpy dtype use ### Reader-types resource divergence `resource/ndr_reader_types.json` still differs between the two repositories. -MATLAB registers 14 readers; Python now registers 13. Python is missing only +MATLAB registers 15 readers; Python now registers 14. Python is missing only `imagestack` (the NANSEN-backed reader). The two files also still disagree on some type aliases: MATLAB maps `smrx`/`ced-smrx` (the sonpipe-backed 64-bit CED path, which Python does not implement) and `WMHS`. diff --git a/pyproject.toml b/pyproject.toml index ee4edd3..d974ff3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,8 @@ dev = [ "tifffile>=2021.7.2", # LZW codec for SmartSPIM All_Channels slices (tests write LZW TIFFs). "imagecodecs>=2021.11.20", + # ndr.reader.video decodes (and its tests write) video through PyAV. + "av>=12.0", ] formats = [ # ndr.format.stereoseq.readGEF reads Stereo-seq .gef files, which are HDF5. @@ -47,6 +49,8 @@ formats = [ # hand-roll chunk reading or Blosc/Zstd decoding. "zarr>=2.13", "numcodecs>=0.10", + # ndr.reader.video reads video through PyAV, whose wheels bundle FFmpeg. + "av>=12.0", ] # CED Spike2 support runs out of process through the sonpipe CLI, which pulls # CED's sonpy binding as its own dependency. Not in the base install because diff --git a/src/ndr/example_data/Readme.md b/src/ndr/example_data/Readme.md index 1a6b758..4dd2792 100644 --- a/src/ndr/example_data/Readme.md +++ b/src/ndr/example_data/Readme.md @@ -363,9 +363,9 @@ Keep in mind Neo designates any channel type to one of the: `'analog_input'`, `' --- -## Synthetic fixtures for the vld and image readers +## Synthetic fixtures for the vld, image and video readers -These four fixtures are **generated**, not recorded: they are small, +These fixtures are **generated**, not recorded: they are small, deterministic files created by `tools/make_example_data.py` in NDR-python and committed byte-identically to both repositories, so the cross-language symmetry tests read exactly the same bytes on both sides. @@ -376,6 +376,8 @@ symmetry tests read exactly the same bytes on both sides. | `example_movie.tif` | `ndr.reader.tiffstack` | 5-page uncompressed baseline TIFF, 16x12 `uint16`. Pixel value is `page*1000 + row*10 + col`, so every page and pixel is distinguishable and a transposed or mis-ordered read shows up immediately. | | `example_movie_frametimes.txt` | `ndr.reader.tiffstack` | One frame time per page (0, 0.05, ... 0.2 s), the sidecar that makes the epoch clocked rather than `no_time`. | | `prairieview/` | `ndr.reader.prairieview` | 2 channels x 4 frames of 8x6 `uint16` named `example_Cycle001_Ch_.tif`, plus a legacy `example_Main.pcf`. Pixel value is `chan*1000 + frame*100 + row*10 + col`. | +| `example_video.avi` | `ndr.reader.video` | 6-frame 12x16 uncompressed 24-bit RGB AVI at 10 frames/s, frames stored bottom-up (positive `biHeight`) as MATLAB's `VideoWriter` writes them. R = `20*frame + col`, G = `10*row + frame`, B = `200 - 10*frame - col` (frame, row, col 0-based), so channel order, orientation and frame order are all checked. Uncompressed so MATLAB's `VideoReader` reads it on every platform and both ports must agree pixel for pixel. | +| `example_video_grey.avi` | `ndr.reader.video` | The same layout as 8-bit greyscale with a 256-entry grey palette (MATLAB's 'Grayscale AVI' storage), read as C = 1. Pixel value is `30*frame + 4*row + col`. | The Prairie View config sets `Frame period (us)` = 16000 and `ScanLine period (us)` = 1500 — deliberately *not* frame period divided by diff --git a/src/ndr/example_data/example_video.avi b/src/ndr/example_data/example_video.avi new file mode 100644 index 0000000..ac94f2e Binary files /dev/null and b/src/ndr/example_data/example_video.avi differ diff --git a/src/ndr/example_data/example_video_grey.avi b/src/ndr/example_data/example_video_grey.avi new file mode 100644 index 0000000..1ccbb87 Binary files /dev/null and b/src/ndr/example_data/example_video_grey.avi differ diff --git a/src/ndr/reader/ndr_matlab_python_bridge.yaml b/src/ndr/reader/ndr_matlab_python_bridge.yaml index 2ac4c5c..466aa32 100644 --- a/src/ndr/reader/ndr_matlab_python_bridge.yaml +++ b/src/ndr/reader/ndr_matlab_python_bridge.yaml @@ -1502,6 +1502,125 @@ classes: - name: t0_t1 - name: metadata + - name: video + type: class + matlab_path: "+ndr/+reader/video.m" + status: regular_port + python_path: "ndr/reader/video.py" + matlab_last_sync_hash: "d2ebe98" + python_class: "ndr_reader_video" + inherits: base + decision_log: >- + Ported 2026-10-20 from NDR-matlab #140; synced to d2ebe98 (NDR-matlab + #143), which defines the 'video' type by the file format rather than + by VideoReader and drops the MATLAB-specific 'videoreader' alias, so + the Python registry drops it too (no change to how frames are read). + Native video reader; sibling of tiffstack, implements the frame API + only. MATLAB uses its built-in + VideoReader; the Python port uses PyAV (`av`, the FFmpeg binding; an + optional dependency, imported lazily). PyAV wheels bundle FFmpeg, so + MPEG-4/QuickTime need no platform media framework as they do for + MATLAB on Linux. Frame times are FFmpeg presentation timestamps + relative to the stream start, the quantity MATLAB reads as + CurrentTime before each readFrame; both are measured per decoded + frame and so are correct for variable-frame-rate files. C follows + VideoReader: grey pixel formats and palettised video with a grey + palette (MATLAB's 'Grayscale AVI') give C=1, everything else RGB; + above 8 bits per sample both give uint16. readframes seeks by pts, + located from the demuxed packet timestamps (no decode) via the + Python-only _framepts helper, where MATLAB's read(v, idx) does the + seek internally. Session caches are module-level dicts where MATLAB + uses persistent containers.Map, keyed the same way (path, size, + modification time). Error identifiers map to ValueError + (novideofile, multiplevideofiles), IndexError (badframeind) and + FileNotFoundError (nofile). + methods: + - name: videofile + input_arguments: + - name: epochstreams + type_python: "list[str] | str" + output_arguments: + - name: filename + type_python: "str" + - name: videoobject + input_arguments: + - name: epochstreams + type_python: "list[str] | str" + output_arguments: + - name: v + type_matlab: "VideoReader" + type_python: "av.container.InputContainer" + decision_log: >- + The PyAV container is the counterpart of the VideoReader + object. The caller owns it and closes it (context manager). + - name: numframes + - name: framesize + - name: dimensionorder + - name: datatype + - name: frametimes + - name: readframes + - name: epochclock + - name: t0_t1 + - name: getchannelsepoch + - name: metadata + static_methods: + - name: videoextensions + input_arguments: [] + output_arguments: + - name: exts + type_matlab: "cell array of char" + type_python: "list[str]" + - name: videonumframes + input_arguments: + - name: v + type_matlab: "VideoReader" + type_python: "av.container.InputContainer" + output_arguments: + - name: n + type_python: "int | float" + decision_log: "NaN (a float) when the container reports no frame count." + - name: channelsanddatatype + input_arguments: + - name: v + type_matlab: "VideoReader" + type_python: "av.container.InputContainer" + output_arguments: + - name: C + type_python: "int" + - name: dt + type_matlab: "char" + type_python: "str" + decision_log: "The MATLAB class name ('uint8'/'uint16'), as for datatype." + - name: checkframeind + input_arguments: + - name: frameind + type_python: "list[int] | numpy.ndarray" + decision_log: "Semantic Parity: frame indices are 1-based." + - name: n + type_python: "int" + output_arguments: [] + - name: endtimes + input_arguments: + - name: folder + type_python: "str" + - name: name + type_python: "str" + output_arguments: + - name: t0t1 + type_matlab: "1x2 double" + type_python: "list[float]" + - name: scanframetimes + input_arguments: + - name: folder + type_python: "str" + - name: name + type_python: "str" + output_arguments: + - name: t + type_matlab: "Nx1 double" + type_python: "numpy.ndarray" + decision_log: "1-D array, as frametimes returns throughout the Python port." + - name: imagestack type: class matlab_path: "+ndr/+reader/imagestack.m" diff --git a/src/ndr/reader/video.py b/src/ndr/reader/video.py new file mode 100644 index 0000000..aede7ce --- /dev/null +++ b/src/ndr/reader/video.py @@ -0,0 +1,578 @@ +"""Reader for video files (.mp4/.avi/.mov). + +Port of +ndr/+reader/video.m + +The ``'video'`` reader type is defined by the FILE FORMAT, not by a library: it +reads the frames of a video container (MPEG-4, AVI, QuickTime) and the time +stamped on each frame, so the two ports read the same files and give the same +times. MATLAB implements it with its built-in ``VideoReader``; this port uses +PyAV (``av``), the FFmpeg binding, which is an optional dependency imported +lazily so it is only required when a video is actually read. PyAV wheels ship +their own FFmpeg, so unlike MATLAB on Linux (which needs GStreamer for +MPEG-4/QuickTime) every container FFmpeg understands is readable everywhere. + +It is a native NDR image reader: it implements only the frame API +(``numframes``, ``framesize``, ``dimensionorder``, ``datatype``, +``frametimes``, ``readframes``, ``epochclock``, ``t0_t1``, +``getchannelsepoch``, ``metadata``) and is a sibling of +``ndr.reader.tiffstack``. + +Epoch layout: an epoch is ONE video file. ``epochstreams`` may also contain +companion files (e.g. an ``epochprobemap.ndi`` that NDI's file navigator hands +along with the video) or a directory; non-video files are ignored and +directories are expanded to the files they contain. Exactly one video file +must remain, otherwise an error is raised. + +Dimension model: each video frame is one timepoint T, with one z-plane (Z=1). +Colour channels are the C axis: C=3 for RGB video, C=1 for greyscale video +(grey pixel formats, and palettised video whose palette is grey, which is +what MATLAB's 'Grayscale AVI' profile writes). Frames are returned in +``'YXCZT'`` order. ``frameind`` is a 1-based index into T. + +Timing: a video has real frame times, so the epoch clock is always +``dev_local_time`` and ``frametimes`` are seconds from the start of the file. +A frame's time is its PRESENTATION TIME, the timestamp the container stores for +that frame, measured from the start of the file. It is read, not assumed, +which is correct for variable-frame-rate files, where ``(k-1)/FrameRate`` would +drift. Here it is FFmpeg's presentation timestamp relative to the stream's +start time, recorded for every frame as the file is decoded once in order (in +MATLAB, ``VideoReader.CurrentTime`` before each ``readFrame``). Because that scan decodes every frame, its result is cached per file +(keyed by full path, size and modification time) for the rest of the Python +session, and only ``frametimes`` (and ``numframes``, when the container +reports no count) runs it. ``t0_t1`` needs only the first and last frame +times, so it measures them the same way without the scan (see ``endtimes``). + +The frame API design is adapted from ``nansen.stack.ImageStack`` (VervaekeLab, +https://github.com/VervaekeLab/NANSEN), as for ``ndr.reader.tiffstack``; only +the design (method names, dimension model) is adapted, no NANSEN code is used. +""" + +from __future__ import annotations + +import math +import os +from pathlib import Path +from typing import Any + +import numpy as np + +from ndr.reader.base import ndr_reader_base +from ndr.time.clocktype import ClockType + +# Session caches of the per-file measurements, keyed as MATLAB keys its +# persistent containers.Map: full path, byte size and modification time. +_SCAN_CACHE: dict[str, tuple[np.ndarray, np.ndarray]] = {} +_ENDTIMES_CACHE: dict[str, list[float]] = {} +_PTS_CACHE: dict[str, np.ndarray] = {} + + +def _av(): + """Import PyAV lazily so it is only required when a video is read.""" + try: + import av + except ImportError as err: # pragma: no cover - exercised only without av + raise ImportError( + "PyAV (av) is required for reading video files. " + 'Install with: pip install "ndr[formats]"' + ) from err + return av + + +def _cachekey(filename: str) -> str: + """Return the cache key of a video file, erroring if it does not exist.""" + p = Path(filename) + if not p.is_file(): + raise FileNotFoundError(f"Video file {filename} not found.") + st = p.stat() + return f"{os.path.abspath(filename)}|{st.st_size}|{st.st_mtime_ns}" + + +def _videostream(v): + """Return the first video stream of an open container.""" + if not v.streams.video: + raise ValueError(f"{v.name} contains no video stream.") + return v.streams.video[0] + + +def _origin(stream) -> int: + """The pts that is time zero: the stream's start time, or 0 if unknown.""" + return stream.start_time if stream.start_time is not None else 0 + + +def _frametime(frame, stream) -> float: + """Presentation time of a decoded frame, in seconds from the start of the file.""" + return float((frame.pts - _origin(stream)) * stream.time_base) + + +def _framerate(stream) -> float: + """The stream's average frame rate (MATLAB's FrameRate), or NaN if unknown.""" + rate = stream.average_rate or stream.guessed_rate + return float(rate) if rate else float("nan") + + +def _frame2array(frame) -> np.ndarray: + """Convert a decoded frame to a ``Y x X x C`` array, as VideoReader returns it. + + Greyscale pixel formats give C=1 (``uint8``, or ``uint16`` above 8 bits, + VideoReader's 'Grayscale'/'Mono8'/'Mono16'); everything else is converted + to RGB, C=3 (``uint8``, or ``uint16`` above 8 bits, as 'RGB24'/'RGB48'). + Palettised frames are looked up in their palette, and give C=1 when the + palette is grey, which is how MATLAB's 'Grayscale AVI' profile is stored. + """ + name = frame.format.name + if name == "pal8": + h, w = frame.height, frame.width + idxplane = frame.planes[0] + stride = idxplane.line_size + idx = np.frombuffer(idxplane, np.uint8).reshape(h, abs(stride))[:, :w] + if stride < 0: + # A bottom-up DIB (as MATLAB's VideoWriter writes AVI) decodes with + # a negative stride; the buffer holds the rows bottom row first. + idx = idx[::-1] + # FFmpeg stores the palette as native-endian uint32 ARGB: bytes B,G,R,A. + pal = np.frombuffer(frame.planes[1], np.uint8)[: 256 * 4].reshape(256, 4) + rgbpal = pal[:, [2, 1, 0]] + if np.all(rgbpal == rgbpal[:, :1]): + return rgbpal[idx, :1] + return rgbpal[idx] + bits = max(c.bits for c in frame.format.components) + if name.startswith(("gray", "mono")): + target = "gray16le" if bits > 8 else "gray" + else: + target = "rgb48le" if bits > 8 else "rgb24" + a = frame.reformat(format=target).to_ndarray() + if a.ndim == 2: + a = a[:, :, np.newaxis] + return a + + +class ndr_reader_video(ndr_reader_base): + """Reader for video files, backed by PyAV (FFmpeg). + + Port of ndr.reader.video. An epoch is exactly one video file, optionally + listed alongside companion files or inside a directory. + + This reader implements the frame API (``numframes``, ``framesize``, + ``dimensionorder``, ``datatype``, ``frametimes``, ``readframes``) rather + than the regularly-sampled channel API. + """ + + def __init__(self) -> None: + super().__init__() + + # ------------------------------------------------------------------ + # Epoch file resolution + # ------------------------------------------------------------------ + + def videofile(self, epochstreams: list[str] | str) -> str: + """Return the single video file of an epoch. + + ``epochstreams`` entries may be files or directories (directories are + expanded to the files they contain). Files whose extension is not a + recognized video extension (see ``videoextensions``) are ignored, so + companion files such as ``epochprobemap.ndi`` do not interfere. + Raises ``ValueError`` if there is not exactly one video file. + """ + if isinstance(epochstreams, (str, Path)): + epochstreams = [epochstreams] + + files: list[str] = [] + for entry in epochstreams: + p = Path(entry) + if p.is_dir(): + files.extend(str(f) for f in p.iterdir() if f.is_file()) + else: + files.append(str(entry)) + + exts = ndr_reader_video.videoextensions() + vidfiles = sorted({f for f in files if Path(f).suffix.lower() in exts}) + + if not vidfiles: + raise ValueError( + "No video file found in the epoch files. Recognized extensions: " + + ", ".join(exts) + + "." + ) + if len(vidfiles) > 1: + raise ValueError( + "An ndr.reader.video epoch must contain exactly one video file, " + f"but {len(vidfiles)} were found: {', '.join(vidfiles)}" + ) + return vidfiles[0] + + def videoobject(self, epochstreams: list[str] | str): + """Open the epoch's video file and return the PyAV input container. + + The counterpart of MATLAB's ``VideoReader`` object. The caller owns + the container and should close it (it is a context manager). + """ + return _av().open(self.videofile(epochstreams)) + + # ------------------------------------------------------------------ + # Frame API + # ------------------------------------------------------------------ + + def numframes(self, epochstreams: list[str] | str, epoch_select: int = 1) -> int: + """Return the number of frames in the video. + + Uses the container's frame count. For containers that report none + (e.g. Matroska), the frames are counted by decoding the file once (see + ``frametimes``; the result is cached). + """ + with self.videoobject(epochstreams) as v: + n = ndr_reader_video.videonumframes(v) + name = v.name + if math.isnan(n): + p = Path(name) + n = len(ndr_reader_video.scanframetimes(str(p.parent), p.name)) + return int(n) + + def framesize(self, epochstreams: list[str] | str, epoch_select: int = 1) -> list[int]: + """Return the ``[Y X C Z T]`` extent of the video. + + Y = height, X = width, C = colour channels (3 for RGB, 1 for + greyscale), Z = 1, T = number of frames. C is taken from the shape of + the first decoded frame, so it always agrees with what ``readframes`` + returns. + """ + with self.videoobject(epochstreams) as v: + C, _dt = ndr_reader_video.channelsanddatatype(v) + cc = _videostream(v).codec_context + Y, X = int(cc.height), int(cc.width) + T = self.numframes(epochstreams, epoch_select) + return [Y, X, C, 1, T] + + def dimensionorder(self, epochstreams: list[str] | str, epoch_select: int = 1) -> str: + """Return the dimension order of returned frames (``'YXCZT'``).""" + return "YXCZT" + + def datatype(self, epochstreams: list[str] | str, epoch_select: int = 1) -> str: + """Return the numeric class of the video frames (usually ``'uint8'``). + + Taken from the class of the first decoded frame. The names are the + MATLAB class names, which numpy spells identically for the integer + types a video decodes to. + """ + with self.videoobject(epochstreams) as v: + _C, dt = ndr_reader_video.channelsanddatatype(v) + return dt + + def frametimes( + self, + epochstreams: list[str] | str, + epoch_select: int = 1, + frameind: list[int] | np.ndarray | None = None, + ) -> np.ndarray: + """Return the time of each requested frame, in seconds from the start of the file. + + The times are on the ``dev_local_time`` clock, measured by decoding the + file once (see the module docstring); the scan is cached per file. If + ``frameind`` (1-based) is omitted, all frames are returned. + """ + p = Path(self.videofile(epochstreams)) + all_t = ndr_reader_video.scanframetimes(str(p.parent), p.name) + if frameind is None or len(np.atleast_1d(frameind)) == 0: + frameind = np.arange(1, len(all_t) + 1) + frameind = np.atleast_1d(np.asarray(frameind, dtype=float)).ravel() + ndr_reader_video.checkframeind(frameind, len(all_t)) + return all_t[frameind.astype(int) - 1] + + def readframes( + self, + epochstreams: list[str] | str, + epoch_select: int = 1, + frameind: list[int] | np.ndarray | None = None, + *, + SelectC: list[int] | np.ndarray | None = None, + SelectZ: list[int] | np.ndarray | None = None, + ) -> np.ndarray: + """Read video frames as a ``[Y X C Z T]`` array. + + Reads the frames indexed by ``frameind`` (1-based, any order, random + access) and returns them with size ``[Y X numel(C) numel(Z) + numel(frameind)]``. Runs of consecutive indices are read with one seek + each. The colour channels of a frame are decoded together, so + ``SelectC``/``SelectZ`` are applied by post-selection. + """ + filename = self.videofile(epochstreams) + n = self.numframes(epochstreams, epoch_select) + if frameind is None or len(np.atleast_1d(frameind)) == 0: + frameind = np.arange(1, n + 1) + frameind = np.atleast_1d(np.asarray(frameind, dtype=float)).ravel() + ndr_reader_video.checkframeind(frameind, n) + frameind = frameind.astype(int) + + # The pts of each frame, in presentation order, so a frame index can + # be sought to. The demuxed packet timestamps give it without + # decoding; if they do not account for every frame, the decoded scan + # (which defines the frames) does. + pts = ndr_reader_video._framepts(filename) + if len(pts) != n: + pts = ndr_reader_video._scan(filename)[0] + position = {int(p): k for k, p in enumerate(pts, start=1)} + + with _av().open(filename) as v: + stream = _videostream(v) + C, dt = ndr_reader_video.channelsanddatatype(v) + Y, X = int(stream.codec_context.height), int(stream.codec_context.width) + frames = np.zeros((Y, X, C, 1, len(frameind)), dtype=dt) + + # split frameind into runs of consecutive increasing indices + breaks = [0, *(np.flatnonzero(np.diff(frameind) != 1) + 1), len(frameind)] + for r in range(len(breaks) - 1): + run = frameind[breaks[r] : breaks[r + 1]] + first, last = int(run[0]), int(run[-1]) + if first > len(pts) or last > len(pts): + raise IndexError(f"Frame {last} could not be located in {filename}.") + v.seek(int(pts[first - 1]), stream=stream, backward=True, any_frame=False) + got = 0 + for frame in v.decode(stream): + k = position.get(frame.pts) + if k is None or k < first: + continue + if k > last: + break + frames[:, :, :, 0, breaks[r] + (k - first)] = _frame2array(frame).astype( + dt, copy=False + ) + got += 1 + if k == last: + break + if got != len(run): + raise IndexError(f"Frames {first}..{last} could not all be decoded.") + + return ndr_reader_base.selectframeCZ(frames, SelectC, SelectZ) + + # ------------------------------------------------------------------ + # Clock, channels and metadata + # ------------------------------------------------------------------ + + def epochclock(self, epochstreams: list[str] | str, epoch_select: int = 1) -> list[ClockType]: + """Return ``[ClockType('dev_local_time')]``: a video always has real frame times.""" + return [ClockType("dev_local_time")] + + def t0_t1(self, epochstreams: list[str] | str, epoch_select: int = 1) -> list[list[float]]: + """Return ``[[firsttime, lasttime]]``, the times of the first and last frames. + + The same convention as ``ndr.reader.tiffstack``. Measured without + decoding the whole file (see ``endtimes``). ``epoch_select`` is + accepted for the reader interface and not used: an epoch is one file. + """ + p = Path(self.videofile(epochstreams)) + return [ndr_reader_video.endtimes(str(p.parent), p.name)] + + def getchannelsepoch( + self, epochstreams: list[str] | str, epoch_select: int = 1 + ) -> list[dict[str, Any]]: + """List the channels of a video epoch: a single ``'image'`` channel ``'image1'``. + + The colour channels of the video are returned together as the C axis + of ``readframes`` rather than as separate NDR channels. + """ + return [{"name": "image1", "type": "image", "time_channel": None}] + + def metadata(self, epochstreams: list[str] | str, epoch_select: int = 1) -> dict[str, Any]: + """Return standardized image-acquisition metadata for a video epoch. + + A video is not a raster scan, so this is the default dict from + ``emptyimagemetadata`` (``israster=False``) with only ``frame_period`` + filled in, as ``1/FrameRate`` (seconds) from the container header. + """ + m = ndr_reader_base.emptyimagemetadata() + with self.videoobject(epochstreams) as v: + fr = _framerate(_videostream(v)) + if fr > 0: + m["frame_period"] = 1.0 / fr + return m + + # ------------------------------------------------------------------ + # Channel API (not applicable to image readers) + # ------------------------------------------------------------------ + + def readchannels_epochsamples( + self, + channeltype: str, + channel: int | list[int], + epochstreams: list[str], + epoch_select: int, + s0: int, + s1: int, + ) -> np.ndarray: + """Not applicable: image readers implement the frame API instead.""" + raise NotImplementedError( + "ndr_reader_video is an image reader; use readframes() instead of " + "readchannels_epochsamples()." + ) + + def readevents_epochsamples_native( + self, + channeltype: str, + channel: int | list[int], + epochstreams: list[str], + epoch_select: int, + t0: float, + t1: float, + ) -> tuple[np.ndarray, np.ndarray]: + """Video files carry no native event channels.""" + return np.array([]), np.array([]) + + # ------------------------------------------------------------------ + # Static helpers + # ------------------------------------------------------------------ + + @staticmethod + def videoextensions() -> list[str]: + """Return the file extensions treated as video files.""" + return [ + ".avi", + ".mp4", + ".m4v", + ".mov", + ".mj2", + ".mpg", + ".mpeg", + ".wmv", + ".asf", + ".ogg", + ".ogv", + ".mkv", + ".webm", + ] + + @staticmethod + def videonumframes(v) -> int | float: + """Return the frame count the container reports, or NaN if unavailable.""" + try: + nf = _videostream(v).frames + except Exception: + return float("nan") + if nf is not None and nf > 0: + return int(nf) + return float("nan") + + @staticmethod + def channelsanddatatype(v) -> tuple[int, str]: + """Return the colour channel count and class name of the first frame. + + Decodes frame 1 of the open container ``v`` and returns its number of + colour channels (3 for RGB, 1 for greyscale) and its numeric class + (e.g. ``'uint8'``). Decoding advances ``v``; callers that read on + afterwards seek first. + """ + stream = _videostream(v) + frame = next(v.decode(stream), None) + if frame is None: + raise ValueError(f"{v.name} contains no decodable video frames.") + a = _frame2array(frame) + return int(a.shape[2]), a.dtype.name + + @staticmethod + def checkframeind(frameind: list[int] | np.ndarray, n: int) -> None: + """Raise ``IndexError`` unless ``frameind`` are integer indices in ``1..n``.""" + f = np.atleast_1d(np.asarray(frameind, dtype=float)) + if np.any(f < 1) or np.any(f > n) or np.any(f != np.round(f)): + raise IndexError(f"Frame indices must be integers between 1 and {n}.") + + @staticmethod + def endtimes(folder: str, name: str) -> list[float]: + """Return ``[first, last]`` frame times of a video file, without decoding it all. + + The presentation times in seconds of the first and last frames of + ``folder/name``, measured as ``scanframetimes`` measures them, so they + agree with ``frametimes``. The first is read by decoding frame 1. For + the last, the reader seeks to a few frame periods before the duration + and decodes to the end, widening the margin if the seek lands past + the last frame; only if that fails (no usable frame rate or duration) + does it fall back to the full scan. ``[NaN, NaN]`` for a file with no + frames. The result is cached for the session, keyed as + ``scanframetimes`` keys it. + """ + filename = str(Path(folder) / name) + key = _cachekey(filename) + if key in _ENDTIMES_CACHE: + return list(_ENDTIMES_CACHE[key]) + + with _av().open(filename) as v: + stream = _videostream(v) + frame = next(v.decode(stream), None) + if frame is None: + t0t1 = [float("nan"), float("nan")] + _ENDTIMES_CACHE[key] = t0t1 + return list(t0t1) + first = _frametime(frame, stream) + last = float("nan") + fr = _framerate(stream) + if stream.duration is not None: + dur = float(stream.duration * stream.time_base) + elif v.duration is not None: + dur = v.duration / 1e6 # container duration is in AV_TIME_BASE units + else: + dur = float("nan") + if math.isfinite(fr) and fr > 0 and math.isfinite(dur) and dur > 0: + margin = 5 + while math.isnan(last): + start = max(0.0, dur - margin / fr) + target = _origin(stream) + int(start / stream.time_base) + v.seek(target, stream=stream, backward=True, any_frame=False) + for frame in v.decode(stream): + last = _frametime(frame, stream) + if start == 0: + break # read from the start: nothing further back to try + margin *= 4 + + if math.isnan(last): + last = float(ndr_reader_video.scanframetimes(folder, name)[-1]) + t0t1 = [first, last] + _ENDTIMES_CACHE[key] = t0t1 + return list(t0t1) + + @staticmethod + def scanframetimes(folder: str, name: str) -> np.ndarray: + """Return the per-frame times of a video file, measured by decoding it once. + + Decodes ``folder/name`` frame by frame and records each frame's + presentation time (seconds from the start of the file). Returns a 1-D + array with one entry per frame. The result is cached for the session, + keyed by the full path, byte size and modification time, so a file + changed on disk is re-scanned. + """ + return ndr_reader_video._scan(str(Path(folder) / name))[1].copy() + + @staticmethod + def _scan(filename: str) -> tuple[np.ndarray, np.ndarray]: + """Decode a video once; return (pts, times) of every frame. Cached.""" + key = _cachekey(filename) + if key not in _SCAN_CACHE: + with _av().open(filename) as v: + stream = _videostream(v) + fr = _framerate(stream) + pts = [] + for k, frame in enumerate(v.decode(stream)): + if frame.pts is not None: + pts.append(frame.pts) + elif math.isfinite(fr) and fr > 0: + # a frame without a timestamp is placed at its nominal time + pts.append(_origin(stream) + round(k / fr / stream.time_base)) + else: + raise ValueError(f"{filename}: frame {k + 1} has no timestamp.") + pts = np.asarray(pts, dtype=np.int64) + t = ((pts - _origin(stream)) * float(stream.time_base)).astype(float) + _SCAN_CACHE[key] = (pts, t) + return _SCAN_CACHE[key] + + @staticmethod + def _framepts(filename: str) -> np.ndarray: + """Presentation-ordered pts of the video packets, read without decoding. Cached. + + Python-only: positions ``readframes``' seeks, the job VideoReader's + random-access ``read`` does internally in MATLAB. + """ + key = _cachekey(filename) + if key not in _PTS_CACHE: + with _av().open(filename) as v: + stream = _videostream(v) + pts = [p.pts for p in v.demux(stream) if p.size > 0] + if any(p is None for p in pts): + pts = [] # unusable; readframes falls back to the decoded scan + _PTS_CACHE[key] = np.sort(np.asarray(pts, dtype=np.int64)) + return _PTS_CACHE[key] diff --git a/src/ndr/resource/ndr_reader_types.json b/src/ndr/resource/ndr_reader_types.json index 0b68cdf..64cb453 100644 --- a/src/ndr/resource/ndr_reader_types.json +++ b/src/ndr/resource/ndr_reader_types.json @@ -108,5 +108,14 @@ "labview" ], "classname": "ndr.reader.vld.ndr_reader_vld" + }, + { + "type": [ + "video", + "mp4", + "avi", + "mov" + ], + "classname": "ndr.reader.video.ndr_reader_video" } ] diff --git a/tests/symmetry/make_artifacts/reader/test_read_video.py b/tests/symmetry/make_artifacts/reader/test_read_video.py new file mode 100644 index 0000000..6a1a1ae --- /dev/null +++ b/tests/symmetry/make_artifacts/reader/test_read_video.py @@ -0,0 +1,88 @@ +"""Generate symmetry artifacts for the video reader. + +Exercises the frame API (numframes, framesize, dimensionorder, datatype, +frametimes, t0_t1, epochclock, getchannelsepoch, metadata, readframes) against +the checked-in example_video.avi (24-bit RGB) and example_video_grey.avi +(8-bit, grey palette) and exports JSON artifacts the MATLAB symmetry suite +re-reads and verifies. Both files are uncompressed, so the two ports must +agree pixel for pixel even though they decode with different libraries +(MATLAB's VideoReader, PyAV/FFmpeg here). + +Each colour channel is stored as its own YxX plane (nested row lists, which is +what MATLAB's jsonencode produces for a matrix), so the comparison is +unambiguous despite MATLAB being column-major and numpy row-major. + +Mirrors: + tools/tests/+ndr/+symmetry/+makeArtifacts/+reader/readVideo.m +""" + +import json +import shutil + +import pytest + +from tests.symmetry.conftest import EXAMPLE_DATA, PYTHON_ARTIFACTS, json_safe + +ARTIFACT_DIR = PYTHON_ARTIFACTS / "reader" / "readVideo" / "testReadVideoArtifacts" + +VIDEOS = {"rgb": "example_video.avi", "grey": "example_video_grey.avi"} + + +def _fresh(d): + if d.exists(): + shutil.rmtree(d) + d.mkdir(parents=True) + + +class TestReadVideo: + @pytest.fixture(autouse=True) + def _setup(self): + for name in VIDEOS.values(): + if not (EXAMPLE_DATA / name).exists(): + pytest.skip(f"Example video {name} not available") + pytest.importorskip("av") + + from ndr.reader.video import ndr_reader_video + + self.reader = ndr_reader_video() + self.epochstreams = {k: [str(EXAMPLE_DATA / v)] for k, v in VIDEOS.items()} + + def _metadata(self, es): + r = self.reader + return { + "numframes": r.numframes(es, 1), + "framesize": r.framesize(es, 1), + "dimensionorder": r.dimensionorder(es, 1), + "datatype": r.datatype(es, 1), + "frametimes": r.frametimes(es, 1), + "t0_t1": r.t0_t1(es, 1)[0], + "epochclock": [c.type for c in r.epochclock(es, 1)], + "channels": [{"name": c["name"], "type": c["type"]} for c in r.getchannelsepoch(es, 1)], + "image_metadata": r.metadata(es, 1), + } + + def test_read_video_artifacts(self): + _fresh(ARTIFACT_DIR) + + metadata = {k: self._metadata(es) for k, es in self.epochstreams.items()} + (ARTIFACT_DIR / "metadata.json").write_text( + json.dumps(json_safe(metadata), indent=2), encoding="utf-8" + ) + + rgb, grey = self.epochstreams["rgb"], self.epochstreams["grey"] + n = metadata["rgb"]["numframes"] + # First and last RGB frame, channel by channel; a SelectC read; and an + # out-of-order greyscale read, which goes through the random-access path. + frames = self.reader.readframes(rgb, 1, [1, n]) + sel = self.reader.readframes(rgb, 1, [3], SelectC=[2]) + greyframes = self.reader.readframes(grey, 1, [4, 2]) + read_struct = {} + for t, frame in ((0, 1), (1, n)): + for c in range(3): + read_struct[f"rgb_frame_{frame}_channel_{c + 1}"] = frames[:, :, c, 0, t] + read_struct["rgb_frame_3_selectC_2"] = sel[:, :, 0, 0, 0] + read_struct["grey_frame_4"] = greyframes[:, :, 0, 0, 0] + read_struct["grey_frame_2"] = greyframes[:, :, 0, 0, 1] + (ARTIFACT_DIR / "readFrames.json").write_text( + json.dumps(json_safe(read_struct), indent=2), encoding="utf-8" + ) diff --git a/tests/symmetry/read_artifacts/reader/test_read_video.py b/tests/symmetry/read_artifacts/reader/test_read_video.py new file mode 100644 index 0000000..27fef24 --- /dev/null +++ b/tests/symmetry/read_artifacts/reader/test_read_video.py @@ -0,0 +1,122 @@ +"""Read and verify symmetry artifacts for the video reader. + +Parameterized over both artifact sources, so one test verifies parity in both +directions. Skips when the artifact directory for a source does not exist. + +Frame times are compared to 1e-6 s rather than the 1e-9 the TIFF readers use: +here each port measures them through its own decoder (VideoReader's +CurrentTime in MATLAB, FFmpeg presentation timestamps here) rather than +reading the same numbers from a text sidecar, and a microsecond is still far +below one frame period. Pixels must match exactly: the files are uncompressed. + +Mirrors: + tools/tests/+ndr/+symmetry/+readArtifacts/+reader/readVideo.m +""" + +import json + +import numpy as np +import pytest + +from tests.symmetry.conftest import EXAMPLE_DATA, SOURCE_TYPES, SYMMETRY_BASE + +VIDEOS = {"rgb": "example_video.avi", "grey": "example_video_grey.avi"} +TIME_TOL = 1e-6 + + +@pytest.fixture(params=SOURCE_TYPES) +def source_type(request): + return request.param + + +class TestReadVideo: + def _artifact_dir(self, source_type): + return SYMMETRY_BASE / source_type / "reader" / "readVideo" / "testReadVideoArtifacts" + + def _reader(self): + for name in VIDEOS.values(): + if not (EXAMPLE_DATA / name).exists(): + pytest.skip(f"Example video {name} not available") + pytest.importorskip("av") + + from ndr.reader.video import ndr_reader_video + + return ndr_reader_video(), {k: [str(EXAMPLE_DATA / v)] for k, v in VIDEOS.items()} + + @pytest.mark.parametrize("video", sorted(VIDEOS)) + def test_metadata(self, source_type, video): + artifact_dir = self._artifact_dir(source_type) + if not artifact_dir.exists(): + pytest.skip(f"No artifacts from {source_type}") + + expected = json.loads((artifact_dir / "metadata.json").read_text())[video] + reader, streams = self._reader() + es = streams[video] + + assert reader.numframes(es, 1) == expected["numframes"] + assert list(reader.framesize(es, 1)) == list(expected["framesize"]) + assert reader.dimensionorder(es, 1) == expected["dimensionorder"] + # Strict string parity: both ports report the MATLAB class name. + assert reader.datatype(es, 1) == expected["datatype"] + np.testing.assert_allclose( + reader.frametimes(es, 1), np.ravel(expected["frametimes"]), atol=TIME_TOL + ) + np.testing.assert_allclose(reader.t0_t1(es, 1)[0], expected["t0_t1"], atol=TIME_TOL) + assert [c.type for c in reader.epochclock(es, 1)] == list(expected["epochclock"]) + + # MATLAB's jsondecode turns a one-element struct array into a struct. + exp_channels = expected["channels"] + if isinstance(exp_channels, dict): + exp_channels = [exp_channels] + assert [ + {"name": c["name"], "type": c["type"]} for c in reader.getchannelsepoch(es, 1) + ] == exp_channels + + @pytest.mark.parametrize("video", sorted(VIDEOS)) + def test_image_metadata(self, source_type, video): + """The image-metadata struct, field by field.""" + artifact_dir = self._artifact_dir(source_type) + if not artifact_dir.exists(): + pytest.skip(f"No artifacts from {source_type}") + + expected = json.loads((artifact_dir / "metadata.json").read_text())[video]["image_metadata"] + reader, streams = self._reader() + actual = reader.metadata(streams[video], 1) + + assert set(actual) == set(expected), "image metadata field sets differ" + for key, exp in expected.items(): + got = actual[key] + if exp is None: + # MATLAB writes NaN as null; the Python side must also be NaN. + assert isinstance(got, float) and np.isnan(got), f"{key} should be NaN" + elif isinstance(exp, bool): + assert bool(got) == exp, f"{key} mismatch" + else: + assert got == pytest.approx(exp, abs=1e-9), f"{key} mismatch" + + def test_frames(self, source_type): + artifact_dir = self._artifact_dir(source_type) + if not artifact_dir.exists(): + pytest.skip(f"No artifacts from {source_type}") + + expected = json.loads((artifact_dir / "readFrames.json").read_text()) + reader, streams = self._reader() + rgb, grey = streams["rgb"], streams["grey"] + + n = reader.numframes(rgb, 1) + frames = reader.readframes(rgb, 1, [1, n]) + for t, frame in ((0, 1), (1, n)): + for c in range(3): + key = f"rgb_frame_{frame}_channel_{c + 1}" + np.testing.assert_array_equal( + frames[:, :, c, 0, t], np.array(expected[key]), err_msg=key + ) + + sel = reader.readframes(rgb, 1, [3], SelectC=[2]) + np.testing.assert_array_equal( + sel[:, :, 0, 0, 0], np.array(expected["rgb_frame_3_selectC_2"]) + ) + + greyframes = reader.readframes(grey, 1, [4, 2]) + np.testing.assert_array_equal(greyframes[:, :, 0, 0, 0], np.array(expected["grey_frame_4"])) + np.testing.assert_array_equal(greyframes[:, :, 0, 0, 1], np.array(expected["grey_frame_2"])) diff --git a/tests/test_reader_registry_parity.py b/tests/test_reader_registry_parity.py index 172fea9..83066fc 100644 --- a/tests/test_reader_registry_parity.py +++ b/tests/test_reader_registry_parity.py @@ -54,6 +54,10 @@ "vhlv": "ndr.reader.vld", "vhlabview": "ndr.reader.vld", "labview": "ndr.reader.vld", + "video": "ndr.reader.video", + "mp4": "ndr.reader.video", + "avi": "ndr.reader.video", + "mov": "ndr.reader.video", } # ndr.reader.imagestack exists in NDR-matlab with aliases imagestack / nansen / diff --git a/tests/test_video_reader.py b/tests/test_video_reader.py new file mode 100644 index 0000000..7178841 --- /dev/null +++ b/tests/test_video_reader.py @@ -0,0 +1,369 @@ +"""Tests for the native video reader. + +Port of NDR-matlab tools/tests/+ndr/+unittest/+reader/TestVideo.m. Small +synthetic videos with known per-frame content are written with PyAV into a +temporary folder: + + - an RGB video, written as MPEG-4 and as Motion JPEG AVI (the two profiles + the MATLAB test tries). Both are lossy, so pixel content is compared + through per-frame, per-channel means with a tolerance; + - a greyscale video written as uncompressed 8-bit palettised AVI with a grey + palette -- what MATLAB's 'Grayscale AVI' profile writes -- which checks the + C=1 path and exact pixel round-trip. + +Beyond the MATLAB test, a variable-frame-rate file checks that frame times are +measured rather than assumed, and a Matroska file (whose header carries no +frame count) checks the counting fallback and 16-bit greyscale. +""" + +from fractions import Fraction + +import numpy as np +import pytest + +av = pytest.importorskip("av") + +from ndr.reader.video import ndr_reader_video # noqa: E402 +from ndr.reader_wrapper import ndr_reader # noqa: E402 + +Y, X, T = 24, 32, 12 +FRAMERATE = 4 +MEANTOL = 6 # tolerance on per-frame channel means for lossy codecs + + +def _write(filename, codec, pix_fmt, frames, fmt_in, rate=FRAMERATE, times=None): + """Write Y x X (x C) frames; ``times`` (seconds) gives explicit timestamps.""" + with av.open(str(filename), "w") as c: + s = c.add_stream(codec, rate=rate) + s.width, s.height, s.pix_fmt = X, Y, pix_fmt + if times is not None: + s.codec_context.time_base = Fraction(1, 1000) + for i, f in enumerate(frames): + vf = av.VideoFrame.from_ndarray(np.ascontiguousarray(f), format=fmt_in) + if times is not None: + vf.pts = round(times[i] * 1000) + vf.time_base = Fraction(1, 1000) + for p in s.encode(vf): + c.mux(p) + for p in s.encode(): + c.mux(p) + return str(filename) + + +def _rgbtruth(n=T): + """Flat frames with a frame-dependent level; the three channels differ.""" + rgb = np.zeros((Y, X, 3, 1, n), dtype=np.uint8) + for i in range(n): + level = 20 + i * 18 # 20 .. 218 + rgb[:, :, 0, 0, i] = level + rgb[:, :, 1, 0, i] = 255 - level + rgb[:, :, 2, 0, i] = 128 + return rgb + + +def _greytruth(): + """A spatial gradient plus a frame-dependent offset.""" + base = np.arange(Y * X).reshape(X, Y).T # MATLAB column-major reshape + grey = np.zeros((Y, X, 1, 1, T), dtype=np.uint8) + for i in range(T): + grey[:, :, 0, 0, i] = np.mod(base + i * 7, 200) + return grey + + +def _writegreyavi(filename, grey): + """Raw 8-bit palettised AVI with a grey palette, as MATLAB 'Grayscale AVI'.""" + with av.open(str(filename), "w") as c: + s = c.add_stream("rawvideo", rate=FRAMERATE) + s.width, s.height, s.pix_fmt = X, Y, "pal8" + # FFmpeg palette entries are uint32 ARGB (bytes B,G,R,A); index k is grey k + palette = np.repeat(np.arange(256, dtype=np.uint8), 4).reshape(256, 4) + palette[:, 3] = 255 + for i in range(grey.shape[4]): + vf = av.VideoFrame(X, Y, "pal8") + idxplane = vf.planes[0] + rows = np.zeros((Y, idxplane.line_size), dtype=np.uint8) + rows[:, :X] = grey[:, :, 0, 0, i] + idxplane.update(rows.tobytes()) + vf.planes[1].update(palette.tobytes()) + for p in s.encode(vf): + c.mux(p) + for p in s.encode(): + c.mux(p) + return str(filename) + + +def _channelmeans(frames): + """C x T matrix of per-frame, per-channel means of a YXCZT array.""" + return frames.astype(float).mean(axis=(0, 1))[:, 0, :] + + +@pytest.fixture(scope="module") +def videos(tmp_path_factory): + d = tmp_path_factory.mktemp("ndr_video_test") + rgb = _rgbtruth() + frames = [rgb[:, :, :, 0, i] for i in range(T)] + grey = _greytruth() + return { + "dir": d, + "rgb": rgb, + "grey": grey, + "mp4": _write(d / "rgbvideo.mp4", "mpeg4", "yuv420p", frames, "rgb24"), + "avi": _write(d / "rgbvideo.avi", "mjpeg", "yuvj420p", frames, "rgb24"), + "greyfile": _writegreyavi(d / "greyvideo.avi", grey), + } + + +@pytest.fixture(params=["mp4", "avi"]) +def rgbfile(request, videos): + return videos[request.param] + + +@pytest.fixture +def reader(): + return ndr_reader("video") + + +class TestVideoReader: + def test_known_readers(self, reader): + assert isinstance(reader.ndr_reader_base, ndr_reader_video) + for alias in ("mp4", "avi", "mov"): + assert isinstance(ndr_reader(alias).ndr_reader_base, ndr_reader_video) + + def test_greyscale_avi_is_palettised_like_matlab(self, videos): + # the fixture really is MATLAB's 'Grayscale AVI' storage, not plain gray + with av.open(videos["greyfile"]) as c: + assert c.streams.video[0].codec_context.pix_fmt == "pal8" + + def test_numframes_and_size(self, reader, rgbfile): + ef = [rgbfile] + assert reader.numframes(ef, 1) == T + assert reader.framesize(ef, 1) == [Y, X, 3, 1, T] + + def test_dimensionorder_and_datatype(self, reader, rgbfile): + ef = [rgbfile] + assert reader.dimensionorder(ef, 1) == "YXCZT" + assert reader.datatype(ef, 1) == "uint8" + + def test_clock_and_times(self, reader, rgbfile): + ef = [rgbfile] + ec = reader.epochclock(ef, 1) + assert len(ec) == 1 + assert ec[0].type == "dev_local_time" + + ft = reader.frametimes(ef, 1) + assert ft.shape == (T,) + np.testing.assert_allclose(ft, np.arange(T) / FRAMERATE, atol=1e-3) + np.testing.assert_allclose(np.diff(ft), 1 / FRAMERATE, atol=1e-3) + + np.testing.assert_array_equal(reader.frametimes(ef, 1, [2, 5, 9]), ft[[1, 4, 8]]) + + t0t1 = reader.t0_t1(ef, 1) + np.testing.assert_allclose(t0t1[0], [ft[0], ft[-1]], atol=1e-9) + + def test_t0_t1_without_a_full_scan(self, reader, videos, rgbfile): + # a fresh copy, so neither the scan's cache nor endtimes' has seen it; + # it must agree with the full scan, taken after + for src in (rgbfile, videos["greyfile"]): + src = videos["dir"] / src.split("/")[-1] + copy = videos["dir"] / f"fresh_{src.stem}{src.suffix}" + copy.write_bytes(src.read_bytes()) + t0t1 = reader.t0_t1([str(copy)], 1) + np.testing.assert_allclose(t0t1[0], [0, (T - 1) / FRAMERATE], atol=1e-3) + ft = reader.frametimes([str(copy)], 1) + np.testing.assert_allclose(t0t1[0], [ft[0], ft[-1]], atol=1e-9) + copy.unlink() + + @pytest.mark.parametrize( + "codec,pix_fmt,ext", [("mpeg4", "yuv420p", ".mp4"), ("mjpeg", "yuvj420p", ".avi")] + ) + def test_endtimes_of_a_longer_video(self, videos, codec, pix_fmt, ext): + # longer than the seek margin (5 frames): the last frame is reached + # by seeking, not by decoding from the start + n = 60 + rgb = np.repeat(_rgbtruth(1), n, axis=4) + for i in range(n): + rgb[:, :, 0, 0, i] = np.mod((i + 1) * 4, 250) + name = f"longvideo{ext}" + _write( + videos["dir"] / name, codec, pix_fmt, [rgb[:, :, :, 0, i] for i in range(n)], "rgb24" + ) + folder = str(videos["dir"]) + t0t1 = ndr_reader_video.endtimes(folder, name) + ft = ndr_reader_video.scanframetimes(folder, name) + assert len(ft) == n + np.testing.assert_allclose(t0t1, [ft[0], ft[-1]], atol=1e-9) + + def test_frames_content(self, reader, videos, rgbfile): + frames = reader.readframes([rgbfile], 1) + assert frames.shape == (Y, X, 3, 1, T) + assert frames.dtype == np.uint8 + np.testing.assert_allclose( + _channelmeans(frames), _channelmeans(videos["rgb"]), atol=MEANTOL + ) + + def test_random_access_subset(self, reader, videos, rgbfile): + ef = [rgbfile] + all_frames = reader.readframes(ef, 1) + idx = [7, 2, 3, 4, 12, 1] # out of order, with a consecutive run + subset = reader.readframes(ef, 1, idx) + assert subset.shape == (Y, X, 3, 1, len(idx)) + sel = np.asarray(idx) - 1 + np.testing.assert_array_equal(subset, all_frames[:, :, :, :, sel]) + np.testing.assert_allclose( + _channelmeans(subset), _channelmeans(videos["rgb"][:, :, :, :, sel]), atol=MEANTOL + ) + one = reader.readframes(ef, 1, [5]) + np.testing.assert_array_equal(one, all_frames[:, :, :, :, [4]]) + + def test_selectc(self, reader, rgbfile): + ef = [rgbfile] + g = reader.readframes(ef, 1, [1, 2], SelectC=[2]) + assert g.shape == (Y, X, 1, 1, 2) + np.testing.assert_array_equal(g, reader.readframes(ef, 1, [1, 2])[:, :, [1], :, :]) + + def test_greyscale_video(self, reader, videos): + ef = [videos["greyfile"]] + grey = videos["grey"] + assert reader.numframes(ef, 1) == T + assert reader.framesize(ef, 1) == [Y, X, 1, 1, T] + assert reader.datatype(ef, 1) == "uint8" + np.testing.assert_array_equal(reader.readframes(ef, 1), grey) + np.testing.assert_array_equal(reader.readframes(ef, 1, [10, 3]), grey[:, :, :, :, [9, 2]]) + np.testing.assert_allclose(reader.frametimes(ef, 1), np.arange(T) / FRAMERATE, atol=1e-3) + + def test_companion_files_ignored(self, reader, videos): + companion = videos["dir"] / "epochprobemap.ndi" + companion.write_text("name reference type\n") + ef = [str(companion), videos["greyfile"]] + assert reader.numframes(ef, 1) == T + np.testing.assert_array_equal( + reader.readframes(ef, 1, [1]), videos["grey"][:, :, :, :, [0]] + ) + + def test_directory_epoch(self, tmp_path, videos): + (tmp_path / "clip.avi").write_bytes(open(videos["greyfile"], "rb").read()) + (tmp_path / "epochprobemap.ndi").write_text("name reference type\n") + assert ndr_reader_video().numframes([str(tmp_path)]) == T + + def test_exactly_one_video_file(self, reader, videos): + with pytest.raises(ValueError, match="exactly one video file"): + reader.numframes([videos["mp4"], videos["greyfile"]], 1) + txt = videos["dir"] / "notes.txt" + txt.write_text("no video here\n") + with pytest.raises(ValueError, match="No video file found"): + reader.numframes([str(txt)], 1) + + def test_bad_frame_index(self, reader, videos): + ef = [videos["greyfile"]] + with pytest.raises(IndexError): + reader.readframes(ef, 1, [T + 1]) + with pytest.raises(IndexError): + reader.frametimes(ef, 1, [0]) + with pytest.raises(IndexError): + reader.frametimes(ef, 1, [1.5]) + + def test_channels_and_metadata(self, reader, rgbfile): + ef = [rgbfile] + channels = reader.getchannelsepoch(ef, 1) + assert len(channels) == 1 + assert channels[0]["type"] == "image" + assert channels[0]["name"] == "image1" + m = reader.metadata(ef, 1) + assert m["israster"] is False + assert m["frame_period"] == pytest.approx(1 / FRAMERATE, abs=1e-6) + + def test_channel_api_not_applicable(self, videos): + with pytest.raises(NotImplementedError): + ndr_reader_video().readchannels_epochsamples("ai", 1, [videos["mp4"]], 1, 1, 10) + + +class TestVideoReaderBeyondMatlab: + def test_variable_frame_rate_times_are_measured(self, tmp_path): + # (k-1)/FrameRate would be wrong here: the gaps are uneven + times = [0.0, 0.1, 0.15, 0.5, 0.55, 1.2] + rgb = _rgbtruth(len(times)) + f = _write( + tmp_path / "vfr.mkv", + "mjpeg", + "yuvj420p", + [rgb[:, :, :, 0, i] for i in range(len(times))], + "rgb24", + times=times, + ) + r = ndr_reader_video() + np.testing.assert_allclose(r.frametimes([f]), times, atol=1e-3) + np.testing.assert_allclose(r.t0_t1([f])[0], [0.0, 1.2], atol=1e-3) + # random access lands on the right frames despite the uneven gaps + sub = r.readframes([f], 1, [5, 2]) + np.testing.assert_allclose( + _channelmeans(sub), _channelmeans(rgb[:, :, :, :, [4, 1]]), atol=MEANTOL + ) + + def test_no_header_frame_count_and_16bit_grey(self, tmp_path, videos): + # Matroska carries no frame count, so numframes counts by decoding + grey16 = videos["grey"].astype(np.uint16) * 300 + f = _write( + tmp_path / "grey16.mkv", + "ffv1", + "gray16le", + [grey16[:, :, 0, 0, i] for i in range(T)], + "gray16le", + ) + r = ndr_reader_video() + with r.videoobject([f]) as v: + assert np.isnan(ndr_reader_video.videonumframes(v)) + assert r.numframes([f]) == T + assert r.datatype([f]) == "uint16" + assert r.framesize([f]) == [Y, X, 1, 1, T] + np.testing.assert_array_equal(r.readframes([f]), grey16) + np.testing.assert_array_equal(r.readframes([f], 1, [11, 4, 5]), grey16[..., [10, 3, 4]]) + + def test_scan_is_cached_and_invalidated_by_a_change(self, tmp_path, videos): + f = tmp_path / "clip.avi" + f.write_bytes(open(videos["greyfile"], "rb").read()) + t1 = ndr_reader_video.scanframetimes(str(tmp_path), "clip.avi") + assert len(t1) == T + rgb = _rgbtruth(5) + _write(f, "mjpeg", "yuvj420p", [rgb[:, :, :, 0, i] for i in range(5)], "rgb24") + assert len(ndr_reader_video.scanframetimes(str(tmp_path), "clip.avi")) == 5 + + def test_missing_file(self, tmp_path): + with pytest.raises(FileNotFoundError): + ndr_reader_video.scanframetimes(str(tmp_path), "absent.mp4") + + +class TestCheckedInExampleVideos: + """The shared symmetry fixtures, which store frames bottom-up as MATLAB does. + + FFmpeg decodes a bottom-up 8-bit DIB with a negative line stride; reading + the plane in memory order instead returns the frame upside down. The + synthetic files above are written top-down by FFmpeg and cannot catch + that. Pixel formulas are those of tools/make_example_data.py. + """ + + @staticmethod + def _grid(): + return np.meshgrid(np.arange(6), np.arange(12), np.arange(16), indexing="ij") + + def test_rgb_exact(self): + from ndr.fun.ndrpath import ndrpath + + f, row, col = self._grid() + truth = np.stack([20 * f + col, 10 * row + f, 200 - 10 * f - col], axis=-1) + es = [str(ndrpath() / "example_data" / "example_video.avi")] + r = ndr_reader_video() + assert r.framesize(es) == [12, 16, 3, 1, 6] + got = np.moveaxis(r.readframes(es)[:, :, :, 0, :], -1, 0) + np.testing.assert_array_equal(got, truth.astype(np.uint8)) + np.testing.assert_allclose(r.frametimes(es), np.arange(6) / 10, atol=1e-9) + + def test_grey_palette_exact(self): + from ndr.fun.ndrpath import ndrpath + + f, row, col = self._grid() + truth = (30 * f + 4 * row + col).astype(np.uint8) + es = [str(ndrpath() / "example_data" / "example_video_grey.avi")] + r = ndr_reader_video() + assert r.framesize(es) == [12, 16, 1, 1, 6] + np.testing.assert_array_equal(r.readframes(es)[:, :, 0, 0, :], np.moveaxis(truth, 0, -1)) + np.testing.assert_array_equal(r.readframes(es, 1, [4, 2])[:, :, 0, 0, 0], truth[3]) diff --git a/tools/make_example_data.py b/tools/make_example_data.py index d5212b3..cd99798 100644 --- a/tools/make_example_data.py +++ b/tools/make_example_data.py @@ -13,10 +13,13 @@ example_movie.tif 5-page 16x12 uint16 TIFF stack example_movie_frametimes.txt one frame time per page, seconds prairieview/ 2 channels x 4 frames + legacy .pcf + example_video.avi 6-frame 12x16 uncompressed RGB AVI + example_video_grey.avi the same, 8-bit greyscale (grey palette) """ from __future__ import annotations +import struct import sys from pathlib import Path @@ -48,6 +51,13 @@ PV_LINE_PERIOD_US = 1500.0 PV_FRAME_PERIOD_US = 16000.0 +# --- video ---------------------------------------------------------------- +VIDEO_FRAMES = 6 +VIDEO_HEIGHT = 12 +# A multiple of 4, so DIB rows need no padding in either bit depth. +VIDEO_WIDTH = 16 +VIDEO_FPS = 10 + def vld_signal() -> np.ndarray: """Deterministic per-channel waveform, shape (VLD_TOTAL, VLD_NCHANS).""" @@ -120,6 +130,108 @@ def write_prairieview(out: Path) -> None: ) +def video_rgb() -> np.ndarray: + """Deterministic RGB video, shape (T, Y, X, 3) uint8, distinct per frame/pixel/channel.""" + f, row, col = np.meshgrid( + np.arange(VIDEO_FRAMES), np.arange(VIDEO_HEIGHT), np.arange(VIDEO_WIDTH), indexing="ij" + ) + return np.stack([20 * f + col, 10 * row + f, 200 - 10 * f - col], axis=-1).astype(np.uint8) + + +def video_grey() -> np.ndarray: + """Deterministic greyscale video, shape (T, Y, X) uint8.""" + f, row, col = np.meshgrid( + np.arange(VIDEO_FRAMES), np.arange(VIDEO_HEIGHT), np.arange(VIDEO_WIDTH), indexing="ij" + ) + return (30 * f + 4 * row + col).astype(np.uint8) + + +def _chunk(fourcc: bytes, payload: bytes) -> bytes: + pad = b"\0" if len(payload) % 2 else b"" + return fourcc + struct.pack(" bytes: + return _chunk(b"LIST", kind + payload) + + +def write_avi(path: Path, frames: np.ndarray) -> None: + """Write an uncompressed (BI_RGB) AVI, laid out as MATLAB's VideoWriter does. + + ``frames`` is (T, Y, X, 3) RGB for a 24-bit file, or (T, Y, X) for an + 8-bit file with a 256-entry grey palette ('Grayscale AVI'). Frames are + stored bottom-up (positive biHeight), the canonical DIB orientation that + every AVI reader handles, rather than FFmpeg's top-down negative height. + """ + t, h, w = frames.shape[:3] + grey = frames.ndim == 3 + bitcount = 8 if grey else 24 + framebytes = w * h * bitcount // 8 + + avih = struct.pack( + " None: + """Write the RGB and greyscale example videos.""" + write_avi(out / "example_video.avi", video_rgb()) + write_avi(out / "example_video_grey.avi", video_grey()) + + def main(targets: list[str]) -> int: if not targets: print(__doc__) @@ -130,6 +242,7 @@ def main(targets: list[str]) -> int: write_vld(out) write_tiffstack(out) write_prairieview(out) + write_video(out) print(f"wrote example data into {out}") return 0