add applyOrientation, toneMapping and gainMap to Animation.decode

Author: matmen

CHANGELOG.md

@@ -9,8 +9,14 @@
   values). The old values (`'RESIZE_BILINEAR'`, `1`-`3`, still the values of the deprecated `Image.RESIZE_*` and
   `Image.SVG_MODE_*`) keep working until 4.0: `image.resize(256, 256, 'bicubic')`.
 
+- **`Animation.decode()` applies the orientation of APNGs and animated WebPs** (their eXIf and EXIF chunks, and XMP
+  in PNGs), like `Image.decode()` does (it already did for JPEG XLs). `applyOrientation: false` keeps the stored
+  frames. Still PNGs decoded by `Animation.decode()` are rotated as well now.
+
 ### New
 
+- `Animation.decode()` takes `applyOrientation`, `toneMapping` and `gainMap` like `Image.decode()`: JPEG XL
+  animations can be tone mapped with `'clamp'`, and their `jhgm` gain map is read into the `gainMap` of every frame.
 - `resize({ width })` and `resize({ height })` on images, frames and animations: the missing side keeps the aspect
   ratio, `resize({ width: 256, mode: 'bicubic' })`.
 - `image.coordinates()` yields `[x, y]` and `image.pixels()` yields `[x, y, color]` for every pixel, row by row.

src/animation.ts

@@ -5,6 +5,8 @@ import * as webp from './codecs/webp.ts';
 import * as jxl from './codecs/jxl.ts';
 import { availableThreads } from './util/threads.ts';
 import { BinaryData, view } from './util/mem.ts';
+import { GainMap } from './gainmap.ts';
+import type { ToneMapping } from './codecs/hdr.ts';
 import { Image, ResizeMode, ResizeModes, ResizeOptions, ResizeToOptions, resizeArguments } from './image.ts';
 import { Frame, FrameDisposalModes } from './frame.ts';
 import { ImageType } from './image-type.ts';
@@ -51,6 +53,22 @@ export interface AnimationDecodeOptions {
     applyColorProfile?: boolean;
     /** At most this many worker threads decode in parallel, like `threads` of {@link Image.decode}. Default: the processors available, at most 8 */
     threads?: number;
+    /**
+     * Whether to rotate/mirror the frames as the orientation tag (Exif, JPEG XL's header, PNG's eXIf chunk or XMP)
+     * says, like `applyOrientation` of {@link Image.decode}: APNGs, animated WebPs and JPEG XLs. Default: `true`
+     */
+    applyOrientation?: boolean;
+    /**
+     * How HDR (PQ or HLG) JPEG XL animations, whose light can be brighter than (SDR) white, become 8 bit sRGB, like
+     * `toneMapping` of {@link Image.decode}: `'reinhard'` compresses the brightest parts, `'clamp'` clamps everything
+     * brighter than white. Default: `'reinhard'`
+     */
+    toneMapping?: ToneMapping;
+    /**
+     * Whether to read the gain map (jhgm box) of JPEG XL animations into {@link Image.gainMap} of every frame, like
+     * `gainMap` of {@link Image.decode}. Default: `true`
+     */
+    gainMap?: boolean;
 }
 
 /** The options all animation encoders have */
@@ -445,6 +463,8 @@ export class Animation extends Array<Frame> {
             const { maxPixels = MAX_PIXELS, threads = 1 } = onlyFirstFrameOrOptions;
             if (!Number.isInteger(threads) || threads < 1) throw new RangeError(`threads must be a positive integer, not ${threads}`);
             if (!(Number(maxPixels) > 0)) throw new RangeError(`maxPixels must be a positive number, not ${maxPixels}`);
+            const { toneMapping = 'reinhard' } = onlyFirstFrameOrOptions;
+            if (toneMapping !== 'reinhard' && toneMapping !== 'clamp') throw new TypeError(`toneMapping must be 'reinhard' or 'clamp', not ${String(toneMapping)}`);
         }
         return decoding(() => Animation.__decode__(input, onlyFirstFrameOrOptions));
     }
@@ -454,7 +474,11 @@ export class Animation extends Array<Frame> {
      * @ignore
      */
     private static async __decode__(input: BinaryData, onlyFirstFrameOrOptions: boolean | AnimationDecodeOptions): Promise<Animation> {
-        const { onlyFirstFrame: onlyExtractFirstFrame = false, maxPixels = MAX_PIXELS, applyColorProfile = true, threads = Math.max(1, Math.min(availableThreads(), 8)) } = isOptions(onlyFirstFrameOrOptions) ? onlyFirstFrameOrOptions : { onlyFirstFrame: onlyFirstFrameOrOptions };
+        const { onlyFirstFrame: onlyExtractFirstFrame = false, maxPixels = MAX_PIXELS, applyColorProfile = true, threads = Math.max(1, Math.min(availableThreads(), 8)), applyOrientation = true, toneMapping = 'reinhard', gainMap = true } = isOptions(onlyFirstFrameOrOptions) ? onlyFirstFrameOrOptions : { onlyFirstFrame: onlyFirstFrameOrOptions };
+        const orient = (frames: Frame[], orientation: number): Frame[] => {
+            if (applyOrientation && orientation > 1) for (const frame of frames) frame.__orient__(orientation);
+            return frames;
+        };
         const profile = (frames: Frame[], icc: Uint8Array | undefined, space?: jxl.ColorSpace): Frame[] => {
             const transform = !applyColorProfile ? null : icc ? profileTransform(icc) : space ? colorSpaceTransform(space.xy, space.curve) : null;
             if (transform) for (const frame of frames) transform(frame.bitmap);
@@ -467,37 +491,55 @@ export class Animation extends Array<Frame> {
 
         if (type === 'webp') {
             const decoded = webp.decodeAnimation(data, onlyExtractFirstFrame, maxPixels);
-            const frames = profile(canvasFrames(decoded.width, decoded.height, webp.composite(decoded)), decoded.icc);
+            const frames = orient(profile(canvasFrames(decoded.width, decoded.height, webp.composite(decoded)), decoded.icc), decoded.orientation);
             return new Animation(frames, fromPlays(decoded.loopCount));
         }
 
-        // (still PNGs decode like images, keeping their orientation as it is like before)
+        // (still PNGs decode like images)
         if (type === 'png' && !png.isAnimated(data)) {
-            const image = await Image.decode(data, { maxPixels, applyOrientation: false, applyColorProfile });
+            const image = await Image.decode(data, { maxPixels, applyOrientation, applyColorProfile });
             return new Animation(canvasFrames(image.width, image.height, [{ pixels: image.bitmap, duration: 0 }]), -1);
         }
         if (type === 'png') {
             const decoded = await png.decodeAnimation(data, onlyExtractFirstFrame, maxPixels);
-            const frames = profile(canvasFrames(decoded.width, decoded.height, png.compositeAnimation(decoded)), decoded.icc);
+            const frames = orient(profile(canvasFrames(decoded.width, decoded.height, png.compositeAnimation(decoded)), decoded.icc), decoded.orientation);
             return new Animation(frames, decoded.frames.length > 1 ? fromPlays(decoded.plays) : -1);
         }
 
         if (type === 'jxl') {
             const frames: Frame[] = [];
             let loopCount = -1, icc: Uint8Array | undefined, space: jxl.ColorSpace | undefined;
-            for await (const decoded of jxl.frames(data, maxPixels, 'reinhard', threads)) {
+            let orientation = 1;
+            for await (const decoded of jxl.frames(data, maxPixels, toneMapping, threads)) {
                 const frame = new Frame(decoded.width, decoded.height, decoded.duration, 0, 0, FrameDisposalModes.BACKGROUND);
                 frame.bitmap = decoded.pixels;
-                frames.push(frame.__orient__(decoded.orientation));
+                frames.push(frame);
+                orientation = decoded.orientation;
                 loopCount = fromPlays(decoded.loopCount);
                 icc = decoded.icc;
                 space = decoded.colorSpace;
                 if (onlyExtractFirstFrame) break;
             }
-            return new Animation(profile(frames, icc, space), frames.length > 1 ? loopCount : -1);
+            // (a gain map that can't be read leaves the frames without one, it doesn't fail the decoding)
+            const map = gainMap && data[0] !== 0xff ? await jxl.decodeGainMap(data, maxPixels).catch(() => undefined) : undefined;
+            if (map) for (const frame of frames) {
+                const image = new Image(map.width, map.height);
+                image.bitmap = map.pixels.slice();
+                frame.gainMap = GainMap.__from_pixels__(image, map.params);
+            }
+            for (const frame of orient(profile(frames, icc, space), orientation)) {
+                const mapOfFrame = frame.gainMap;
+                if (!mapOfFrame || !applyOrientation || orientation <= 1) continue;
+                try {
+                    mapOfFrame.image = mapOfFrame.image.__orient__(orientation);
+                } catch {
+                    frame.gainMap = null;
+                }
+            }
+            return new Animation(frames, frames.length > 1 ? loopCount : -1);
         }
 
-        if (type) return new Animation([Frame.from(await Image.decode(data, { maxPixels, applyColorProfile }), 0)]);
+        if (type) return new Animation([Frame.from(await Image.decode(data, { maxPixels, applyOrientation, applyColorProfile, toneMapping, gainMap, threads }), 0)]);
         throw new Error('Unsupported image type');
     }
 

src/codecs/jxl.ts

@@ -198,7 +198,7 @@ export async function decode(data: Uint8Array, maxPixels = MAX_PIXELS, toneMappi
 
 // the jhgm box's gain map (libjxl's JxlGainMapReadBundle): version 0, the ISO 21496-1 metadata (its size, 16 bits),
 // the alternate image's color encoding and ICC profile (their sizes, 8 and 32 bits), the map's codestream
-async function decodeGainMap(data: Uint8Array, maxPixels: number): Promise<{ width: number, height: number, pixels: Uint8Array, params: Params } | undefined> {
+export async function decodeGainMap(data: Uint8Array, maxPixels: number): Promise<{ width: number, height: number, pixels: Uint8Array, params: Params } | undefined> {
     for (const { type, content: c } of boxesOf(data)) {
         if (type !== 'jhgm') continue;
         if (c.length < 3 || c[0] !== 0) return undefined;

src/codecs/png.ts

@@ -718,16 +718,17 @@ export function isAnimated(array: Uint8Array): boolean {
  * a single frame
  * @return plays: 0 = forever
  */
-export async function decodeAnimation(array: Uint8Array, onlyFirstFrame = false, maxPixels = MAX_PIXELS): Promise<{ width: number, height: number, plays: number, frames: APNGFrame[], icc?: Uint8Array }> {
+export async function decodeAnimation(array: Uint8Array, onlyFirstFrame = false, maxPixels = MAX_PIXELS): Promise<{ width: number, height: number, plays: number, frames: APNGFrame[], icc?: Uint8Array, orientation: number }> {
     const p = parse(array);
     const { width, height } = p.header;
+    const orientation = p.exif ? exifOrientation(p.exif) : await textOrientation(p.text);
     // every frame is composited onto a canvas of its own
     checkPixels(width, height, maxPixels, !p.animation || p.animationError || onlyFirstFrame ? 1 : Math.max(1, p.frames.length));
 
     // broken animations show the default image, like browsers do
     if (!p.animation || p.animationError) {
         const pixels = await decodeImageData(p, p.idat, width, height);
-        return { width, height, plays: 0, frames: [{ x: 0, y: 0, width, height, duration: 0, dispose: 0, blend: 0, pixels }], icc: await profile(p.iccp) };
+        return { width, height, plays: 0, frames: [{ x: 0, y: 0, width, height, duration: 0, dispose: 0, blend: 0, pixels }], icc: await profile(p.iccp), orientation };
     }
 
     const frames: APNGFrame[] = [];
@@ -743,7 +744,7 @@ export async function decodeAnimation(array: Uint8Array, onlyFirstFrame = false,
         if (onlyFirstFrame) break;
     }
 
-    return { width, height, plays: p.animation.plays, frames, icc: await profile(p.iccp) };
+    return { width, height, plays: p.animation.plays, frames, icc: await profile(p.iccp), orientation };
 }
 
 /**

src/codecs/webp.ts

@@ -43,6 +43,8 @@ export interface WebPAnimation {
     frames: WebPFrame[];
     /** the embedded ICC profile */
     icc?: Uint8Array;
+    /** Exif orientation (1-8) */
+    orientation: number;
 }
 
 interface Chunk {
@@ -186,7 +188,7 @@ export function decodeAnimation(data: Uint8Array, onlyFirstFrame = false, maxPix
     const header = list[0];
     if (!header || header.type !== VP8X || !list.some(c => c.type === ANMF)) {
         const { width, height, pixels } = decodeBitstream(list, maxPixels);
-        return { width, height, loopCount: 0, background: 0, frames: [{ x: 0, y: 0, width, height, duration: 0, blend: false, dispose: false, pixels }], icc: list.find(c => c.type === ICCP)?.data };
+        return { width, height, loopCount: 0, background: 0, frames: [{ x: 0, y: 0, width, height, duration: 0, blend: false, dispose: false, pixels }], icc: list.find(c => c.type === ICCP)?.data, orientation: orientationOf(list) };
     }
 
     if (header.data.length < 10) throw new Error('WebP: invalid VP8X chunk');
@@ -221,7 +223,13 @@ export function decodeAnimation(data: Uint8Array, onlyFirstFrame = false, maxPix
     }
 
     if (!frames.length) throw new Error('WebP: animation without frames');
-    return { width, height, loopCount, background, frames, icc: list.find(c => c.type === ICCP)?.data };
+    return { width, height, loopCount, background, frames, icc: list.find(c => c.type === ICCP)?.data, orientation: orientationOf(list) };
+}
+
+// the Exif orientation of a WebP's chunks, 1 without one
+function orientationOf(list: Chunk[]): number {
+    const exif = list.find(c => c.type === EXIF);
+    return exif ? exifOrientation(exif.data) : 1;
 }
 
 /**

tests/regressions.js

@@ -507,6 +507,86 @@ const rgba = (image, x, y) => Image.colorToRGBA(image.getPixelAt(x, y));
         }
     }
 
+    // Animation.decode: orientation, tone mapping and gain maps like Image.decode
+    {
+        const {GainMap} = require('..');
+        const fs = require('fs');
+        const pixel = (image, x, y) => Array.from(image.bitmap.subarray((y * image.width + x) * 4, (y * image.width + x) * 4 + 4));
+        const tiff = orientation => Buffer.from([0x4d, 0x4d, 0, 42, 0, 0, 0, 8, 0, 1, 0x01, 0x12, 0, 3, 0, 0, 0, 1, 0, orientation, 0, 0, 0, 0, 0, 0, 0, 0]);
+        const source = () => [0, 1].map(i => Frame.from(new Image(3, 2).fill((x, y) => ColorHelpers.fromRGBA(x * 100, y * 100, i * 120, 255)), 100));
+
+        // APNG (eXIf) and animated WebP (EXIF): orientation 6 turns the top left into the top right
+        const png = Buffer.from(await new Animation(source(), 2).encodeAPNG({metadata: {software: ''}}));
+        const chunk = (type, body) => {
+            const out = Buffer.alloc(12 + body.length);
+            out.writeUInt32BE(body.length);
+            out.write(type, 4, 'latin1');
+            body.copy(out, 8);
+            out.writeUInt32BE(zlib.crc32(out.subarray(4, 8 + body.length)), 8 + body.length);
+            return out;
+        };
+        const withExif = Buffer.concat([png.subarray(0, png.length - 12), chunk('eXIf', tiff(6)), png.subarray(png.length - 12)]);
+        const webp = Buffer.from(await new Animation(source(), 2).encodeWebP({metadata: {software: ''}}));
+        const exifChunk = Buffer.concat([Buffer.from('EXIF'), Buffer.from(new Uint8Array(new Uint32Array([tiff(6).length]).buffer)), tiff(6)]);
+        // (the encoder's own EXIF chunk, with the creation time, becomes one decoders skip)
+        const own = webp.indexOf('EXIF', 12, 'latin1');
+        assert(own > 12, 'the animated WebP has an EXIF chunk of its own');
+        webp.write('JUNK', own, 'latin1');
+        const webpWithExif = Buffer.concat([webp, exifChunk]);
+        webpWithExif.writeUInt32LE(webpWithExif.length - 8, 4);
+        // (the VP8X flags: bit 3 is Exif)
+        webpWithExif[20] |= 0x08;
+
+        for (const [name, data] of [['APNG', withExif], ['WebP', webpWithExif]]) {
+            const stored = await Animation.decode(data, {applyOrientation: false});
+            assert(stored.width === 3 && stored.height === 2 && stored.length === 2, `${name}: applyOrientation false keeps the stored pixels`);
+            for (const options of [undefined, {}, {applyOrientation: true}]) {
+                const oriented = await Animation.decode(data, options);
+                assert(oriented.width === 2 && oriented.height === 3 && oriented.length === 2, `${name}: rotated (${JSON.stringify(options)})`);
+                for (let i = 0; i < 2; i++)
+                    assert(pixel(oriented[i], 1, 0).join() === pixel(stored[i], 0, 0).join() && pixel(oriented[i], 0, 2).join() === pixel(stored[i], 2, 1).join(), `${name}: frame ${i} not oriented like Image.decode`);
+            }
+        }
+        // (without an orientation tag nothing changes)
+        assert((await Animation.decode(png)).width === 3, 'APNG without orientation');
+
+        // JPEG XL: the gain map of a jhgm box on every frame, in a container around the animation's codestream
+        const boxes = data => {
+            const out = [];
+            for (let o = 0; o < data.length;) {
+                const size = data.readUInt32BE(o), type = data.toString('latin1', o + 4, o + 8);
+                out.push({type, box: data.subarray(o, o + size)});
+                o += size;
+            }
+            return out;
+        };
+        const box = (type, ...body) => {
+            const content = Buffer.concat(body);
+            const out = Buffer.alloc(8 + content.length);
+            out.writeUInt32BE(out.length);
+            out.write(type, 4, 'latin1');
+            content.copy(out, 8);
+            return out;
+        };
+        const mapped = new Image(8, 8).fill(0x808080ff);
+        mapped.gainMap = new GainMap(new Image(4, 4).fill(0xc0c0c0ff), {gainMapMin: 0, gainMapMax: 2});
+        const jhgm = boxes(Buffer.from(await mapped.encodeJXL())).find(b => b.type === 'jhgm').box;
+        const codestream = fs.readFileSync('./tests/jxl/animation.jxl');
+        const container = Buffer.concat([box('JXL ', Buffer.from([0x0d, 0x0a, 0x87, 0x0a])), box('ftyp', Buffer.from('jxl \0\0\0\0jxl ', 'latin1')), jhgm, box('jxlc', codestream)]);
+        const plain = await Animation.decode(container, {gainMap: false});
+        const withMap = await Animation.decode(container);
+        assert(plain.length > 1 && withMap.length === plain.length, 'JPEG XL animation with a jhgm box decodes');
+        assert(plain.every(f => f.gainMap === null) && (await Animation.decode(codestream)).every(f => f.gainMap === null), 'JPEG XL: gainMap false, or no jhgm box');
+        assert(withMap.every(f => f.gainMap && f.gainMap.width === 4 && f.gainMap.gainMapMax[0] === 2), 'JPEG XL: every frame has the gain map');
+        assert(withMap[0].gainMap !== withMap[1].gainMap && withMap.every((f, i) => Buffer.from(f.bitmap).equals(Buffer.from(plain[i].bitmap))), 'JPEG XL: the frames are the same with the map, with maps of their own');
+
+        // toneMapping is checked
+        let threw = false;
+        try { await Animation.decode(png, {toneMapping: 'linear'}); } catch (e) { threw = e instanceof TypeError; }
+        assert(threw, 'Animation.decode validates toneMapping');
+        assert((await Animation.decode(codestream, {toneMapping: 'clamp'})).length === plain.length, 'Animation.decode with toneMapping clamp');
+    }
+
     // Colors: the color helpers, and the deprecated Image ones they replaced, give the same values
     {
         const { Colors } = require('..');