diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index d839192..71e63a7 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -474,12 +474,20 @@ pub async fn get_event_body( /// request simply never arrives and the element reports "format not supported". /// So media comes over IPC instead and the frontend wraps it in a `blob:` URL, /// which AVFoundation will load. +/// +/// It's also how the frontend reads an image's pixels: one loaded from +/// `uwum://` is cross-origin, so a canvas it's drawn into can't be read back. #[tauri::command] pub async fn get_media_bytes( state: State<'_, AppState>, mxc: String, + width: Option, + height: Option, ) -> Result { - let media = media::fetch(&state.core().await?.client, &mxc, None, None).await?; + // A size asks for the same thumbnail an `` would get, so a caller that + // wants to look at what's already on screen hits the media cache. + let size = width.zip(height); + let media = media::fetch(&state.core().await?.client, &mxc, size, None).await?; Ok(tauri::ipc::Response::new(media.bytes)) } diff --git a/src/components/MessageRow.tsx b/src/components/MessageRow.tsx index 180e754..5165bed 100644 --- a/src/components/MessageRow.tsx +++ b/src/components/MessageRow.tsx @@ -11,6 +11,7 @@ import { import * as ipc from "../lib/ipc"; import { mediaUrl } from "../lib/ipc"; import { useMediaBlob } from "../lib/blobMedia"; +import { useIsCutout } from "../lib/cutout"; import { saveAttachment } from "../lib/download"; import { imageLookup, reactionImage, reactionKeyFor } from "../lib/packs"; import { linkify, renderFormattedBody } from "../lib/richText"; @@ -954,6 +955,12 @@ function ImageBody({ height: Math.round(height), }); + // A transparent image is drawn the way a sticker is: on the timeline itself, + // not on a card. It keeps the size it would have had as a picture, so + // finding out doesn't shift the timeline under the reader. + const cutout = useIsCutout(sticker ? null : media.mxc, media.mimetype, { width, height }); + const frameless = sticker || cutout; + if (!src) return
{body}
; return ( @@ -976,10 +983,12 @@ function ImageBody({ maxWidth: "100%", height: "auto", aspectRatio: `${Math.round(width)} / ${Math.round(height)}`, - objectFit: "cover", - borderRadius: sticker ? 8 : 16, - border: sticker ? "none" : "1px solid var(--border-subtle)", - background: "var(--surface-card)", + // `contain` for a cut-out: cropping one would slice off the edges of + // whatever was cut out. + objectFit: cutout ? "contain" : "cover", + borderRadius: frameless ? 8 : 16, + border: frameless ? "none" : "1px solid var(--border-subtle)", + background: cutout ? "transparent" : "var(--surface-card)", }} /> ); diff --git a/src/lib/cutout.test.ts b/src/lib/cutout.test.ts new file mode 100644 index 0000000..a68b2f7 --- /dev/null +++ b/src/lib/cutout.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { isCutout } from "./cutout"; + +/** `total` RGBA pixels, the first `clear` of them fully transparent. */ +function pixels(total: number, clear: number): Uint8ClampedArray { + const rgba = new Uint8ClampedArray(total * 4).fill(255); + for (let i = 0; i < clear; i++) rgba[i * 4 + 3] = 0; + return rgba; +} + +describe("isCutout", () => { + it("leaves an opaque image a picture", () => { + expect(isCutout(pixels(2304, 0))).toBe(false); + }); + + it("leaves a screenshot with rounded corners a picture", () => { + // Four small transparent corners on a 48×48 sample: well under 4%. + expect(isCutout(pixels(2304, 40))).toBe(false); + }); + + it("calls a character on an empty background a cut-out", () => { + expect(isCutout(pixels(2304, 1200))).toBe(true); + }); + + it("doesn't count half-transparent pixels as see-through", () => { + const rgba = new Uint8ClampedArray(2304 * 4).fill(255); + for (let i = 3; i < rgba.length; i += 4) rgba[i] = 200; + expect(isCutout(rgba)).toBe(false); + }); + + it("has nothing to say about no pixels", () => { + expect(isCutout(new Uint8ClampedArray())).toBe(false); + }); +}); diff --git a/src/lib/cutout.ts b/src/lib/cutout.ts new file mode 100644 index 0000000..a1219f7 --- /dev/null +++ b/src/lib/cutout.ts @@ -0,0 +1,158 @@ +/** + * Telling a cut-out apart from a photo. + * + * A transparent PNG pasted into a room is nearly always meant as a sticker — a + * character, a reaction, something cropped out of its background — and drawing + * it on a card with a border puts back the very box it was cut out of. So an + * image that's mostly see-through is shown frameless, the way a sticker is. + * + * Only the pixels can say. The format can't (a PNG screenshot is opaque), and + * neither can an alpha channel (plenty of opaque PNGs carry one anyway). And + * since an `` from `uwum://` is cross-origin and would taint a canvas, the + * bytes come over IPC — the same thumbnail the timeline shows, so it's a + * cache hit rather than a second download. + */ + +import { invoke } from "@tauri-apps/api/core"; +import { useEffect, useState } from "react"; +import { thumbnailSize } from "./ipc"; + +/** Formats that can be transparent at all; anything else is never asked about. */ +const ALPHA_FORMATS = new Set(["image/png", "image/apng", "image/webp", "image/gif", "image/avif"]); + +/** + * The share of see-through pixels that makes an image a cut-out. + * + * Low enough to catch a character on an empty background, high enough that a + * screenshot with rounded corners stays a picture. + */ +const CUTOUT_SHARE = 0.04; + +/** Below this alpha a pixel counts as see-through. */ +const SEE_THROUGH = 128; + +/** How small the image is drawn before counting — plenty to judge by. */ +const SAMPLE = 48; + +/** + * Whether these RGBA pixels are a cut-out. + * + * Separate from the decoding so it can be tested without a canvas. + */ +export function isCutout(rgba: Uint8ClampedArray): boolean { + const pixels = rgba.length / 4; + if (pixels === 0) return false; + + let clear = 0; + for (let i = 3; i < rgba.length; i += 4) { + if (rgba[i] < SEE_THROUGH) clear += 1; + } + return clear / pixels >= CUTOUT_SHARE; +} + +// Answers are remembered across launches: they never change for a given mxc, +// and without this every transparent image would flash its frame once per +// session before losing it. +const STORAGE_KEY = "uwum:cutouts"; +const REMEMBERED = 500; + +const known = loadKnown(); +const inFlight = new Map>(); + +function loadKnown(): Map { + try { + const raw = localStorage.getItem(STORAGE_KEY); + if (raw) return new Map(JSON.parse(raw) as [string, boolean][]); + } catch { + // Unreadable or blocked storage just means asking again. + } + return new Map(); +} + +function remember(mxc: string, cutout: boolean) { + known.set(mxc, cutout); + // Oldest first, so dropping from the front keeps the recent ones. + while (known.size > REMEMBERED) known.delete(known.keys().next().value as string); + try { + localStorage.setItem(STORAGE_KEY, JSON.stringify([...known])); + } catch { + // Not worth failing over; it's only a cache. + } +} + +async function inspect( + mxc: string, + mimetype: string, + size: { width: number; height: number }, +): Promise { + const { width, height } = thumbnailSize(size); + const bytes = await invoke("get_media_bytes", { mxc, width, height }); + const bitmap = await createImageBitmap(new Blob([bytes], { type: mimetype })); + + try { + const canvas = document.createElement("canvas"); + canvas.width = SAMPLE; + canvas.height = SAMPLE; + const context = canvas.getContext("2d", { willReadFrequently: true }); + if (!context) return false; + context.drawImage(bitmap, 0, 0, SAMPLE, SAMPLE); + return isCutout(context.getImageData(0, 0, SAMPLE, SAMPLE).data); + } finally { + bitmap.close(); + } +} + +/** + * Whether the image at `mxc` is a cut-out, once that's known. + * + * `false` until then, and for good if it can't be checked — showing a sticker + * with a frame is a much smaller mistake than losing one around a photo. + */ +export function useIsCutout( + mxc: string | null | undefined, + mimetype: string | null, + size: { width: number; height: number }, +): boolean { + const candidate = mxc && mimetype && ALPHA_FORMATS.has(mimetype) ? mxc : null; + const [cutout, setCutout] = useState(() => (candidate ? (known.get(candidate) ?? false) : false)); + + const { width, height } = size; + useEffect(() => { + if (!candidate || !mimetype) { + setCutout(false); + return; + } + + const remembered = known.get(candidate); + if (remembered !== undefined) { + setCutout(remembered); + return; + } + + let cancelled = false; + let request = inFlight.get(candidate); + if (!request) { + // A failure isn't remembered: it's an answer for this session, not + // for the image, and the next launch may well be online. + request = inspect(candidate, mimetype, { width, height }) + .then( + (answer) => { + remember(candidate, answer); + return answer; + }, + () => false, + ) + .finally(() => inFlight.delete(candidate)); + inFlight.set(candidate, request); + } + + void request.then((answer) => { + if (!cancelled) setCutout(answer); + }); + return () => { + cancelled = true; + }; + }, [candidate, mimetype, width, height]); + + return cutout; +} diff --git a/src/lib/ipc.ts b/src/lib/ipc.ts index 7b4d35e..2a74faf 100644 --- a/src/lib/ipc.ts +++ b/src/lib/ipc.ts @@ -349,11 +349,23 @@ export function mediaUrl( : `uwum://media/${encodeURIComponent(mxc)}`; if (!size) return base; + const { width, height } = thumbnailSize(size); + return `${base}?w=${width}&h=${height}`; +} + +/** + * The thumbnail actually requested for an image shown at `size`. + * + * Exported so anything else fetching the same picture asks for the same bytes + * and shares the cache entry. + */ +export function thumbnailSize(size: { width: number; height: number }): { + width: number; + height: number; +} { // Ask for 2× so the image stays sharp on retina displays. const scale = Math.min(window.devicePixelRatio || 1, 2); - const w = snapUp(size.width * scale); - const h = snapUp(size.height * scale); - return `${base}?w=${w}&h=${h}`; + return { width: snapUp(size.width * scale), height: snapUp(size.height * scale) }; } /** The only thumbnail sizes we ever ask a homeserver for. */