From 08b708c68effca63f10b7226c21153d832c5c482 Mon Sep 17 00:00:00 2001 From: Gintautas Kazlauskas Date: Wed, 7 Oct 2026 22:07:31 +0300 Subject: [PATCH] feat: show transparent images frameless, like stickers A mostly see-through PNG/WebP/GIF/AVIF is nearly always meant as a sticker, and drawing it on a bordered card puts back the box it was cut out of. Check the pixels of the thumbnail already on screen (over IPC, so the canvas isn't tainted) and drop the frame, background and cropping for cut-outs. Size is unchanged so the timeline doesn't shift; answers are remembered across launches. get_media_bytes takes an optional size so the check hits the same cache entry as the . Co-Authored-By: Claude Opus 5.5 --- src-tauri/src/commands.rs | 10 ++- src/components/MessageRow.tsx | 17 +++- src/lib/cutout.test.ts | 34 ++++++++ src/lib/cutout.ts | 158 ++++++++++++++++++++++++++++++++++ src/lib/ipc.ts | 18 +++- 5 files changed, 229 insertions(+), 8 deletions(-) create mode 100644 src/lib/cutout.test.ts create mode 100644 src/lib/cutout.ts diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index d839192..71e63a7 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -474,12 +474,20 @@ pub async fn get_event_body( /// request simply never arrives and the element reports "format not supported". /// So media comes over IPC instead and the frontend wraps it in a `blob:` URL, /// which AVFoundation will load. +/// +/// It's also how the frontend reads an image's pixels: one loaded from +/// `uwum://` is cross-origin, so a canvas it's drawn into can't be read back. #[tauri::command] pub async fn get_media_bytes( state: State<'_, AppState>, mxc: String, + width: Option, + height: Option, ) -> Result { - let media = media::fetch(&state.core().await?.client, &mxc, None, None).await?; + // A size asks for the same thumbnail an `` would get, so a caller that + // wants to look at what's already on screen hits the media cache. + let size = width.zip(height); + let media = media::fetch(&state.core().await?.client, &mxc, size, None).await?; Ok(tauri::ipc::Response::new(media.bytes)) } diff --git a/src/components/MessageRow.tsx b/src/components/MessageRow.tsx index 180e754..5165bed 100644 --- a/src/components/MessageRow.tsx +++ b/src/components/MessageRow.tsx @@ -11,6 +11,7 @@ import { import * as ipc from "../lib/ipc"; import { mediaUrl } from "../lib/ipc"; import { useMediaBlob } from "../lib/blobMedia"; +import { useIsCutout } from "../lib/cutout"; import { saveAttachment } from "../lib/download"; import { imageLookup, reactionImage, reactionKeyFor } from "../lib/packs"; import { linkify, renderFormattedBody } from "../lib/richText"; @@ -954,6 +955,12 @@ function ImageBody({ height: Math.round(height), }); + // A transparent image is drawn the way a sticker is: on the timeline itself, + // not on a card. It keeps the size it would have had as a picture, so + // finding out doesn't shift the timeline under the reader. + const cutout = useIsCutout(sticker ? null : media.mxc, media.mimetype, { width, height }); + const frameless = sticker || cutout; + if (!src) return
{body}
; return ( @@ -976,10 +983,12 @@ function ImageBody({ maxWidth: "100%", height: "auto", aspectRatio: `${Math.round(width)} / ${Math.round(height)}`, - objectFit: "cover", - borderRadius: sticker ? 8 : 16, - border: sticker ? "none" : "1px solid var(--border-subtle)", - background: "var(--surface-card)", + // `contain` for a cut-out: cropping one would slice off the edges of + // whatever was cut out. + objectFit: cutout ? "contain" : "cover", + borderRadius: frameless ? 8 : 16, + border: frameless ? "none" : "1px solid var(--border-subtle)", + background: cutout ? "transparent" : "var(--surface-card)", }} /> ); diff --git a/src/lib/cutout.test.ts b/src/lib/cutout.test.ts new file mode 100644 index 0000000..a68b2f7 --- /dev/null +++ b/src/lib/cutout.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { isCutout } from "./cutout"; + +/** `total` RGBA pixels, the first `clear` of them fully transparent. */ +function pixels(total: number, clear: number): Uint8ClampedArray { + const rgba = new Uint8ClampedArray(total * 4).fill(255); + for (let i = 0; i < clear; i++) rgba[i * 4 + 3] = 0; + return rgba; +} + +describe("isCutout", () => { + it("leaves an opaque image a picture", () => { + expect(isCutout(pixels(2304, 0))).toBe(false); + }); + + it("leaves a screenshot with rounded corners a picture", () => { + // Four small transparent corners on a 48×48 sample: well under 4%. + expect(isCutout(pixels(2304, 40))).toBe(false); + }); + + it("calls a character on an empty background a cut-out", () => { + expect(isCutout(pixels(2304, 1200))).toBe(true); + }); + + it("doesn't count half-transparent pixels as see-through", () => { + const rgba = new Uint8ClampedArray(2304 * 4).fill(255); + for (let i = 3; i < rgba.length; i += 4) rgba[i] = 200; + expect(isCutout(rgba)).toBe(false); + }); + + it("has nothing to say about no pixels", () => { + expect(isCutout(new Uint8ClampedArray())).toBe(false); + }); +}); diff --git a/src/lib/cutout.ts b/src/lib/cutout.ts new file mode 100644 index 0000000..a1219f7 --- /dev/null +++ b/src/lib/cutout.ts @@ -0,0 +1,158 @@ +/** + * Telling a cut-out apart from a photo. + * + * A transparent PNG pasted into a room is nearly always meant as a sticker — a + * character, a reaction, something cropped out of its background — and drawing + * it on a card with a border puts back the very box it was cut out of. So an + * image that's mostly see-through is shown frameless, the way a sticker is. + * + * Only the pixels can say. The format can't (a PNG screenshot is opaque), and + * neither can an alpha channel (plenty of opaque PNGs carry one anyway). And + * since an `` from `uwum://` is cross-origin and would taint a canvas, the + * bytes come over IPC — the same thumbnail the timeline shows, so it's a + * cache hit rather than a second download. + */ + +import { invoke } from "@tauri-apps/api/core"; +import { useEffect, useState } from "react"; +import { thumbnailSize } from "./ipc"; + +/** Formats that can be transparent at all; anything else is never asked about. */ +const ALPHA_FORMATS = new Set(["image/png", "image/apng", "image/webp", "image/gif", "image/avif"]); + +/** + * The share of see-through pixels that makes an image a cut-out. + * + * Low enough to catch a character on an empty background, high enough that a + * screenshot with rounded corners stays a picture. + */ +const CUTOUT_SHARE = 0.04; + +/** Below this alpha a pixel counts as see-through. */ +const SEE_THROUGH = 128; + +/** How small the image is drawn before counting — plenty to judge by. */ +const SAMPLE = 48; + +/** + * Whether these RGBA pixels are a cut-out. + * + * Separate from the decoding so it can be tested without a canvas. + */ +export function isCutout(rgba: Uint8ClampedArray): boolean { + const pixels = rgba.length / 4; + if (pixels === 0) return false; + + let clear = 0; + for (let i = 3; i < rgba.length; i += 4) { + if (rgba[i] < SEE_THROUGH) clear += 1; + } + return clear / pixels >= CUTOUT_SHARE; +} + +// Answers are remembered across launches: they never change for a given mxc, +// and without this every transparent image would flash its frame once per +// session before losing it. +const STORAGE_KEY = "uwum:cutouts"; +const REMEMBERED = 500; + +const known = loadKnown(); +const inFlight = new Map>(); + +function loadKnown(): Map { + try { + const raw = localStorage.getItem(STORAGE_KEY); + if (raw) return new Map(JSON.parse(raw) as [string, boolean][]); + } catch { + // Unreadable or blocked storage just means asking again. + } + return new Map(); +} + +function remember(mxc: string, cutout: boolean) { + known.set(mxc, cutout); + // Oldest first, so dropping from the front keeps the recent ones. + while (known.size > REMEMBERED) known.delete(known.keys().next().value as string); + try { + localStorage.setItem(STORAGE_KEY, JSON.stringify([...known])); + } catch { + // Not worth failing over; it's only a cache. + } +} + +async function inspect( + mxc: string, + mimetype: string, + size: { width: number; height: number }, +): Promise { + const { width, height } = thumbnailSize(size); + const bytes = await invoke("get_media_bytes", { mxc, width, height }); + const bitmap = await createImageBitmap(new Blob([bytes], { type: mimetype })); + + try { + const canvas = document.createElement("canvas"); + canvas.width = SAMPLE; + canvas.height = SAMPLE; + const context = canvas.getContext("2d", { willReadFrequently: true }); + if (!context) return false; + context.drawImage(bitmap, 0, 0, SAMPLE, SAMPLE); + return isCutout(context.getImageData(0, 0, SAMPLE, SAMPLE).data); + } finally { + bitmap.close(); + } +} + +/** + * Whether the image at `mxc` is a cut-out, once that's known. + * + * `false` until then, and for good if it can't be checked — showing a sticker + * with a frame is a much smaller mistake than losing one around a photo. + */ +export function useIsCutout( + mxc: string | null | undefined, + mimetype: string | null, + size: { width: number; height: number }, +): boolean { + const candidate = mxc && mimetype && ALPHA_FORMATS.has(mimetype) ? mxc : null; + const [cutout, setCutout] = useState(() => (candidate ? (known.get(candidate) ?? false) : false)); + + const { width, height } = size; + useEffect(() => { + if (!candidate || !mimetype) { + setCutout(false); + return; + } + + const remembered = known.get(candidate); + if (remembered !== undefined) { + setCutout(remembered); + return; + } + + let cancelled = false; + let request = inFlight.get(candidate); + if (!request) { + // A failure isn't remembered: it's an answer for this session, not + // for the image, and the next launch may well be online. + request = inspect(candidate, mimetype, { width, height }) + .then( + (answer) => { + remember(candidate, answer); + return answer; + }, + () => false, + ) + .finally(() => inFlight.delete(candidate)); + inFlight.set(candidate, request); + } + + void request.then((answer) => { + if (!cancelled) setCutout(answer); + }); + return () => { + cancelled = true; + }; + }, [candidate, mimetype, width, height]); + + return cutout; +} diff --git a/src/lib/ipc.ts b/src/lib/ipc.ts index 7b4d35e..2a74faf 100644 --- a/src/lib/ipc.ts +++ b/src/lib/ipc.ts @@ -349,11 +349,23 @@ export function mediaUrl( : `uwum://media/${encodeURIComponent(mxc)}`; if (!size) return base; + const { width, height } = thumbnailSize(size); + return `${base}?w=${width}&h=${height}`; +} + +/** + * The thumbnail actually requested for an image shown at `size`. + * + * Exported so anything else fetching the same picture asks for the same bytes + * and shares the cache entry. + */ +export function thumbnailSize(size: { width: number; height: number }): { + width: number; + height: number; +} { // Ask for 2× so the image stays sharp on retina displays. const scale = Math.min(window.devicePixelRatio || 1, 2); - const w = snapUp(size.width * scale); - const h = snapUp(size.height * scale); - return `${base}?w=${w}&h=${h}`; + return { width: snapUp(size.width * scale), height: snapUp(size.height * scale) }; } /** The only thumbnail sizes we ever ask a homeserver for. */