Show transparent images frameless, like stickers

src-tauri/src/commands.rs

@@ -474,12 +474,20 @@ pub async fn get_event_body(
 /// request simply never arrives and the element reports "format not supported".
 /// So media comes over IPC instead and the frontend wraps it in a `blob:` URL,
 /// which AVFoundation will load.
+///
+/// It's also how the frontend reads an image's pixels: one loaded from
+/// `uwum://` is cross-origin, so a canvas it's drawn into can't be read back.
 #[tauri::command]
 pub async fn get_media_bytes(
     state: State<'_, AppState>,
     mxc: String,
+    width: Option<u32>,
+    height: Option<u32>,
 ) -> Result<tauri::ipc::Response> {
-    let media = media::fetch(&state.core().await?.client, &mxc, None, None).await?;
+    // A size asks for the same thumbnail an `<img>` would get, so a caller that
+    // wants to look at what's already on screen hits the media cache.
+    let size = width.zip(height);
+    let media = media::fetch(&state.core().await?.client, &mxc, size, None).await?;
     Ok(tauri::ipc::Response::new(media.bytes))
 }
 

src/components/MessageRow.tsx

@@ -11,6 +11,7 @@ import {
 import * as ipc from "../lib/ipc";
 import { mediaUrl } from "../lib/ipc";
 import { useMediaBlob } from "../lib/blobMedia";
+import { useIsCutout } from "../lib/cutout";
 import { saveAttachment } from "../lib/download";
 import { imageLookup, reactionImage, reactionKeyFor } from "../lib/packs";
 import { linkify, renderFormattedBody } from "../lib/richText";
@@ -954,6 +955,12 @@ function ImageBody({
     height: Math.round(height),
   });
 
+  // A transparent image is drawn the way a sticker is: on the timeline itself,
+  // not on a card. It keeps the size it would have had as a picture, so
+  // finding out doesn't shift the timeline under the reader.
+  const cutout = useIsCutout(sticker ? null : media.mxc, media.mimetype, { width, height });
+  const frameless = sticker || cutout;
+
   if (!src) return <div style={{ fontSize: 13, color: "var(--text-tertiary)" }}>{body}</div>;
 
   return (
@@ -976,10 +983,12 @@ function ImageBody({
         maxWidth: "100%",
         height: "auto",
         aspectRatio: `${Math.round(width)} / ${Math.round(height)}`,
-        objectFit: "cover",
-        borderRadius: sticker ? 8 : 16,
-        border: sticker ? "none" : "1px solid var(--border-subtle)",
-        background: "var(--surface-card)",
+        // `contain` for a cut-out: cropping one would slice off the edges of
+        // whatever was cut out.
+        objectFit: cutout ? "contain" : "cover",
+        borderRadius: frameless ? 8 : 16,
+        border: frameless ? "none" : "1px solid var(--border-subtle)",
+        background: cutout ? "transparent" : "var(--surface-card)",
       }}
     />
   );

src/lib/cutout.test.ts

@@ -0,0 +1,34 @@
+import { describe, expect, it } from "vitest";
+import { isCutout } from "./cutout";
+
+/** `total` RGBA pixels, the first `clear` of them fully transparent. */
+function pixels(total: number, clear: number): Uint8ClampedArray {
+  const rgba = new Uint8ClampedArray(total * 4).fill(255);
+  for (let i = 0; i < clear; i++) rgba[i * 4 + 3] = 0;
+  return rgba;
+}
+
+describe("isCutout", () => {
+  it("leaves an opaque image a picture", () => {
+    expect(isCutout(pixels(2304, 0))).toBe(false);
+  });
+
+  it("leaves a screenshot with rounded corners a picture", () => {
+    // Four small transparent corners on a 48×48 sample: well under 4%.
+    expect(isCutout(pixels(2304, 40))).toBe(false);
+  });
+
+  it("calls a character on an empty background a cut-out", () => {
+    expect(isCutout(pixels(2304, 1200))).toBe(true);
+  });
+
+  it("doesn't count half-transparent pixels as see-through", () => {
+    const rgba = new Uint8ClampedArray(2304 * 4).fill(255);
+    for (let i = 3; i < rgba.length; i += 4) rgba[i] = 200;
+    expect(isCutout(rgba)).toBe(false);
+  });
+
+  it("has nothing to say about no pixels", () => {
+    expect(isCutout(new Uint8ClampedArray())).toBe(false);
+  });
+});

src/lib/cutout.ts

@@ -0,0 +1,158 @@
+/**
+ * Telling a cut-out apart from a photo.
+ *
+ * A transparent PNG pasted into a room is nearly always meant as a sticker — a
+ * character, a reaction, something cropped out of its background — and drawing
+ * it on a card with a border puts back the very box it was cut out of. So an
+ * image that's mostly see-through is shown frameless, the way a sticker is.
+ *
+ * Only the pixels can say. The format can't (a PNG screenshot is opaque), and
+ * neither can an alpha channel (plenty of opaque PNGs carry one anyway). And
+ * since an `<img>` from `uwum://` is cross-origin and would taint a canvas, the
+ * bytes come over IPC — the same thumbnail the timeline shows, so it's a
+ * cache hit rather than a second download.
+ */
+
+import { invoke } from "@tauri-apps/api/core";
+import { useEffect, useState } from "react";
+import { thumbnailSize } from "./ipc";
+
+/** Formats that can be transparent at all; anything else is never asked about. */
+const ALPHA_FORMATS = new Set(["image/png", "image/apng", "image/webp", "image/gif", "image/avif"]);
+
+/**
+ * The share of see-through pixels that makes an image a cut-out.
+ *
+ * Low enough to catch a character on an empty background, high enough that a
+ * screenshot with rounded corners stays a picture.
+ */
+const CUTOUT_SHARE = 0.04;
+
+/** Below this alpha a pixel counts as see-through. */
+const SEE_THROUGH = 128;
+
+/** How small the image is drawn before counting — plenty to judge by. */
+const SAMPLE = 48;
+
+/**
+ * Whether these RGBA pixels are a cut-out.
+ *
+ * Separate from the decoding so it can be tested without a canvas.
+ */
+export function isCutout(rgba: Uint8ClampedArray): boolean {
+  const pixels = rgba.length / 4;
+  if (pixels === 0) return false;
+
+  let clear = 0;
+  for (let i = 3; i < rgba.length; i += 4) {
+    if (rgba[i] < SEE_THROUGH) clear += 1;
+  }
+  return clear / pixels >= CUTOUT_SHARE;
+}
+
+// Answers are remembered across launches: they never change for a given mxc,
+// and without this every transparent image would flash its frame once per
+// session before losing it.
+const STORAGE_KEY = "uwum:cutouts";
+const REMEMBERED = 500;
+
+const known = loadKnown();
+const inFlight = new Map<string, Promise<boolean>>();
+
+function loadKnown(): Map<string, boolean> {
+  try {
+    const raw = localStorage.getItem(STORAGE_KEY);
+    if (raw) return new Map(JSON.parse(raw) as [string, boolean][]);
+  } catch {
+    // Unreadable or blocked storage just means asking again.
+  }
+  return new Map();
+}
+
+function remember(mxc: string, cutout: boolean) {
+  known.set(mxc, cutout);
+  // Oldest first, so dropping from the front keeps the recent ones.
+  while (known.size > REMEMBERED) known.delete(known.keys().next().value as string);
+  try {
+    localStorage.setItem(STORAGE_KEY, JSON.stringify([...known]));
+  } catch {
+    // Not worth failing over; it's only a cache.
+  }
+}
+
+async function inspect(
+  mxc: string,
+  mimetype: string,
+  size: { width: number; height: number },
+): Promise<boolean> {
+  const { width, height } = thumbnailSize(size);
+  const bytes = await invoke<ArrayBuffer>("get_media_bytes", { mxc, width, height });
+  const bitmap = await createImageBitmap(new Blob([bytes], { type: mimetype }));
+
+  try {
+    const canvas = document.createElement("canvas");
+    canvas.width = SAMPLE;
+    canvas.height = SAMPLE;
+    const context = canvas.getContext("2d", { willReadFrequently: true });
+    if (!context) return false;
+    context.drawImage(bitmap, 0, 0, SAMPLE, SAMPLE);
+    return isCutout(context.getImageData(0, 0, SAMPLE, SAMPLE).data);
+  } finally {
+    bitmap.close();
+  }
+}
+
+/**
+ * Whether the image at `mxc` is a cut-out, once that's known.
+ *
+ * `false` until then, and for good if it can't be checked — showing a sticker
+ * with a frame is a much smaller mistake than losing one around a photo.
+ */
+export function useIsCutout(
+  mxc: string | null | undefined,
+  mimetype: string | null,
+  size: { width: number; height: number },
+): boolean {
+  const candidate = mxc && mimetype && ALPHA_FORMATS.has(mimetype) ? mxc : null;
+  const [cutout, setCutout] = useState(() => (candidate ? (known.get(candidate) ?? false) : false));
+
+  const { width, height } = size;
+  useEffect(() => {
+    if (!candidate || !mimetype) {
+      setCutout(false);
+      return;
+    }
+
+    const remembered = known.get(candidate);
+    if (remembered !== undefined) {
+      setCutout(remembered);
+      return;
+    }
+
+    let cancelled = false;
+    let request = inFlight.get(candidate);
+    if (!request) {
+      // A failure isn't remembered: it's an answer for this session, not
+      // for the image, and the next launch may well be online.
+      request = inspect(candidate, mimetype, { width, height })
+        .then(
+          (answer) => {
+            remember(candidate, answer);
+            return answer;
+          },
+          () => false,
+        )
+        .finally(() => inFlight.delete(candidate));
+      inFlight.set(candidate, request);
+    }
+
+    void request.then((answer) => {
+      if (!cancelled) setCutout(answer);
+    });
+    return () => {
+      cancelled = true;
+    };
+  }, [candidate, mimetype, width, height]);
+
+  return cutout;
+}

src/lib/ipc.ts

@@ -349,11 +349,23 @@ export function mediaUrl(
     : `uwum://media/${encodeURIComponent(mxc)}`;
   if (!size) return base;
 
+  const { width, height } = thumbnailSize(size);
+  return `${base}?w=${width}&h=${height}`;
+}
+
+/**
+ * The thumbnail actually requested for an image shown at `size`.
+ *
+ * Exported so anything else fetching the same picture asks for the same bytes
+ * and shares the cache entry.
+ */
+export function thumbnailSize(size: { width: number; height: number }): {
+  width: number;
+  height: number;
+} {
   // Ask for 2× so the image stays sharp on retina displays.
   const scale = Math.min(window.devicePixelRatio || 1, 2);
-  const w = snapUp(size.width * scale);
-  const h = snapUp(size.height * scale);
-  return `${base}?w=${w}&h=${h}`;
+  return { width: snapUp(size.width * scale), height: snapUp(size.height * scale) };
 }
 
 /** The only thumbnail sizes we ever ask a homeserver for. */