ritmo-tools pixel: read the grid on the subject, and never call an upscale 1x art

.changeset/pixel-grid-subject.md

@@ -0,0 +1,8 @@
+---
+"@typeonce/ritmo-tools": minor
+---
+
+`ritmo-tools pixel`: find the grid of a small sprite on a large canvas, and never call an upscale pixel art at its own size. Measured with the benchmark of [xprite-pixelate](https://github.com/rhinoc/xprite-pixelate) (MIT) and [pixel-bench](https://github.com/Retro-Diffusion/pixel-bench) (MIT).
+
+- **The grid is read on the subject.** `pixel check`, `pixel fix`, and the page find the cell size on the picture's visible pixels, or, on an opaque picture whose border is one color (an image generator's off-white page), on the pixels away from that color, then place the grid on the whole picture. A sprite filling a fifth of its canvas came out with cells several times too large; on the AI benchmark, exact sizes rise from 65% to 76%, and from 24% to 99% for a small sprite on a clear canvas.
+- **An upscale is not pixel art at its own size.** A picture whose neighbouring pixels rarely change color and that has more than 32 colors is judged as a fix, so a grid that cannot be found reports `needs-person` ("Unsure of the grid") where it reported `clean` or `fixable` at the picture's full size. Flat pixel art of a few colors (a parallax layer) is still pixel art at its own size.

packages/tools/README.md

@@ -115,7 +115,9 @@ node packages/tools/bin/ritmo-tools.mjs pixel check art.png   # from its source,
 
 ### How to use it
 
-1. **Grid.** The grid is detected when a picture opens. The note under
+1. **Grid.** The grid is detected when a picture opens, from its subject
+   only: a small sprite on a large clear or one-color canvas finds its own
+   cells. The note under
    **Auto-detect** gives the cell size, the output size, and a confidence. Below
    about 50%, check the pink grid lines against the picture. Fix it by typing
    the cell size and offset (fractions are fine: AI pictures are often scaled

packages/tools/src/pixel/core/grid.ts

@@ -384,8 +384,8 @@ export const detectAxis = (
   return { cell, offset: offset >= cell ? 0 : offset, confidence: Math.min(1, Math.max(0, confidence)) }
 }
 
-/** Finds the grid of an upscaled pixel-art picture; undefined when it has no edges at all. */
-export const detectGrid = (image: RgbaImage): Detection | undefined => {
+/** `detectGrid` on the whole picture. */
+const detectWhole = (image: RgbaImage): Detection | undefined => {
   const { columns, rows } = edgeProfiles(image)
   let x = detectAxis(image, "x", columns)
   let y = detectAxis(image, "y", rows)
@@ -410,6 +410,99 @@ export const detectGrid = (image: RgbaImage): Detection | undefined => {
   }
 }
 
+/** A border color is the background when this share of the border is within `BORDER_TOLERANCE` of it. */
+const BORDER_SHARE = 0.9
+/** How far (largest channel step) a pixel may be from the border color and still be background. */
+const BORDER_TOLERANCE = 24
+/** Pixels kept around the subject, so its outline (its strongest edges) is inside the box, where the edge profiles see it. */
+const CROP_MARGIN = 4
+
+/**
+ * The box holding the picture's subject: its visible pixels (alpha 128 or more), or, in an
+ * opaque picture whose border is one color, the pixels away from that color. Undefined
+ * when the subject fills the picture.
+ */
+const contentBox = (image: RgbaImage): readonly [number, number, number, number] | undefined => {
+  const { width, height, data } = image
+  let transparent = false
+  for (let i = 3; i < data.length; i += 4) {
+    if (data[i]! < 128) {
+      transparent = true
+      break
+    }
+  }
+  let key: readonly [number, number, number] | undefined
+  if (!transparent) {
+    const border: Array<number> = []
+    for (let x = 0; x < width; x++) border.push(x * 4, ((height - 1) * width + x) * 4)
+    for (let y = 1; y < height - 1; y++) border.push(y * width * 4, (y * width + width - 1) * 4)
+    const median = (channel: number): number => {
+      const values = border.map((i) => data[i + channel]!).sort((a, b) => a - b)
+      return values[Math.floor(values.length / 2)]!
+    }
+    const candidate = [median(0), median(1), median(2)] as const
+    const near = border.filter((i) => Math.max(Math.abs(data[i]! - candidate[0]), Math.abs(data[i + 1]! - candidate[1]), Math.abs(data[i + 2]! - candidate[2])) <= BORDER_TOLERANCE).length
+    if (near < BORDER_SHARE * border.length) return undefined
+    key = candidate
+  }
+  let left = width
+  let top = height
+  let right = -1
+  let bottom = -1
+  for (let y = 0; y < height; y++) {
+    for (let x = 0; x < width; x++) {
+      const i = (y * width + x) * 4
+      const content =
+        key === undefined
+          ? data[i + 3]! >= 128
+          : Math.max(Math.abs(data[i]! - key[0]), Math.abs(data[i + 1]! - key[1]), Math.abs(data[i + 2]! - key[2])) > BORDER_TOLERANCE
+      if (!content) continue
+      if (x < left) left = x
+      if (x > right) right = x
+      if (y < top) top = y
+      if (y > bottom) bottom = y
+    }
+  }
+  if (right < 0) return undefined
+  const box = [Math.max(0, left - CROP_MARGIN), Math.max(0, top - CROP_MARGIN), Math.min(width, right + 1 + CROP_MARGIN), Math.min(height, bottom + 1 + CROP_MARGIN)] as const
+  if (box[0] === 0 && box[1] === 0 && box[2] === width && box[3] === height) return undefined
+  return box
+}
+
+/** The part of a picture inside a box. */
+const cropImage = (image: RgbaImage, [left, top, right, bottom]: readonly [number, number, number, number]): RgbaImage => {
+  const width = right - left
+  const height = bottom - top
+  const data = new Uint8ClampedArray(width * height * 4)
+  for (let y = 0; y < height; y++) {
+    data.set(image.data.subarray(((y + top) * image.width + left) * 4, ((y + top) * image.width + right) * 4), y * width * 4)
+  }
+  return { width, height, data }
+}
+
+/**
+ * Finds the grid of an upscaled pixel-art picture; undefined when it has no edges at all.
+ * It is read on the subject only, then placed back on the whole picture.
+ */
+export const detectGrid = (image: RgbaImage): Detection | undefined => {
+  const box = contentBox(image)
+  if (box === undefined) return detectWhole(image)
+  const found = detectWhole(cropImage(image, box))
+  if (found === undefined) return undefined
+  const shift = (offset: number, by: number, cell: number): number => {
+    const moved = Math.round(((((offset + by) % cell) + cell) % cell) * 100) / 100
+    return moved >= cell ? 0 : moved
+  }
+  const offsetX = shift(found.spec.offsetX, box[0], found.spec.cellWidth)
+  const offsetY = shift(found.spec.offsetY, box[1], found.spec.cellHeight)
+  return {
+    ...found,
+    spec: { ...found.spec, offsetX, offsetY },
+    x: { ...found.x, offset: offsetX },
+    y: { ...found.y, offset: offsetY }
+  }
+}
+
 /**
  * Outline pixels, for keeping grid lines off dark strokes: 1 where a pixel is
  * one, row by row (`outline.ts`'s `outlineMask` makes it).

packages/tools/src/pixel/core/report.ts

@@ -115,15 +115,55 @@ const round = (value: number, places = 3): number => Math.round(value * 10 ** pl
 /** Pixel art at its own size has few colors; a blurry or upscaled-and-smoothed picture has many. */
 const FEW_COLORS = 256
 
+/** Below this change rate (see `changeRate`), a picture with more than `SMOOTH_COLORS` colors is an upscale, never pixel art at its own size. */
+const SMOOTH_LIMIT = 0.05
+/**
+ * Flat pixel art (a parallax layer, a tile of a few colors) also changes color rarely;
+ * an upscale carries the generator's or the resize's in-between colors.
+ */
+const SMOOTH_COLORS = 32
+
+/**
+ * The share of neighbouring visible pixels (across and down) whose colors differ by more than a
+ * small step: pixel art at its own size changes color often (a median of about a third), an
+ * upscale only on its cell lines (a few hundredths).
+ */
+const changeRate = (image: RgbaImage): number => {
+  const { width, height, data } = image
+  let changed = 0
+  let pairs = 0
+  const step = (i: number, j: number): boolean =>
+    Math.max(Math.abs(data[i]! - data[j]!), Math.abs(data[i + 1]! - data[j + 1]!), Math.abs(data[i + 2]! - data[j + 2]!)) > 24
+  for (let y = 0; y < height; y++) {
+    for (let x = 0; x < width; x++) {
+      const i = (y * width + x) * 4
+      if (data[i + 3]! < 128) continue
+      if (x + 1 < width && data[i + 7]! >= 128) {
+        pairs++
+        if (step(i, i + 4)) changed++
+      }
+      if (y + 1 < height && data[i + width * 4 + 3]! >= 128) {
+        pairs++
+        if (step(i, i + width * 4)) changed++
+      }
+    }
+  }
+  return pairs === 0 ? 1 : changed / pairs
+}
+
+
 /**
  * True when a picture is already pixel art at its own size: no grid of fake
  * pixels found in it, cells under 1.5 pixels, a grid found with little
  * confidence, or an unsure one in a picture with few colors (a small sprite's
- * own shapes can look like a grid). `outcome` is the fix with the grid found.
+ * own shapes can look like a grid). Never for a smooth picture with many
+ * colors (an upscale whose grid was not found): that one is judged as a fix,
+ * so an unsure grid is reported as one. `outcome` is the fix with the grid found.
  * The CLI's `check` and the page both decide with it, so one picture opens
  * the same way in both.
  */
 export const isNative = (image: RgbaImage, outcome: Outcome | undefined): boolean => {
+  if (changeRate(image) < SMOOTH_LIMIT && artOfImage(image).art.palette.length > SMOOTH_COLORS) return false
   if (outcome === undefined) return true
   const { spec, detection } = outcome
   if (Math.min(spec.cellWidth, spec.cellHeight) < 1.5) return true

packages/tools/test/agent.test.ts

@@ -244,6 +244,24 @@ describe("check", () => {
     expect(checkImage(image, defaultRecipe).report.output).toMatchObject({ width: 24, height: 24 })
   })
 
+  it("asks for a person, never calling it pixel art at its own size, when an upscale's grid is not found", () => {
+    // Drifting cells hide the grid; the picture is smooth and has many colors, so it is no 1x art.
+    const image = upscale(makeSprite(24, 24, sweetie.colors, 7), { scale: 26, offset: 0, drift: 0.6, blur: 1, noise: 4, seed: 2 })
+    const { report } = checkImage(image, defaultRecipe)
+    expect(report.kind).toBe("upscaled")
+    expect(report.verdict).toBe("needs-person")
+  })
+
+  it("takes flat pixel art of a few colors as pixel art at its own size", () => {
+    // A parallax layer changes color as rarely as an upscale, but has only its few colors.
+    const width = 120
+    const height = 80
+    const indices = new Int32Array(width * height).map((_, i) => (Math.floor(i / width) < 50 + Math.round(8 * Math.sin((i % width) / 9)) ? 0 : (i % width) % 40 < 30 ? 1 : 2))
+    const { report } = checkImage(toRgba({ width, height, palette: sweetie.colors.slice(0, 3), indices }), defaultRecipe)
+    expect(report.kind).toBe("pixel-art")
+    expect(report.output).toMatchObject({ width, height })
+  })
+
   it("asks for a person when the grid given does not fit", () => {
     const sprite = makeSprite(20, 20, sweetie.colors, 7)
     const image = upscale(sprite, { scale: 10, offset: 0, blur: 1, noise: 4, seed: 3 })

packages/tools/test/core.test.ts

@@ -6,7 +6,7 @@ import { toJson, toRects, toRgba, toSvg } from "@typeonce/ritmo-tools/pixel/core
 import { cellColors, defaultSampleOptions, quantize, sampleCells, TRANSPARENT, type PixelArt } from "@typeonce/ritmo-tools/pixel/core/fix.ts"
 import { detectGrid, edgeProfiles, refineCuts, refineGrid, uniformCuts, uniformGrid } from "@typeonce/ritmo-tools/pixel/core/grid.ts"
 import { extractPalette, parsePalette, presets, toGpl, toHexFile } from "@typeonce/ritmo-tools/pixel/core/palette.ts"
-import { boundaries, makeSprite, pad, upscale, type Upscale } from "./synthetic.ts"
+import { boundaries, makeSprite, onCanvas, pad, upscale, type Upscale } from "./synthetic.ts"
 
 const sweetie = presets.find((palette) => palette.name === "Sweetie 16")!.colors
 
@@ -108,6 +108,21 @@ describe("grid", () => {
     expect(mismatches(sprite, art)).toBe(0)
   })
 
+  it("finds the grid of a small sprite on a much larger clear canvas", () => {
+    // The empty canvas made every cell size look clean, and the grid came out 12 times too large.
+    const sprite = pad(makeSprite(16, 12, sweetie, 21), 40)
+    const { detection, art } = fix(sprite, { scale: 9.5, offset: 0, blur: 1, noise: 3 })
+    expect(detection.spec.cellWidth).toBeCloseTo(9.5, 1)
+    expect(mismatches(sprite, art)).toBe(0)
+  })
+
+  it("finds the grid of a small sprite on a much larger off-white canvas", () => {
+    const sprite = onCanvas(makeSprite(16, 12, sweetie, 21), 30, { r: 244, g: 240, b: 236 })
+    const { detection, art } = fix(sprite, { scale: 12.3, offset: 0, blur: 1, noise: 3 })
+    expect(detection.spec.cellWidth).toBeCloseTo(12.3, 1)
+    expect(mismatches(sprite, art)).toBe(0)
+  })
+
   it("finds a large grid in a large picture quickly", () => {
     const sprite = makeSprite(48, 48, sweetie, 8)
     const started = performance.now()

packages/tools/test/synthetic.ts

@@ -55,6 +55,13 @@ export const pad = (sprite: PixelArt, margin: number): PixelArt => {
   return { ...sprite, width, height, indices }
 }
 
+/** The sprite in the middle of an opaque canvas of one color, `margin` pixels wider on every side: an image generator's off-white page. */
+export const onCanvas = (sprite: PixelArt, margin: number, color: Rgb): PixelArt => {
+  const padded = pad(sprite, margin)
+  const palette = [...padded.palette, color]
+  return { ...padded, palette, indices: padded.indices.map((index) => (index === TRANSPARENT ? palette.length - 1 : index)) }
+}
+
 /** A sprite moved by whole pixels inside a transparent picture of the given size. */
 export const place = (sprite: PixelArt, width: number, height: number, x: number, y: number): PixelArt => {
   const indices = new Int32Array(width * height).fill(TRANSPARENT)