diff --git a/.changeset/pixel-grid-subject.md b/.changeset/pixel-grid-subject.md new file mode 100644 index 00000000..c5265f5b --- /dev/null +++ b/.changeset/pixel-grid-subject.md @@ -0,0 +1,8 @@ +--- +"@typeonce/ritmo-tools": minor +--- + +`ritmo-tools pixel`: find the grid of a small sprite on a large canvas, and never call an upscale pixel art at its own size. Measured with the benchmark of [xprite-pixelate](https://github.com/rhinoc/xprite-pixelate) (MIT) and [pixel-bench](https://github.com/Retro-Diffusion/pixel-bench) (MIT). + +- **The grid is read on the subject.** `pixel check`, `pixel fix`, and the page find the cell size on the picture's visible pixels, or, on an opaque picture whose border is one color (an image generator's off-white page), on the pixels away from that color, then place the grid on the whole picture. A sprite filling a fifth of its canvas came out with cells several times too large; on the AI benchmark, exact sizes rise from 65% to 76%, and from 24% to 99% for a small sprite on a clear canvas. +- **An upscale is not pixel art at its own size.** A picture whose neighbouring pixels rarely change color and that has more than 32 colors is judged as a fix, so a grid that cannot be found reports `needs-person` ("Unsure of the grid") where it reported `clean` or `fixable` at the picture's full size. Flat pixel art of a few colors (a parallax layer) is still pixel art at its own size. diff --git a/packages/tools/README.md b/packages/tools/README.md index ae3fa2dc..f5c1d269 100644 --- a/packages/tools/README.md +++ b/packages/tools/README.md @@ -115,7 +115,9 @@ node packages/tools/bin/ritmo-tools.mjs pixel check art.png # from its source, ### How to use it -1. **Grid.** The grid is detected when a picture opens. The note under +1. **Grid.** The grid is detected when a picture opens, from its subject + only: a small sprite on a large clear or one-color canvas finds its own + cells. The note under **Auto-detect** gives the cell size, the output size, and a confidence. Below about 50%, check the pink grid lines against the picture. Fix it by typing the cell size and offset (fractions are fine: AI pictures are often scaled diff --git a/packages/tools/src/pixel/core/grid.ts b/packages/tools/src/pixel/core/grid.ts index f2bed916..b0d73fa7 100644 --- a/packages/tools/src/pixel/core/grid.ts +++ b/packages/tools/src/pixel/core/grid.ts @@ -384,8 +384,8 @@ export const detectAxis = ( return { cell, offset: offset >= cell ? 0 : offset, confidence: Math.min(1, Math.max(0, confidence)) } } -/** Finds the grid of an upscaled pixel-art picture; undefined when it has no edges at all. */ -export const detectGrid = (image: RgbaImage): Detection | undefined => { +/** `detectGrid` on the whole picture. */ +const detectWhole = (image: RgbaImage): Detection | undefined => { const { columns, rows } = edgeProfiles(image) let x = detectAxis(image, "x", columns) let y = detectAxis(image, "y", rows) @@ -410,6 +410,99 @@ export const detectGrid = (image: RgbaImage): Detection | undefined => { } } +/** A border color is the background when this share of the border is within `BORDER_TOLERANCE` of it. */ +const BORDER_SHARE = 0.9 +/** How far (largest channel step) a pixel may be from the border color and still be background. */ +const BORDER_TOLERANCE = 24 +/** Pixels kept around the subject, so its outline (its strongest edges) is inside the box, where the edge profiles see it. */ +const CROP_MARGIN = 4 + +/** + * The box holding the picture's subject: its visible pixels (alpha 128 or more), or, in an + * opaque picture whose border is one color, the pixels away from that color. Undefined + * when the subject fills the picture. + */ +const contentBox = (image: RgbaImage): readonly [number, number, number, number] | undefined => { + const { width, height, data } = image + let transparent = false + for (let i = 3; i < data.length; i += 4) { + if (data[i]! < 128) { + transparent = true + break + } + } + let key: readonly [number, number, number] | undefined + if (!transparent) { + const border: Array = [] + for (let x = 0; x < width; x++) border.push(x * 4, ((height - 1) * width + x) * 4) + for (let y = 1; y < height - 1; y++) border.push(y * width * 4, (y * width + width - 1) * 4) + const median = (channel: number): number => { + const values = border.map((i) => data[i + channel]!).sort((a, b) => a - b) + return values[Math.floor(values.length / 2)]! + } + const candidate = [median(0), median(1), median(2)] as const + const near = border.filter((i) => Math.max(Math.abs(data[i]! - candidate[0]), Math.abs(data[i + 1]! - candidate[1]), Math.abs(data[i + 2]! - candidate[2])) <= BORDER_TOLERANCE).length + if (near < BORDER_SHARE * border.length) return undefined + key = candidate + } + let left = width + let top = height + let right = -1 + let bottom = -1 + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const i = (y * width + x) * 4 + const content = + key === undefined + ? data[i + 3]! >= 128 + : Math.max(Math.abs(data[i]! - key[0]), Math.abs(data[i + 1]! - key[1]), Math.abs(data[i + 2]! - key[2])) > BORDER_TOLERANCE + if (!content) continue + if (x < left) left = x + if (x > right) right = x + if (y < top) top = y + if (y > bottom) bottom = y + } + } + if (right < 0) return undefined + const box = [Math.max(0, left - CROP_MARGIN), Math.max(0, top - CROP_MARGIN), Math.min(width, right + 1 + CROP_MARGIN), Math.min(height, bottom + 1 + CROP_MARGIN)] as const + if (box[0] === 0 && box[1] === 0 && box[2] === width && box[3] === height) return undefined + return box +} + +/** The part of a picture inside a box. */ +const cropImage = (image: RgbaImage, [left, top, right, bottom]: readonly [number, number, number, number]): RgbaImage => { + const width = right - left + const height = bottom - top + const data = new Uint8ClampedArray(width * height * 4) + for (let y = 0; y < height; y++) { + data.set(image.data.subarray(((y + top) * image.width + left) * 4, ((y + top) * image.width + right) * 4), y * width * 4) + } + return { width, height, data } +} + +/** + * Finds the grid of an upscaled pixel-art picture; undefined when it has no edges at all. + * It is read on the subject only, then placed back on the whole picture. + */ +export const detectGrid = (image: RgbaImage): Detection | undefined => { + const box = contentBox(image) + if (box === undefined) return detectWhole(image) + const found = detectWhole(cropImage(image, box)) + if (found === undefined) return undefined + const shift = (offset: number, by: number, cell: number): number => { + const moved = Math.round(((((offset + by) % cell) + cell) % cell) * 100) / 100 + return moved >= cell ? 0 : moved + } + const offsetX = shift(found.spec.offsetX, box[0], found.spec.cellWidth) + const offsetY = shift(found.spec.offsetY, box[1], found.spec.cellHeight) + return { + ...found, + spec: { ...found.spec, offsetX, offsetY }, + x: { ...found.x, offset: offsetX }, + y: { ...found.y, offset: offsetY } + } +} + /** * Outline pixels, for keeping grid lines off dark strokes: 1 where a pixel is * one, row by row (`outline.ts`'s `outlineMask` makes it). diff --git a/packages/tools/src/pixel/core/report.ts b/packages/tools/src/pixel/core/report.ts index 3f2f653d..f079f2e8 100644 --- a/packages/tools/src/pixel/core/report.ts +++ b/packages/tools/src/pixel/core/report.ts @@ -115,15 +115,55 @@ const round = (value: number, places = 3): number => Math.round(value * 10 ** pl /** Pixel art at its own size has few colors; a blurry or upscaled-and-smoothed picture has many. */ const FEW_COLORS = 256 +/** Below this change rate (see `changeRate`), a picture with more than `SMOOTH_COLORS` colors is an upscale, never pixel art at its own size. */ +const SMOOTH_LIMIT = 0.05 +/** + * Flat pixel art (a parallax layer, a tile of a few colors) also changes color rarely; + * an upscale carries the generator's or the resize's in-between colors. + */ +const SMOOTH_COLORS = 32 + +/** + * The share of neighbouring visible pixels (across and down) whose colors differ by more than a + * small step: pixel art at its own size changes color often (a median of about a third), an + * upscale only on its cell lines (a few hundredths). + */ +const changeRate = (image: RgbaImage): number => { + const { width, height, data } = image + let changed = 0 + let pairs = 0 + const step = (i: number, j: number): boolean => + Math.max(Math.abs(data[i]! - data[j]!), Math.abs(data[i + 1]! - data[j + 1]!), Math.abs(data[i + 2]! - data[j + 2]!)) > 24 + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const i = (y * width + x) * 4 + if (data[i + 3]! < 128) continue + if (x + 1 < width && data[i + 7]! >= 128) { + pairs++ + if (step(i, i + 4)) changed++ + } + if (y + 1 < height && data[i + width * 4 + 3]! >= 128) { + pairs++ + if (step(i, i + width * 4)) changed++ + } + } + } + return pairs === 0 ? 1 : changed / pairs +} + + /** * True when a picture is already pixel art at its own size: no grid of fake * pixels found in it, cells under 1.5 pixels, a grid found with little * confidence, or an unsure one in a picture with few colors (a small sprite's - * own shapes can look like a grid). `outcome` is the fix with the grid found. + * own shapes can look like a grid). Never for a smooth picture with many + * colors (an upscale whose grid was not found): that one is judged as a fix, + * so an unsure grid is reported as one. `outcome` is the fix with the grid found. * The CLI's `check` and the page both decide with it, so one picture opens * the same way in both. */ export const isNative = (image: RgbaImage, outcome: Outcome | undefined): boolean => { + if (changeRate(image) < SMOOTH_LIMIT && artOfImage(image).art.palette.length > SMOOTH_COLORS) return false if (outcome === undefined) return true const { spec, detection } = outcome if (Math.min(spec.cellWidth, spec.cellHeight) < 1.5) return true diff --git a/packages/tools/test/agent.test.ts b/packages/tools/test/agent.test.ts index 38981550..f8949f47 100644 --- a/packages/tools/test/agent.test.ts +++ b/packages/tools/test/agent.test.ts @@ -244,6 +244,24 @@ describe("check", () => { expect(checkImage(image, defaultRecipe).report.output).toMatchObject({ width: 24, height: 24 }) }) + it("asks for a person, never calling it pixel art at its own size, when an upscale's grid is not found", () => { + // Drifting cells hide the grid; the picture is smooth and has many colors, so it is no 1x art. + const image = upscale(makeSprite(24, 24, sweetie.colors, 7), { scale: 26, offset: 0, drift: 0.6, blur: 1, noise: 4, seed: 2 }) + const { report } = checkImage(image, defaultRecipe) + expect(report.kind).toBe("upscaled") + expect(report.verdict).toBe("needs-person") + }) + + it("takes flat pixel art of a few colors as pixel art at its own size", () => { + // A parallax layer changes color as rarely as an upscale, but has only its few colors. + const width = 120 + const height = 80 + const indices = new Int32Array(width * height).map((_, i) => (Math.floor(i / width) < 50 + Math.round(8 * Math.sin((i % width) / 9)) ? 0 : (i % width) % 40 < 30 ? 1 : 2)) + const { report } = checkImage(toRgba({ width, height, palette: sweetie.colors.slice(0, 3), indices }), defaultRecipe) + expect(report.kind).toBe("pixel-art") + expect(report.output).toMatchObject({ width, height }) + }) + it("asks for a person when the grid given does not fit", () => { const sprite = makeSprite(20, 20, sweetie.colors, 7) const image = upscale(sprite, { scale: 10, offset: 0, blur: 1, noise: 4, seed: 3 }) diff --git a/packages/tools/test/core.test.ts b/packages/tools/test/core.test.ts index 3136cbe3..483781b7 100644 --- a/packages/tools/test/core.test.ts +++ b/packages/tools/test/core.test.ts @@ -6,7 +6,7 @@ import { toJson, toRects, toRgba, toSvg } from "@typeonce/ritmo-tools/pixel/core import { cellColors, defaultSampleOptions, quantize, sampleCells, TRANSPARENT, type PixelArt } from "@typeonce/ritmo-tools/pixel/core/fix.ts" import { detectGrid, edgeProfiles, refineCuts, refineGrid, uniformCuts, uniformGrid } from "@typeonce/ritmo-tools/pixel/core/grid.ts" import { extractPalette, parsePalette, presets, toGpl, toHexFile } from "@typeonce/ritmo-tools/pixel/core/palette.ts" -import { boundaries, makeSprite, pad, upscale, type Upscale } from "./synthetic.ts" +import { boundaries, makeSprite, onCanvas, pad, upscale, type Upscale } from "./synthetic.ts" const sweetie = presets.find((palette) => palette.name === "Sweetie 16")!.colors @@ -108,6 +108,21 @@ describe("grid", () => { expect(mismatches(sprite, art)).toBe(0) }) + it("finds the grid of a small sprite on a much larger clear canvas", () => { + // The empty canvas made every cell size look clean, and the grid came out 12 times too large. + const sprite = pad(makeSprite(16, 12, sweetie, 21), 40) + const { detection, art } = fix(sprite, { scale: 9.5, offset: 0, blur: 1, noise: 3 }) + expect(detection.spec.cellWidth).toBeCloseTo(9.5, 1) + expect(mismatches(sprite, art)).toBe(0) + }) + + it("finds the grid of a small sprite on a much larger off-white canvas", () => { + const sprite = onCanvas(makeSprite(16, 12, sweetie, 21), 30, { r: 244, g: 240, b: 236 }) + const { detection, art } = fix(sprite, { scale: 12.3, offset: 0, blur: 1, noise: 3 }) + expect(detection.spec.cellWidth).toBeCloseTo(12.3, 1) + expect(mismatches(sprite, art)).toBe(0) + }) + it("finds a large grid in a large picture quickly", () => { const sprite = makeSprite(48, 48, sweetie, 8) const started = performance.now() diff --git a/packages/tools/test/synthetic.ts b/packages/tools/test/synthetic.ts index 48559056..4e615ee2 100644 --- a/packages/tools/test/synthetic.ts +++ b/packages/tools/test/synthetic.ts @@ -55,6 +55,13 @@ export const pad = (sprite: PixelArt, margin: number): PixelArt => { return { ...sprite, width, height, indices } } +/** The sprite in the middle of an opaque canvas of one color, `margin` pixels wider on every side: an image generator's off-white page. */ +export const onCanvas = (sprite: PixelArt, margin: number, color: Rgb): PixelArt => { + const padded = pad(sprite, margin) + const palette = [...padded.palette, color] + return { ...padded, palette, indices: padded.indices.map((index) => (index === TRANSPARENT ? palette.length - 1 : index)) } +} + /** A sprite moved by whole pixels inside a transparent picture of the given size. */ export const place = (sprite: PixelArt, width: number, height: number, x: number, y: number): PixelArt => { const indices = new Int32Array(width * height).fill(TRANSPARENT)