From ee004a553495a9df4106fcde2ebeefa96e97028e Mon Sep 17 00:00:00 2001 From: oliverne Date: Sat, 17 Oct 2026 08:41:20 +0900 Subject: [PATCH 1/2] =?UTF-8?q?Keep=20Unicode=20letters=20and=20numbers=20?= =?UTF-8?q?in=20slugs,=20so=20a=20Korean=20heading=20names=20its=20file=20?= =?UTF-8?q?=ED=95=9C=EA=B5=AD=EC=96=B4-=EC=A0=9C=EB=AA=A9.md=20instead=20o?= =?UTF-8?q?f=20falling=20back=20to=20untitled?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit slugify strips everything outside \p{L}\p{N} rather than [a-z0-9], so a heading in any non-Latin script survives as a filename instead of collapsing into separators. Hangul that arrives decomposed (NFD, as macOS input methods produce) is recomposed to NFC after accent folding, so the file keeps composed syllables and resolves to one name on every platform. Truncation counts code points instead of UTF-16 units, so a long CJK heading is not cut mid-letter and stays inside the 255-byte filename limit; MAX_SLUG is now documented as 60 code points, which is at most 240 UTF-8 bytes. The note and workspace names come from the same shared slug.ts that the iOS and Android clients load, so all three platforms name files the same way. docs/contributor/architecture.md (the trust boundary and the filesystem invariants) and remote.md (why .server is dotted) stated "[a-z0-9-]" as the slug invariant; both now say Unicode letters, numbers and hyphens. --- docs/contributor/architecture.md | 12 +++++++----- docs/contributor/remote.md | 6 +++--- src/bun/notes.fs.test.ts | 18 ++++++++++++++++++ src/bun/notes.ts | 5 ++--- src/bun/workspaces.fs.test.ts | 4 ++++ src/bun/workspaces.ts | 11 +++++------ src/shared/slug.test.ts | 24 +++++++++++++++++++++++- src/shared/slug.ts | 27 ++++++++++++++------------- 8 files changed, 76 insertions(+), 31 deletions(-) diff --git a/docs/contributor/architecture.md b/docs/contributor/architecture.md index c49ce1a..00742c8 100644 --- a/docs/contributor/architecture.md +++ b/docs/contributor/architecture.md @@ -220,8 +220,9 @@ Bun therefore validates everything and derives anything derivable: requirement is load-bearing, not tidiness: an in-root write of any other file would be arbitrary-file storage in a folder the user syncs. - **The view never names a file.** It sends a note's *text*; Bun slugs the - heading itself (`slugOf` emits only `[a-z0-9-]`), so filenames are safe by - construction and there is no name parameter to validate or smuggle through. + heading itself (`slugOf` emits only Unicode letters, numbers and hyphens). + Filenames are safe by construction: no dots, path separators or control + characters, and no name parameter to validate or smuggle through. Workspace folder names get the same treatment (`createManaged` slugs the display name; renaming a workspace is display-only and touches no folder). - **Paths are opaque handles.** The view holds only paths and roots it was @@ -379,9 +380,10 @@ Bun therefore validates everything and derives anything derivable: purge, empty, or Trash listing can see them. - **A folder is the only name a caller chooses, so it is the only name validated.** Filenames never needed a guard: `slugOf` builds them from the - note's own heading and emits only `[a-z0-9-]`, so there is no name to check - and no way for the view to ask for a path. A folder has to match what is - already on disk, so it cannot be slugged into safety, and it arrives from + note's own heading and emits only Unicode letters, numbers and hyphens. + There is no name to check and no way for the view to ask for a path. + A folder has to match what is already on disk, so it cannot be slugged into + safety, and it arrives from the view, from MCP and from the CLI alike. `folderPathOf` is the one gate: relative only, no backslashes, no `.` or `..` segment, no dot-entry, inside the root once resolved. `..` is rejected on the raw segments rather than diff --git a/docs/contributor/remote.md b/docs/contributor/remote.md index 69e3a5c..f07e6e4 100644 --- a/docs/contributor/remote.md +++ b/docs/contributor/remote.md @@ -568,9 +568,9 @@ installed on sshd's own PATH, such as `/usr/local/bin` on Linux. **`.server` is inside the app home**, beside `.client` (§5), because both are this machine's role-specific state and one `~/.ledge` is the whole of Ledge on a disk. It is dotted because managed workspace folders are created directly in -the app home under `[a-z0-9-]` slugs, so `server/` or `bin/` could be a -workspace. `backup.ts` excludes it, and the root watcher and note listing skip -dotted entries already. The path is spelled literally in `SERVE_COMMAND` and +the app home under slugs of Unicode letters, numbers and hyphens, so `server/` +or `bin/` could be a workspace. `backup.ts` excludes it, and the root watcher +and note listing skip dotted entries already. The path is spelled literally in `SERVE_COMMAND` and in the forced command on every paired server, so `LEDGE_NOTES_ROOT` moves everything in the app home except this: a run with it set still finds its server under `$HOME/.ledge/.server`, which is why the install probe moves diff --git a/src/bun/notes.fs.test.ts b/src/bun/notes.fs.test.ts index b013182..eb2e5dc 100644 --- a/src/bun/notes.fs.test.ts +++ b/src/bun/notes.fs.test.ts @@ -80,6 +80,16 @@ beforeEach(async () => { }); describe("createNote / writeNote / readNote", () => { + test("Korean headings name files and duplicate headings enumerate without clobbering", async () => { + const a = await createNote(ROOT, "# 한국어 제목\n\n첫 번째"); + const b = await createNote(ROOT, "# 한국어 제목\n\n두 번째"); + expect(a.path).toBe(join(ROOT, "한국어-제목.md")); + expect(b.path).toBe(join(ROOT, "한국어-제목-2.md")); + expect(a.title).toBe("한국어 제목"); + expect(await textAt(a.path)).toBe("# 한국어 제목\n\n첫 번째"); + expect(await textAt(b.path)).toBe("# 한국어 제목\n\n두 번째"); + }); + test("names the file from the note's H1 and round-trips its text", async () => { const note = await createNote(ROOT, "# Shipping Notes\n\nhello"); expect(note.path).toBe(join(ROOT, "shipping-notes.md")); @@ -476,6 +486,14 @@ describe("notesTagged", () => { }); describe("retitleNote", () => { + test("an untitled note can be renamed to a Korean heading", async () => { + const note = await createNote(ROOT, "본문"); + const moved = await retitleNote(note.path, "# 한국어 제목\n\n본문"); + expect(moved.path).toBe(join(ROOT, "한국어-제목.md")); + expect(await textAt(moved.path)).toBe("본문"); + expect(await textAt(note.path)).toBeNull(); + }); + test("moves the file to match its heading", async () => { const note = await createNote(ROOT, "# Draft\n"); const moved = await retitleNote(note.path, "# Final\n"); diff --git a/src/bun/notes.ts b/src/bun/notes.ts index c684168..b9e6283 100644 --- a/src/bun/notes.ts +++ b/src/bun/notes.ts @@ -74,9 +74,8 @@ export { titleOf }; // The filename a note's text asks for: its first-line H1 as a slug, or // "untitled" when the note has no usable heading. Bun slugs the heading itself -// rather than taking a name from the view. slugify emits only [a-z0-9-], so -// there is no name to validate and no way for the view to ask for a path -// (architecture.md §2). +// rather than taking a name from the view. slugify emits only Unicode letters, +// numbers and hyphens, so the view cannot ask for a path (architecture.md §2). function baseFor(text: string): string { return slugOf(text) ?? "untitled"; } diff --git a/src/bun/workspaces.fs.test.ts b/src/bun/workspaces.fs.test.ts index 49de305..77d7098 100644 --- a/src/bun/workspaces.fs.test.ts +++ b/src/bun/workspaces.fs.test.ts @@ -98,6 +98,10 @@ describe("createManaged", () => { expect(rules).toEqual([".ledge-trash/", ".*.md.tmp-*", ".DS_Store"]); }); + test("a Korean display name creates a Korean workspace folder", async () => { + expect(await createManaged("한국어 노트")).toBe(join(resolve(APP_HOME), "한국어-노트")); + }); + test("a name that slugs to nothing falls back rather than failing", async () => { const root = await createManaged("???"); expect(root).toBe(join(resolve(APP_HOME), "workspace")); diff --git a/src/bun/workspaces.ts b/src/bun/workspaces.ts index 0db0432..f38cada 100644 --- a/src/bun/workspaces.ts +++ b/src/bun/workspaces.ts @@ -32,7 +32,7 @@ export const WORKSPACES_PATH = join(APP_HOME, ".workspaces.json"); // .workspaces.json, and every mutating seam in notes.ts and assets.ts refuses // it by calling assertWritableRoot below. The name is dotted and // .ledge-prefixed like every app-owned entry, so it cannot collide with a -// managed workspace slug (slugify emits only [a-z0-9-], never a leading dot). +// managed workspace slug (slugify never emits a leading dot). export const DOCS_ROOT = join(APP_HOME, ".ledge-docs"); // The ignore file a new managed workspace is seeded with, so that `git init` @@ -349,11 +349,10 @@ export async function ensureDefault(): Promise { } // Creates a managed workspace folder from a display name. Bun slugs the name -// itself, the same trust move as noteCreate: slugify emits only [a-z0-9-], so -// the view has no path to smuggle through. uniqueName allocates the folder name -// against a readdir snapshot of APP_HOME, so the mkdir cannot land on something -// that exists. settings.jsonc, dot-entries, and squatting files all count as -// taken. +// itself, the same trust move as noteCreate: slugify emits only Unicode letters, +// numbers and hyphens, so the view has no path to smuggle through. uniqueName +// allocates against a readdir snapshot of APP_HOME, including settings.jsonc, +// dot-entries and squatting files, so mkdir cannot land on something that exists. export async function createManaged(name: string): Promise { await ensureAppHome(); const base = slugify(name) ?? "workspace"; diff --git a/src/shared/slug.test.ts b/src/shared/slug.test.ts index 0e12259..30f978b 100644 --- a/src/shared/slug.test.ts +++ b/src/shared/slug.test.ts @@ -103,7 +103,28 @@ describe("slugify", () => { // The caller falls back to the enumerated "untitled" name. expect(slugify("!!!")).toBeNull(); expect(slugify(" ")).toBeNull(); - expect(slugify("日本語")).toBeNull(); + expect(slugify("🎉")).toBeNull(); + }); + + test("Korean and mixed-language headings keep their letters", () => { + expect(slugify("한국어 제목")).toBe("한국어-제목"); + expect(slugify("Ledge 한국어 노트 2026")).toBe("ledge-한국어-노트-2026"); + expect(slugify("日本語")).toBe("日本語"); + }); + + test("decomposed Hangul produces the same composed filename", () => { + expect(slugify("한국어 제목".normalize("NFD"))).toBe("한국어-제목"); + }); + + test("Unicode headings still remove unsafe path characters", () => { + expect(slugify("../../한국어/제목\\초안:메모\u0000")).toBe("한국어-제목-초안-메모"); + }); + + test("long Unicode headings fit the filename limit without splitting a letter", () => { + expect(slugify("한".repeat(200))).toBe("한".repeat(60)); + const slug = slugify("𠀀".repeat(200))!; + expect(slug).toBe("𠀀".repeat(60)); + expect(new TextEncoder().encode(`${slug}-2.md`).length).toBeLessThan(255); }); test("the slug is safe by construction: no dots, separators, or leading dot", () => { @@ -130,6 +151,7 @@ describe("slugify", () => { describe("slugOf", () => { test("a titled note asks for its heading's slug", () => { expect(slugOf("# Shipping Notes\n\nbody")).toBe("shipping-notes"); + expect(slugOf("# 한국어 제목\n\n본문")).toBe("한국어-제목"); }); test("an untitled note asks for nothing", () => { diff --git a/src/shared/slug.ts b/src/shared/slug.ts index 2183787..74a9c02 100644 --- a/src/shared/slug.ts +++ b/src/shared/slug.ts @@ -10,9 +10,8 @@ // Renaming is safe only while a note's identity is its docId and not its path // (architecture.md §4). -// Long enough for a real heading, short enough to stay clear of the 255-byte -// filename limit every filesystem here shares, with room for a "-2" suffix and -// the extension. +// At most 60 Unicode code points (240 UTF-8 bytes), leaving room under the +// 255-byte filename limit for an enumerated suffix and the extension. const MAX_SLUG = 60; // Far longer than any real heading, short enough that a runaway one cannot @@ -44,25 +43,27 @@ export function headingOf(text: string): string | null { } // Turns a heading into a filesystem-safe slug, or null if nothing usable is -// left. Everything outside [a-z0-9] becomes a separator, which makes the -// result safe by construction: no dots, no slashes, no colons, no control -// characters, no leading dot. This is the whole trust boundary for a name -// arriving from the view (architecture.md §2). +// left. Everything outside Unicode letters and numbers becomes a separator: +// no dots, no slashes, no colons, no control characters, no leading dot. +// This is the whole trust boundary for a name arriving from the view +// (architecture.md §2). export function slugify(heading: string): string | null { // Fold accents first ("Café" -> "cafe"). Otherwise each accented letter // becomes a separator and breaks the word into pieces. - const folded = heading.normalize("NFD").replace(/\p{M}/gu, ""); + // Recompose Hangul after folding so filenames keep syllables, not jamo. + const folded = heading.normalize("NFD").replace(/\p{M}/gu, "").normalize("NFC"); const slug = folded .toLowerCase() - .replace(/[^a-z0-9]+/g, "-") + .replace(/[^\p{L}\p{N}]+/gu, "-") .replace(/^-+|-+$/g, ""); - if (!slug) return null; // a heading of pure punctuation or non-latin script - if (slug.length <= MAX_SLUG) return slug; + if (!slug) return null; // a heading of pure punctuation + const characters = Array.from(slug); + if (characters.length <= MAX_SLUG) return slug; // Truncate on a word boundary when one is near the end, so a long heading is // cut at "shipping-notes" rather than "shipping-note". - const cut = slug.slice(0, MAX_SLUG); + const cut = characters.slice(0, MAX_SLUG); const lastDash = cut.lastIndexOf("-"); - const trimmed = lastDash >= MAX_SLUG / 2 ? cut.slice(0, lastDash) : cut; + const trimmed = (lastDash >= MAX_SLUG / 2 ? cut.slice(0, lastDash) : cut).join(""); return trimmed.replace(/-+$/, ""); } From 5afc51ef2b3b0037a563f9b2d84f85a25e36dfd9 Mon Sep 17 00:00:00 2001 From: Dan Stevens <1061040+danhstevens@users.noreply.github.com> Date: Sun, 18 Oct 2026 15:01:08 -0600 Subject: [PATCH 2/2] Compare filenames ignoring Unicode normalization so a new or renamed note cannot overwrite a decomposed Korean filename on APFS, keep combining marks outside Latin script, and cap slugs at 200 UTF-8 bytes so a save's temp file fits on ext4 --- docs/contributor/architecture.md | 24 +++++++++------- docs/contributor/remote.md | 6 ++-- src/bun/notes.fs.test.ts | 25 ++++++++++++++++ src/bun/notes.ts | 8 +++--- src/bun/workspaces.test.ts | 6 ++++ src/bun/workspaces.ts | 22 ++++++++------ src/shared/slug.test.ts | 29 +++++++++++++++++-- src/shared/slug.ts | 49 ++++++++++++++++++++------------ 8 files changed, 123 insertions(+), 46 deletions(-) diff --git a/docs/contributor/architecture.md b/docs/contributor/architecture.md index 00742c8..8a58d4f 100644 --- a/docs/contributor/architecture.md +++ b/docs/contributor/architecture.md @@ -220,11 +220,12 @@ Bun therefore validates everything and derives anything derivable: requirement is load-bearing, not tidiness: an in-root write of any other file would be arbitrary-file storage in a folder the user syncs. - **The view never names a file.** It sends a note's *text*; Bun slugs the - heading itself (`slugOf` emits only Unicode letters, numbers and hyphens). - Filenames are safe by construction: no dots, path separators or control - characters, and no name parameter to validate or smuggle through. - Workspace folder names get the same treatment (`createManaged` slugs the - display name; renaming a workspace is display-only and touches no folder). + heading itself (`slugOf` emits only Unicode letters, their combining marks, + numbers and hyphens, in NFC). Filenames are safe by construction: no dots, + path separators or control characters, and no name parameter to validate or + smuggle through. Workspace folder names get the same treatment + (`createManaged` slugs the display name; renaming a workspace is + display-only and touches no folder). - **Paths are opaque handles.** The view holds only paths and roots it was handed by Bun (`workspaceList`, `noteList`, `noteCreate`, `noteDelete`'s trashed location) and passes them back unmodified. View code that @@ -380,11 +381,11 @@ Bun therefore validates everything and derives anything derivable: purge, empty, or Trash listing can see them. - **A folder is the only name a caller chooses, so it is the only name validated.** Filenames never needed a guard: `slugOf` builds them from the - note's own heading and emits only Unicode letters, numbers and hyphens. - There is no name to check and no way for the view to ask for a path. - A folder has to match what is already on disk, so it cannot be slugged into - safety, and it arrives from - the view, from MCP and from the CLI alike. `folderPathOf` is the one gate: + note's own heading and emits only Unicode letters, their combining marks, + numbers and hyphens, so there is no name to check and no way for the view + to ask for a path. A folder has to match what is already on disk, so it + cannot be slugged into safety, and it arrives from the view, from MCP and + from the CLI alike. `folderPathOf` is the one gate: relative only, no backslashes, no `.` or `..` segment, no dot-entry, inside the root once resolved. `..` is rejected on the raw segments rather than left to the containment check, so `a/../b` cannot quietly mean `b` — the @@ -461,7 +462,8 @@ Bun therefore validates everything and derives anything derivable: volume) degrades to focus-refresh only, warned, like every other unavailable-root path. `rename(2)` overwrites silently, so no call site may pick its own destination name: `uniqueName` - (case-insensitive, because APFS is) allocates against a `readdir` snapshot + (ignoring case and Unicode normalization, because APFS does) allocates + against a `readdir` snapshot plus the `reserved` set, and the rename that follows is safe *because* the allocation already skipped every taken name. A rename whose target didn't come from `uniqueName` is a latent data-loss bug. diff --git a/docs/contributor/remote.md b/docs/contributor/remote.md index f07e6e4..331ed28 100644 --- a/docs/contributor/remote.md +++ b/docs/contributor/remote.md @@ -568,9 +568,9 @@ installed on sshd's own PATH, such as `/usr/local/bin` on Linux. **`.server` is inside the app home**, beside `.client` (§5), because both are this machine's role-specific state and one `~/.ledge` is the whole of Ledge on a disk. It is dotted because managed workspace folders are created directly in -the app home under slugs of Unicode letters, numbers and hyphens, so `server/` -or `bin/` could be a workspace. `backup.ts` excludes it, and the root watcher -and note listing skip dotted entries already. The path is spelled literally in `SERVE_COMMAND` and +the app home under slugs of their names, so `server/` or `bin/` could be a +workspace. `backup.ts` excludes it, and the root watcher and note listing skip +dotted entries already. The path is spelled literally in `SERVE_COMMAND` and in the forced command on every paired server, so `LEDGE_NOTES_ROOT` moves everything in the app home except this: a run with it set still finds its server under `$HOME/.ledge/.server`, which is why the install probe moves diff --git a/src/bun/notes.fs.test.ts b/src/bun/notes.fs.test.ts index eb2e5dc..8f0346e 100644 --- a/src/bun/notes.fs.test.ts +++ b/src/bun/notes.fs.test.ts @@ -90,6 +90,16 @@ describe("createNote / writeNote / readNote", () => { expect(await textAt(b.path)).toBe("# 한국어 제목\n\n두 번째"); }); + // Finder writes Korean names decomposed (NFD). On APFS that is the same file + // as the composed name slugify emits, so writing there would replace it. + test("a note beside a decomposed same-titled file does not overwrite it", async () => { + const existing = join(ROOT, "회의록.md".normalize("NFD")); + await writeRaw(existing, "# 회의록\n\nolder"); + const made = await createNote(ROOT, "# 회의록\n\nnewer"); + expect(made.path).toBe(join(ROOT, "회의록-2.md")); + expect(await textAt(existing)).toBe("# 회의록\n\nolder"); + }); + test("names the file from the note's H1 and round-trips its text", async () => { const note = await createNote(ROOT, "# Shipping Notes\n\nhello"); expect(note.path).toBe(join(ROOT, "shipping-notes.md")); @@ -494,6 +504,21 @@ describe("retitleNote", () => { expect(await textAt(note.path)).toBeNull(); }); + test("a retitle onto a decomposed file's name does not overwrite it", async () => { + const existing = join(ROOT, "회의록.md".normalize("NFD")); + await writeRaw(existing, "# 회의록\n\nolder"); + const note = await createNote(ROOT, "draft"); + const moved = await retitleNote(note.path, "# 회의록\n\ndraft"); + expect(moved.path).toBe(join(ROOT, "회의록-2.md")); + expect(await textAt(existing)).toBe("# 회의록\n\nolder"); + }); + + test("a decomposed file already named for its heading stays put", async () => { + const existing = join(ROOT, "회의록.md".normalize("NFD")); + await writeRaw(existing, "# 회의록\n"); + expect((await retitleNote(existing, "# 회의록\n")).path).toBe(existing); + }); + test("moves the file to match its heading", async () => { const note = await createNote(ROOT, "# Draft\n"); const moved = await retitleNote(note.path, "# Final\n"); diff --git a/src/bun/notes.ts b/src/bun/notes.ts index b9e6283..2a8b8d4 100644 --- a/src/bun/notes.ts +++ b/src/bun/notes.ts @@ -21,7 +21,7 @@ import { collectHits, type SearchHit } from "../shared/search"; import { resolveWikiTitle, wikiRefsOf } from "../shared/wikilinks"; import { normalizeTag, tagDirectoryOf, tagRefsOf, type TagInfo } from "../shared/tags"; import { loadIgnore } from "./ignore"; -import { assertRegisteredRoot, assertWritableRoot, GITIGNORE, isInside, kindOf, rootContaining, uniqueName } from "./workspaces"; +import { assertRegisteredRoot, assertWritableRoot, GITIGNORE, isInside, kindOf, nameKey, rootContaining, uniqueName } from "./workspaces"; import { beginPassphraseChange, commitPassphraseChange, @@ -74,8 +74,8 @@ export { titleOf }; // The filename a note's text asks for: its first-line H1 as a slug, or // "untitled" when the note has no usable heading. Bun slugs the heading itself -// rather than taking a name from the view. slugify emits only Unicode letters, -// numbers and hyphens, so the view cannot ask for a path (architecture.md §2). +// rather than taking a name from the view. slugify never emits a dot or a path +// separator, so the view cannot ask for a path (architecture.md §2). function baseFor(text: string): string { return slugOf(text) ?? "untitled"; } @@ -1270,7 +1270,7 @@ export async function retitleNote(path: string, text: string): Promise for (const name of reserved) taken.add(name); const name = uniqueName(baseFor(text), taken); - if (name.toLowerCase() === current.toLowerCase()) { + if (nameKey(name) === nameKey(current)) { // Already correctly named, or uniqueName landed back on its own name. return metaFor(path, text); } diff --git a/src/bun/workspaces.test.ts b/src/bun/workspaces.test.ts index 8204057..1b4d693 100644 --- a/src/bun/workspaces.test.ts +++ b/src/bun/workspaces.test.ts @@ -19,6 +19,12 @@ describe("uniqueName", () => { expect(uniqueName("notes", new Set(["Notes.md", "notes-2.MD"]))).toBe("notes-3.md"); }); + // APFS also treats composed and decomposed spellings as one file, and Finder + // writes Korean names decomposed (NFD) while slugify emits NFC. + test("comparison ignores Unicode normalization, so an NFD name counts as taken", () => { + expect(uniqueName("회의록", new Set(["회의록.md".normalize("NFD")]))).toBe("회의록-2.md"); + }); + test("suffixes from 2 upward, skipping every taken name", () => { const taken = new Set(["untitled.md", "untitled-2.md", "untitled-3.md"]); expect(uniqueName("untitled", taken)).toBe("untitled-4.md"); diff --git a/src/bun/workspaces.ts b/src/bun/workspaces.ts index f38cada..d59d6ae 100644 --- a/src/bun/workspaces.ts +++ b/src/bun/workspaces.ts @@ -70,17 +70,23 @@ export function isInside(root: string, p: string): boolean { // Allocate a filename not already in `taken`: shipping-notes.md, // shipping-notes-2.md, and so on (with ext "", a workspace folder name). -// Comparison is case-insensitive because macOS's default APFS is: "Foo.md" and -// "foo.md" are one file there, so a case-sensitive check would return a name -// whose rename overwrites the other note. On Linux it only means an extra -// foo-2.md where foo.md was free. +// Comparison ignores case and Unicode normalization because macOS's default +// APFS does: "Foo.md" and "foo.md" are one file there, and so are a composed +// "회의록.md" and the decomposed one Finder writes. A stricter check would +// return a name whose rename overwrites the other note. On Linux it only means +// an extra foo-2.md where foo.md was free. export function uniqueName(base: string, taken: Set, ext = ".md"): string { - const lower = new Set([...taken].map((t) => t.toLowerCase())); + const keys = new Set([...taken].map(nameKey)); let name = `${base}${ext}`; - for (let n = 2; lower.has(name.toLowerCase()); n += 1) name = `${base}-${n}${ext}`; + for (let n = 2; keys.has(nameKey(name)); n += 1) name = `${base}-${n}${ext}`; return name; } +// The form two names share when APFS treats them as one file (uniqueName). +export function nameKey(name: string): string { + return name.normalize("NFC").toLowerCase(); +} + // The roots a loosely-spelled workspace name could mean: a root path (~ // expands), or the folder name of a registered root as shorthand. The CLI's // --workspace argument (cli.ts resolveWorkspaceArg) and the daily.workspace @@ -349,8 +355,8 @@ export async function ensureDefault(): Promise { } // Creates a managed workspace folder from a display name. Bun slugs the name -// itself, the same trust move as noteCreate: slugify emits only Unicode letters, -// numbers and hyphens, so the view has no path to smuggle through. uniqueName +// itself, the same trust move as noteCreate: slugify never emits a dot or a +// path separator, so the view has no path to smuggle through. uniqueName // allocates against a readdir snapshot of APP_HOME, including settings.jsonc, // dot-entries and squatting files, so mkdir cannot land on something that exists. export async function createManaged(name: string): Promise { diff --git a/src/shared/slug.test.ts b/src/shared/slug.test.ts index 30f978b..caab264 100644 --- a/src/shared/slug.test.ts +++ b/src/shared/slug.test.ts @@ -116,15 +116,40 @@ describe("slugify", () => { expect(slugify("한국어 제목".normalize("NFD"))).toBe("한국어-제목"); }); + // Only Latin letters lose their accents. Elsewhere a mark changes the letter + // or is part of the word. + test("marks outside Latin script are kept, and the result is composed", () => { + expect(slugify("ガイド")).toBe("ガイド"); + expect(slugify("ガイド".normalize("NFD"))).toBe("ガイド".normalize("NFC")); + expect(slugify("हिन्दी नोट्स")).toBe("हिन्दी-नोट्स"); + expect(slugify("Мой план")).toBe("мой-план"); + expect(slugify("Tiếng Việt")).toBe("tieng-viet"); + }); + + test("a mark with no letter before it is dropped with the separator", () => { + expect(slugify("́abc")).toBe("abc"); + expect(slugify("가 ́나")).toBe("가-나"); + }); + test("Unicode headings still remove unsafe path characters", () => { expect(slugify("../../한국어/제목\\초안:메모\u0000")).toBe("한국어-제목-초안-메모"); }); + // The temp file a save writes is the longest name a slug is part of + // (notes.ts writeNote), and ext4 refuses a name over 255 bytes. test("long Unicode headings fit the filename limit without splitting a letter", () => { expect(slugify("한".repeat(200))).toBe("한".repeat(60)); const slug = slugify("𠀀".repeat(200))!; - expect(slug).toBe("𠀀".repeat(60)); - expect(new TextEncoder().encode(`${slug}-2.md`).length).toBeLessThan(255); + expect(slug).toBe("𠀀".repeat(50)); + const temp = `.${slug}-99.md.tmp-4194304-4294967295`; + expect(new TextEncoder().encode(temp).length).toBeLessThanOrEqual(255); + }); + + test("truncation never separates a letter from its marks", () => { + // 60 code points end on ह, whose vowel sign ि is the 61st. + expect(slugify(`क${"हि".repeat(40)}`)).toBe(`क${"हि".repeat(29)}`); + // A letter buried under marks has nothing left once they are kept whole. + expect(slugify(`क${"́".repeat(150)}`)).toBeNull(); }); test("the slug is safe by construction: no dots, separators, or leading dot", () => { diff --git a/src/shared/slug.ts b/src/shared/slug.ts index 74a9c02..87da052 100644 --- a/src/shared/slug.ts +++ b/src/shared/slug.ts @@ -10,9 +10,12 @@ // Renaming is safe only while a note's identity is its docId and not its path // (architecture.md §4). -// At most 60 Unicode code points (240 UTF-8 bytes), leaving room under the -// 255-byte filename limit for an enumerated suffix and the extension. +// A slug is at most 60 code points and at most 200 UTF-8 bytes. The longest +// name built from one is a save's temp file (".-2.md.tmp--", +// notes.ts writeNote), about 30 bytes more, and ext4 refuses a name over 255 +// bytes. Hangul and kanji fit all 60; 4-byte letters stop at 50. const MAX_SLUG = 60; +const MAX_SLUG_BYTES = 200; // Far longer than any real heading, short enough that a runaway one cannot // bloat the store. Only the on-screen label is capped; the note's text is @@ -43,28 +46,38 @@ export function headingOf(text: string): string | null { } // Turns a heading into a filesystem-safe slug, or null if nothing usable is -// left. Everything outside Unicode letters and numbers becomes a separator: -// no dots, no slashes, no colons, no control characters, no leading dot. -// This is the whole trust boundary for a name arriving from the view -// (architecture.md §2). +// left. Everything outside Unicode letters, their combining marks and numbers +// becomes a separator: no dots, no slashes, no colons, no control characters, +// no leading dot. The result is NFC. This is the whole trust boundary for a +// name arriving from the view (architecture.md §2). export function slugify(heading: string): string | null { - // Fold accents first ("Café" -> "cafe"). Otherwise each accented letter - // becomes a separator and breaks the word into pieces. - // Recompose Hangul after folding so filenames keep syllables, not jamo. - const folded = heading.normalize("NFD").replace(/\p{M}/gu, "").normalize("NFC"); - const slug = folded + // Fold accents off Latin letters ("Café" -> "cafe"), so a Latin heading + // names an ASCII file. Other scripts keep their marks: が is not か, and a + // Devanagari vowel sign is part of the word. NFC recomposes what NFD split. + const folded = heading + .normalize("NFD") + .replace(/(\p{Script=Latin})\p{M}+/gu, "$1") .toLowerCase() - .replace(/[^\p{L}\p{N}]+/gu, "-") - .replace(/^-+|-+$/g, ""); + .normalize("NFC"); + // A mark with no letter before it goes with the separator it follows. + const slug = folded.replace(/^\p{M}+|[^\p{L}\p{M}\p{N}]+\p{M}*/gu, "-").replace(/^-+|-+$/g, ""); if (!slug) return null; // a heading of pure punctuation - const characters = Array.from(slug); - if (characters.length <= MAX_SLUG) return slug; + const chars = Array.from(slug); + if (chars.length <= MAX_SLUG && utf8Length(slug) <= MAX_SLUG_BYTES) return slug; + const cut = chars.slice(0, MAX_SLUG); + while (utf8Length(cut.join("")) > MAX_SLUG_BYTES) cut.pop(); + // Never cut between a letter and its marks. + while (cut.length > 0 && /\p{M}/u.test(chars[cut.length] ?? "")) cut.pop(); // Truncate on a word boundary when one is near the end, so a long heading is // cut at "shipping-notes" rather than "shipping-note". - const cut = characters.slice(0, MAX_SLUG); const lastDash = cut.lastIndexOf("-"); - const trimmed = (lastDash >= MAX_SLUG / 2 ? cut.slice(0, lastDash) : cut).join(""); - return trimmed.replace(/-+$/, ""); + const trimmed = (lastDash >= cut.length / 2 ? cut.slice(0, lastDash) : cut).join(""); + return trimmed.replace(/-+$/, "") || null; +} + +const utf8 = new TextEncoder(); +function utf8Length(s: string): number { + return utf8.encode(s).length; } // The slug for a note's text, or null if it has no usable first-line H1. A note