diff --git a/docs/contributor/architecture.md b/docs/contributor/architecture.md index c49ce1a..8a58d4f 100644 --- a/docs/contributor/architecture.md +++ b/docs/contributor/architecture.md @@ -220,10 +220,12 @@ Bun therefore validates everything and derives anything derivable: requirement is load-bearing, not tidiness: an in-root write of any other file would be arbitrary-file storage in a folder the user syncs. - **The view never names a file.** It sends a note's *text*; Bun slugs the - heading itself (`slugOf` emits only `[a-z0-9-]`), so filenames are safe by - construction and there is no name parameter to validate or smuggle through. - Workspace folder names get the same treatment (`createManaged` slugs the - display name; renaming a workspace is display-only and touches no folder). + heading itself (`slugOf` emits only Unicode letters, their combining marks, + numbers and hyphens, in NFC). Filenames are safe by construction: no dots, + path separators or control characters, and no name parameter to validate or + smuggle through. Workspace folder names get the same treatment + (`createManaged` slugs the display name; renaming a workspace is + display-only and touches no folder). - **Paths are opaque handles.** The view holds only paths and roots it was handed by Bun (`workspaceList`, `noteList`, `noteCreate`, `noteDelete`'s trashed location) and passes them back unmodified. View code that @@ -379,10 +381,11 @@ Bun therefore validates everything and derives anything derivable: purge, empty, or Trash listing can see them. - **A folder is the only name a caller chooses, so it is the only name validated.** Filenames never needed a guard: `slugOf` builds them from the - note's own heading and emits only `[a-z0-9-]`, so there is no name to check - and no way for the view to ask for a path. A folder has to match what is - already on disk, so it cannot be slugged into safety, and it arrives from - the view, from MCP and from the CLI alike. `folderPathOf` is the one gate: + note's own heading and emits only Unicode letters, their combining marks, + numbers and hyphens, so there is no name to check and no way for the view + to ask for a path. A folder has to match what is already on disk, so it + cannot be slugged into safety, and it arrives from the view, from MCP and + from the CLI alike. `folderPathOf` is the one gate: relative only, no backslashes, no `.` or `..` segment, no dot-entry, inside the root once resolved. `..` is rejected on the raw segments rather than left to the containment check, so `a/../b` cannot quietly mean `b` — the @@ -459,7 +462,8 @@ Bun therefore validates everything and derives anything derivable: volume) degrades to focus-refresh only, warned, like every other unavailable-root path. `rename(2)` overwrites silently, so no call site may pick its own destination name: `uniqueName` - (case-insensitive, because APFS is) allocates against a `readdir` snapshot + (ignoring case and Unicode normalization, because APFS does) allocates + against a `readdir` snapshot plus the `reserved` set, and the rename that follows is safe *because* the allocation already skipped every taken name. A rename whose target didn't come from `uniqueName` is a latent data-loss bug. diff --git a/docs/contributor/remote.md b/docs/contributor/remote.md index 69e3a5c..331ed28 100644 --- a/docs/contributor/remote.md +++ b/docs/contributor/remote.md @@ -568,7 +568,7 @@ installed on sshd's own PATH, such as `/usr/local/bin` on Linux. **`.server` is inside the app home**, beside `.client` (§5), because both are this machine's role-specific state and one `~/.ledge` is the whole of Ledge on a disk. It is dotted because managed workspace folders are created directly in -the app home under `[a-z0-9-]` slugs, so `server/` or `bin/` could be a +the app home under slugs of their names, so `server/` or `bin/` could be a workspace. `backup.ts` excludes it, and the root watcher and note listing skip dotted entries already. The path is spelled literally in `SERVE_COMMAND` and in the forced command on every paired server, so `LEDGE_NOTES_ROOT` moves diff --git a/src/bun/notes.fs.test.ts b/src/bun/notes.fs.test.ts index b013182..8f0346e 100644 --- a/src/bun/notes.fs.test.ts +++ b/src/bun/notes.fs.test.ts @@ -80,6 +80,26 @@ beforeEach(async () => { }); describe("createNote / writeNote / readNote", () => { + test("Korean headings name files and duplicate headings enumerate without clobbering", async () => { + const a = await createNote(ROOT, "# 한국어 제목\n\n첫 번째"); + const b = await createNote(ROOT, "# 한국어 제목\n\n두 번째"); + expect(a.path).toBe(join(ROOT, "한국어-제목.md")); + expect(b.path).toBe(join(ROOT, "한국어-제목-2.md")); + expect(a.title).toBe("한국어 제목"); + expect(await textAt(a.path)).toBe("# 한국어 제목\n\n첫 번째"); + expect(await textAt(b.path)).toBe("# 한국어 제목\n\n두 번째"); + }); + + // Finder writes Korean names decomposed (NFD). On APFS that is the same file + // as the composed name slugify emits, so writing there would replace it. + test("a note beside a decomposed same-titled file does not overwrite it", async () => { + const existing = join(ROOT, "회의록.md".normalize("NFD")); + await writeRaw(existing, "# 회의록\n\nolder"); + const made = await createNote(ROOT, "# 회의록\n\nnewer"); + expect(made.path).toBe(join(ROOT, "회의록-2.md")); + expect(await textAt(existing)).toBe("# 회의록\n\nolder"); + }); + test("names the file from the note's H1 and round-trips its text", async () => { const note = await createNote(ROOT, "# Shipping Notes\n\nhello"); expect(note.path).toBe(join(ROOT, "shipping-notes.md")); @@ -476,6 +496,29 @@ describe("notesTagged", () => { }); describe("retitleNote", () => { + test("an untitled note can be renamed to a Korean heading", async () => { + const note = await createNote(ROOT, "본문"); + const moved = await retitleNote(note.path, "# 한국어 제목\n\n본문"); + expect(moved.path).toBe(join(ROOT, "한국어-제목.md")); + expect(await textAt(moved.path)).toBe("본문"); + expect(await textAt(note.path)).toBeNull(); + }); + + test("a retitle onto a decomposed file's name does not overwrite it", async () => { + const existing = join(ROOT, "회의록.md".normalize("NFD")); + await writeRaw(existing, "# 회의록\n\nolder"); + const note = await createNote(ROOT, "draft"); + const moved = await retitleNote(note.path, "# 회의록\n\ndraft"); + expect(moved.path).toBe(join(ROOT, "회의록-2.md")); + expect(await textAt(existing)).toBe("# 회의록\n\nolder"); + }); + + test("a decomposed file already named for its heading stays put", async () => { + const existing = join(ROOT, "회의록.md".normalize("NFD")); + await writeRaw(existing, "# 회의록\n"); + expect((await retitleNote(existing, "# 회의록\n")).path).toBe(existing); + }); + test("moves the file to match its heading", async () => { const note = await createNote(ROOT, "# Draft\n"); const moved = await retitleNote(note.path, "# Final\n"); diff --git a/src/bun/notes.ts b/src/bun/notes.ts index c684168..2a8b8d4 100644 --- a/src/bun/notes.ts +++ b/src/bun/notes.ts @@ -21,7 +21,7 @@ import { collectHits, type SearchHit } from "../shared/search"; import { resolveWikiTitle, wikiRefsOf } from "../shared/wikilinks"; import { normalizeTag, tagDirectoryOf, tagRefsOf, type TagInfo } from "../shared/tags"; import { loadIgnore } from "./ignore"; -import { assertRegisteredRoot, assertWritableRoot, GITIGNORE, isInside, kindOf, rootContaining, uniqueName } from "./workspaces"; +import { assertRegisteredRoot, assertWritableRoot, GITIGNORE, isInside, kindOf, nameKey, rootContaining, uniqueName } from "./workspaces"; import { beginPassphraseChange, commitPassphraseChange, @@ -74,9 +74,8 @@ export { titleOf }; // The filename a note's text asks for: its first-line H1 as a slug, or // "untitled" when the note has no usable heading. Bun slugs the heading itself -// rather than taking a name from the view. slugify emits only [a-z0-9-], so -// there is no name to validate and no way for the view to ask for a path -// (architecture.md §2). +// rather than taking a name from the view. slugify never emits a dot or a path +// separator, so the view cannot ask for a path (architecture.md §2). function baseFor(text: string): string { return slugOf(text) ?? "untitled"; } @@ -1271,7 +1270,7 @@ export async function retitleNote(path: string, text: string): Promise for (const name of reserved) taken.add(name); const name = uniqueName(baseFor(text), taken); - if (name.toLowerCase() === current.toLowerCase()) { + if (nameKey(name) === nameKey(current)) { // Already correctly named, or uniqueName landed back on its own name. return metaFor(path, text); } diff --git a/src/bun/workspaces.fs.test.ts b/src/bun/workspaces.fs.test.ts index 49de305..77d7098 100644 --- a/src/bun/workspaces.fs.test.ts +++ b/src/bun/workspaces.fs.test.ts @@ -98,6 +98,10 @@ describe("createManaged", () => { expect(rules).toEqual([".ledge-trash/", ".*.md.tmp-*", ".DS_Store"]); }); + test("a Korean display name creates a Korean workspace folder", async () => { + expect(await createManaged("한국어 노트")).toBe(join(resolve(APP_HOME), "한국어-노트")); + }); + test("a name that slugs to nothing falls back rather than failing", async () => { const root = await createManaged("???"); expect(root).toBe(join(resolve(APP_HOME), "workspace")); diff --git a/src/bun/workspaces.test.ts b/src/bun/workspaces.test.ts index 8204057..1b4d693 100644 --- a/src/bun/workspaces.test.ts +++ b/src/bun/workspaces.test.ts @@ -19,6 +19,12 @@ describe("uniqueName", () => { expect(uniqueName("notes", new Set(["Notes.md", "notes-2.MD"]))).toBe("notes-3.md"); }); + // APFS also treats composed and decomposed spellings as one file, and Finder + // writes Korean names decomposed (NFD) while slugify emits NFC. + test("comparison ignores Unicode normalization, so an NFD name counts as taken", () => { + expect(uniqueName("회의록", new Set(["회의록.md".normalize("NFD")]))).toBe("회의록-2.md"); + }); + test("suffixes from 2 upward, skipping every taken name", () => { const taken = new Set(["untitled.md", "untitled-2.md", "untitled-3.md"]); expect(uniqueName("untitled", taken)).toBe("untitled-4.md"); diff --git a/src/bun/workspaces.ts b/src/bun/workspaces.ts index 0db0432..d59d6ae 100644 --- a/src/bun/workspaces.ts +++ b/src/bun/workspaces.ts @@ -32,7 +32,7 @@ export const WORKSPACES_PATH = join(APP_HOME, ".workspaces.json"); // .workspaces.json, and every mutating seam in notes.ts and assets.ts refuses // it by calling assertWritableRoot below. The name is dotted and // .ledge-prefixed like every app-owned entry, so it cannot collide with a -// managed workspace slug (slugify emits only [a-z0-9-], never a leading dot). +// managed workspace slug (slugify never emits a leading dot). export const DOCS_ROOT = join(APP_HOME, ".ledge-docs"); // The ignore file a new managed workspace is seeded with, so that `git init` @@ -70,17 +70,23 @@ export function isInside(root: string, p: string): boolean { // Allocate a filename not already in `taken`: shipping-notes.md, // shipping-notes-2.md, and so on (with ext "", a workspace folder name). -// Comparison is case-insensitive because macOS's default APFS is: "Foo.md" and -// "foo.md" are one file there, so a case-sensitive check would return a name -// whose rename overwrites the other note. On Linux it only means an extra -// foo-2.md where foo.md was free. +// Comparison ignores case and Unicode normalization because macOS's default +// APFS does: "Foo.md" and "foo.md" are one file there, and so are a composed +// "회의록.md" and the decomposed one Finder writes. A stricter check would +// return a name whose rename overwrites the other note. On Linux it only means +// an extra foo-2.md where foo.md was free. export function uniqueName(base: string, taken: Set, ext = ".md"): string { - const lower = new Set([...taken].map((t) => t.toLowerCase())); + const keys = new Set([...taken].map(nameKey)); let name = `${base}${ext}`; - for (let n = 2; lower.has(name.toLowerCase()); n += 1) name = `${base}-${n}${ext}`; + for (let n = 2; keys.has(nameKey(name)); n += 1) name = `${base}-${n}${ext}`; return name; } +// The form two names share when APFS treats them as one file (uniqueName). +export function nameKey(name: string): string { + return name.normalize("NFC").toLowerCase(); +} + // The roots a loosely-spelled workspace name could mean: a root path (~ // expands), or the folder name of a registered root as shorthand. The CLI's // --workspace argument (cli.ts resolveWorkspaceArg) and the daily.workspace @@ -349,11 +355,10 @@ export async function ensureDefault(): Promise { } // Creates a managed workspace folder from a display name. Bun slugs the name -// itself, the same trust move as noteCreate: slugify emits only [a-z0-9-], so -// the view has no path to smuggle through. uniqueName allocates the folder name -// against a readdir snapshot of APP_HOME, so the mkdir cannot land on something -// that exists. settings.jsonc, dot-entries, and squatting files all count as -// taken. +// itself, the same trust move as noteCreate: slugify never emits a dot or a +// path separator, so the view has no path to smuggle through. uniqueName +// allocates against a readdir snapshot of APP_HOME, including settings.jsonc, +// dot-entries and squatting files, so mkdir cannot land on something that exists. export async function createManaged(name: string): Promise { await ensureAppHome(); const base = slugify(name) ?? "workspace"; diff --git a/src/shared/slug.test.ts b/src/shared/slug.test.ts index 0e12259..caab264 100644 --- a/src/shared/slug.test.ts +++ b/src/shared/slug.test.ts @@ -103,7 +103,53 @@ describe("slugify", () => { // The caller falls back to the enumerated "untitled" name. expect(slugify("!!!")).toBeNull(); expect(slugify(" ")).toBeNull(); - expect(slugify("日本語")).toBeNull(); + expect(slugify("🎉")).toBeNull(); + }); + + test("Korean and mixed-language headings keep their letters", () => { + expect(slugify("한국어 제목")).toBe("한국어-제목"); + expect(slugify("Ledge 한국어 노트 2026")).toBe("ledge-한국어-노트-2026"); + expect(slugify("日本語")).toBe("日本語"); + }); + + test("decomposed Hangul produces the same composed filename", () => { + expect(slugify("한국어 제목".normalize("NFD"))).toBe("한국어-제목"); + }); + + // Only Latin letters lose their accents. Elsewhere a mark changes the letter + // or is part of the word. + test("marks outside Latin script are kept, and the result is composed", () => { + expect(slugify("ガイド")).toBe("ガイド"); + expect(slugify("ガイド".normalize("NFD"))).toBe("ガイド".normalize("NFC")); + expect(slugify("हिन्दी नोट्स")).toBe("हिन्दी-नोट्स"); + expect(slugify("Мой план")).toBe("мой-план"); + expect(slugify("Tiếng Việt")).toBe("tieng-viet"); + }); + + test("a mark with no letter before it is dropped with the separator", () => { + expect(slugify("́abc")).toBe("abc"); + expect(slugify("가 ́나")).toBe("가-나"); + }); + + test("Unicode headings still remove unsafe path characters", () => { + expect(slugify("../../한국어/제목\\초안:메모\u0000")).toBe("한국어-제목-초안-메모"); + }); + + // The temp file a save writes is the longest name a slug is part of + // (notes.ts writeNote), and ext4 refuses a name over 255 bytes. + test("long Unicode headings fit the filename limit without splitting a letter", () => { + expect(slugify("한".repeat(200))).toBe("한".repeat(60)); + const slug = slugify("𠀀".repeat(200))!; + expect(slug).toBe("𠀀".repeat(50)); + const temp = `.${slug}-99.md.tmp-4194304-4294967295`; + expect(new TextEncoder().encode(temp).length).toBeLessThanOrEqual(255); + }); + + test("truncation never separates a letter from its marks", () => { + // 60 code points end on ह, whose vowel sign ि is the 61st. + expect(slugify(`क${"हि".repeat(40)}`)).toBe(`क${"हि".repeat(29)}`); + // A letter buried under marks has nothing left once they are kept whole. + expect(slugify(`क${"́".repeat(150)}`)).toBeNull(); }); test("the slug is safe by construction: no dots, separators, or leading dot", () => { @@ -130,6 +176,7 @@ describe("slugify", () => { describe("slugOf", () => { test("a titled note asks for its heading's slug", () => { expect(slugOf("# Shipping Notes\n\nbody")).toBe("shipping-notes"); + expect(slugOf("# 한국어 제목\n\n본문")).toBe("한국어-제목"); }); test("an untitled note asks for nothing", () => { diff --git a/src/shared/slug.ts b/src/shared/slug.ts index 2183787..87da052 100644 --- a/src/shared/slug.ts +++ b/src/shared/slug.ts @@ -10,10 +10,12 @@ // Renaming is safe only while a note's identity is its docId and not its path // (architecture.md §4). -// Long enough for a real heading, short enough to stay clear of the 255-byte -// filename limit every filesystem here shares, with room for a "-2" suffix and -// the extension. +// A slug is at most 60 code points and at most 200 UTF-8 bytes. The longest +// name built from one is a save's temp file (".-2.md.tmp--", +// notes.ts writeNote), about 30 bytes more, and ext4 refuses a name over 255 +// bytes. Hangul and kanji fit all 60; 4-byte letters stop at 50. const MAX_SLUG = 60; +const MAX_SLUG_BYTES = 200; // Far longer than any real heading, short enough that a runaway one cannot // bloat the store. Only the on-screen label is capped; the note's text is @@ -44,26 +46,38 @@ export function headingOf(text: string): string | null { } // Turns a heading into a filesystem-safe slug, or null if nothing usable is -// left. Everything outside [a-z0-9] becomes a separator, which makes the -// result safe by construction: no dots, no slashes, no colons, no control -// characters, no leading dot. This is the whole trust boundary for a name -// arriving from the view (architecture.md §2). +// left. Everything outside Unicode letters, their combining marks and numbers +// becomes a separator: no dots, no slashes, no colons, no control characters, +// no leading dot. The result is NFC. This is the whole trust boundary for a +// name arriving from the view (architecture.md §2). export function slugify(heading: string): string | null { - // Fold accents first ("Café" -> "cafe"). Otherwise each accented letter - // becomes a separator and breaks the word into pieces. - const folded = heading.normalize("NFD").replace(/\p{M}/gu, ""); - const slug = folded + // Fold accents off Latin letters ("Café" -> "cafe"), so a Latin heading + // names an ASCII file. Other scripts keep their marks: が is not か, and a + // Devanagari vowel sign is part of the word. NFC recomposes what NFD split. + const folded = heading + .normalize("NFD") + .replace(/(\p{Script=Latin})\p{M}+/gu, "$1") .toLowerCase() - .replace(/[^a-z0-9]+/g, "-") - .replace(/^-+|-+$/g, ""); - if (!slug) return null; // a heading of pure punctuation or non-latin script - if (slug.length <= MAX_SLUG) return slug; + .normalize("NFC"); + // A mark with no letter before it goes with the separator it follows. + const slug = folded.replace(/^\p{M}+|[^\p{L}\p{M}\p{N}]+\p{M}*/gu, "-").replace(/^-+|-+$/g, ""); + if (!slug) return null; // a heading of pure punctuation + const chars = Array.from(slug); + if (chars.length <= MAX_SLUG && utf8Length(slug) <= MAX_SLUG_BYTES) return slug; + const cut = chars.slice(0, MAX_SLUG); + while (utf8Length(cut.join("")) > MAX_SLUG_BYTES) cut.pop(); + // Never cut between a letter and its marks. + while (cut.length > 0 && /\p{M}/u.test(chars[cut.length] ?? "")) cut.pop(); // Truncate on a word boundary when one is near the end, so a long heading is // cut at "shipping-notes" rather than "shipping-note". - const cut = slug.slice(0, MAX_SLUG); const lastDash = cut.lastIndexOf("-"); - const trimmed = lastDash >= MAX_SLUG / 2 ? cut.slice(0, lastDash) : cut; - return trimmed.replace(/-+$/, ""); + const trimmed = (lastDash >= cut.length / 2 ? cut.slice(0, lastDash) : cut).join(""); + return trimmed.replace(/-+$/, "") || null; +} + +const utf8 = new TextEncoder(); +function utf8Length(s: string): number { + return utf8.encode(s).length; } // The slug for a note's text, or null if it has no usable first-line H1. A note