From 1d76372a08cfc0537e5d8491a6db4695d28aba5e Mon Sep 17 00:00:00 2001 From: Jon Hadfield Date: Mon, 21 Sep 2026 18:44:33 +0100 Subject: [PATCH] Add sn delete duplicates Standard Notes sets duplicate_of when the app duplicates a note, and 'sn stats' already lists those copies, but there was no way to remove them. 'sn delete duplicates' deletes them, after listing what it will remove and asking for confirmation. --dry-run lists them and stops; --yes skips the prompt. A duplicate whose original is no longer in the account is the only copy of that content left, so it is kept and reported rather than deleted. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 6 +- cmd/sncli/delete.go | 30 ++++++- cmd/sncli/duplicates.go | 116 ++++++++++++++++++++++++ internal/sncli/duplicates.go | 144 ++++++++++++++++++++++++++++++ internal/sncli/duplicates_test.go | 99 ++++++++++++++++++++ 5 files changed, 393 insertions(+), 2 deletions(-) create mode 100644 cmd/sncli/duplicates.go create mode 100644 internal/sncli/duplicates.go create mode 100644 internal/sncli/duplicates_test.go diff --git a/README.md b/README.md index 7554eaa..a37cd10 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ Run `sn --help` for the flags of any command. | `add` | Add notes or tags | | `backup` | Create, inspect and restore zip backups (alias: `bak`) | | `debug` | Debugging tools, such as decrypting a single string | -| `delete` | Delete notes, tags or arbitrary items by title or UUID | +| `delete` | Delete notes, tags or items by title or UUID, or clear duplicates | | `edit` | Edit a note or tag in your `$EDITOR` | | `editor` | Show or set the editor Standard Notes associates with new notes | | `export` | Export notes to Markdown, HTML or JSON (alias: `exp`) | @@ -224,6 +224,10 @@ sn get notes --title "Meeting Notes" --rich # Edit a note in $EDITOR sn edit note --title "Meeting Notes" +# See which notes the app marked as copies, then delete them +sn delete duplicates --dry-run +sn delete duplicates + # Tag every note whose title matches, in one go sn tag --find-title "invoice" --title finance --ignore-case diff --git a/cmd/sncli/delete.go b/cmd/sncli/delete.go index 991ebf7..e37bebd 100644 --- a/cmd/sncli/delete.go +++ b/cmd/sncli/delete.go @@ -11,7 +11,7 @@ func cmdDelete() *cli.Command { Name: "delete", Usage: "delete items", BashComplete: func(c *cli.Context) { - addTasks := []string{"tag", "note"} + addTasks := []string{"tag", "note", "duplicates"} if c.NArg() > 0 { return } @@ -100,6 +100,34 @@ func cmdDelete() *cli.Command { return processDeleteItems(c, opts) }, }, + { + Name: "duplicates", + Usage: "delete notes that Standard Notes marked as copies of another note", + BashComplete: func(c *cli.Context) { + delDupeOpts := []string{"--dry-run", "--yes"} + if c.NArg() > 0 { + return + } + for _, t := range delDupeOpts { + fmt.Println(t) + } + }, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "dry-run", + Usage: "list the duplicates without deleting them", + }, + &cli.BoolFlag{ + Name: "yes", + Usage: "delete without asking for confirmation", + }, + }, + Action: func(c *cli.Context) error { + opts := getOpts(c) + + return processDeleteDuplicates(c, opts) + }, + }, }, } } diff --git a/cmd/sncli/duplicates.go b/cmd/sncli/duplicates.go new file mode 100644 index 0000000..117a4cd --- /dev/null +++ b/cmd/sncli/duplicates.go @@ -0,0 +1,116 @@ +package main + +import ( + "fmt" + + "github.com/gookit/color" + "github.com/jonhadfield/gosn-v2/cache" + "github.com/jonhadfield/gosn-v2/common" + sncli "github.com/jonhadfield/sn-cli/internal/sncli" + "github.com/urfave/cli/v2" +) + +// maxDuplicatesListed caps how many duplicates are listed before the rest are +// summarised, so a large account does not fill the terminal. +const maxDuplicatesListed = 20 + +func processDeleteDuplicates(c *cli.Context, opts configOptsOutput) error { + sess, _, err := cache.GetSession(common.NewHTTPClient(), opts.useSession, opts.sessKey, opts.server, opts.debug) + if err != nil { + return err + } + + if sess.CacheDBPath, err = cache.GenCacheDBPath(sess, opts.cacheDBDir, snAppName); err != nil { + return err + } + + dryRun := c.Bool("dry-run") + + // find the duplicates first, so they can be shown before anything is + // deleted + find := sncli.DeleteDuplicateNotesConfig{Session: &sess, DryRun: true, Debug: opts.debug} + + found, err := find.Run() + if err != nil { + return fmt.Errorf("failed to find duplicate notes: %w", err) + } + + writeDuplicatesKept(c, found.KeptOriginalMissing) + + if len(found.Deleted) == 0 { + _, _ = fmt.Fprintln(c.App.Writer, "no duplicate notes found") + + return nil + } + + writeDuplicatesFound(c, found.Deleted, dryRun) + + if dryRun { + return nil + } + + if !c.Bool("yes") && !confirmDeleteDuplicates(c, len(found.Deleted)) { + return nil + } + + del := sncli.DeleteDuplicateNotesConfig{Session: &sess, Debug: opts.debug} + + out, err := del.Run() + if err != nil { + return fmt.Errorf("failed to delete duplicate notes: %w", err) + } + + _, _ = fmt.Fprintln(c.App.Writer, color.Green.Sprintf("%d duplicate notes deleted", len(out.Deleted))) + + return nil +} + +func writeDuplicatesFound(c *cli.Context, dups []sncli.DuplicateNote, dryRun bool) { + heading := fmt.Sprintf("%d duplicate notes to delete:", len(dups)) + if dryRun { + heading = fmt.Sprintf("%d duplicate notes would be deleted:", len(dups)) + } + + _, _ = fmt.Fprintln(c.App.Writer, heading) + + for x, dup := range dups { + if x == maxDuplicatesListed { + _, _ = fmt.Fprintln(c.App.Writer, color.Gray.Sprintf(" ... and %d more", len(dups)-maxDuplicatesListed)) + + break + } + + _, _ = fmt.Fprintf(c.App.Writer, " %s %s\n", dup.UUID, dup.Title) + } +} + +func writeDuplicatesKept(c *cli.Context, kept []sncli.DuplicateNote) { + if len(kept) == 0 { + return + } + + _, _ = fmt.Fprintln(c.App.Writer, color.Yellow.Sprintf( + "%d duplicates kept, as the note each was copied from is no longer in the account:", len(kept))) + + for x, dup := range kept { + if x == maxDuplicatesListed { + _, _ = fmt.Fprintln(c.App.Writer, color.Gray.Sprintf(" ... and %d more", len(kept)-maxDuplicatesListed)) + + break + } + + _, _ = fmt.Fprintf(c.App.Writer, " %s %s\n", dup.UUID, dup.Title) + } +} + +func confirmDeleteDuplicates(c *cli.Context, num int) bool { + _, _ = fmt.Fprintf(c.App.Writer, "delete %d duplicate notes? ", num) + + var input string + + if _, err := fmt.Scanln(&input); err != nil { + return false + } + + return sncli.StringInSlice(input, []string{"y", "yes"}, false) +} diff --git a/internal/sncli/duplicates.go b/internal/sncli/duplicates.go new file mode 100644 index 0000000..86eedc2 --- /dev/null +++ b/internal/sncli/duplicates.go @@ -0,0 +1,144 @@ +package sncli + +import ( + "fmt" + + "github.com/jonhadfield/gosn-v2/cache" + "github.com/jonhadfield/gosn-v2/common" + "github.com/jonhadfield/gosn-v2/items" +) + +// DuplicateNote is a note that Standard Notes marked as a copy of another, +// by setting duplicate_of when the note was duplicated. +type DuplicateNote struct { + UUID string + // Title of the duplicate itself + Title string + // OriginalUUID is the note this one was copied from + OriginalUUID string + // OriginalPresent reports whether that original is still in the account. + // When it is not, deleting the duplicate would lose the content, so the + // duplicate is kept. + OriginalPresent bool +} + +// FindDuplicateNotes returns the notes marked as duplicates, and whether the +// note each was copied from still exists. +func FindDuplicateNotes(in items.Items) []DuplicateNote { + live := make(map[string]struct{}) + + for _, item := range in { + if item.GetContentType() == common.SNItemTypeNote && !item.IsDeleted() { + live[item.GetUUID()] = struct{}{} + } + } + + var duplicates []DuplicateNote + + for _, item := range in { + note, isNote := item.(*items.Note) + if !isNote || note.IsDeleted() || note.GetDuplicateOf() == "" { + continue + } + + original := note.GetDuplicateOf() + _, originalPresent := live[original] + + duplicates = append(duplicates, DuplicateNote{ + UUID: note.GetUUID(), + Title: note.Content.GetTitle(), + OriginalUUID: original, + OriginalPresent: originalPresent, + }) + } + + return duplicates +} + +// DeleteDuplicateNotesConfig deletes notes marked as duplicates of another +// note. +type DeleteDuplicateNotesConfig struct { + Session *cache.Session + // DryRun reports what would be deleted without deleting anything. + DryRun bool + Debug bool +} + +// DeleteDuplicateNotesOutput describes what was, or would be, deleted. +type DeleteDuplicateNotesOutput struct { + // Deleted holds the duplicates removed, or, for a dry run, those that + // would be removed. + Deleted []DuplicateNote + // KeptOriginalMissing holds duplicates left alone because the note they + // were copied from is no longer in the account. + KeptOriginalMissing []DuplicateNote +} + +// Run finds duplicates and, unless this is a dry run, deletes them. +func (i *DeleteDuplicateNotesConfig) Run() (DeleteDuplicateNotesOutput, error) { + var out DeleteDuplicateNotesOutput + + so, err := Sync(cache.SyncInput{Session: i.Session}, true) + if err != nil { + return out, err + } + + var allPersistedItems cache.Items + + if err = so.DB.All(&allPersistedItems); err != nil { + return out, fmt.Errorf("getting items from db: %w", err) + } + + var all items.Items + + if all, err = allPersistedItems.ToItems(i.Session); err != nil { + return out, err + } + + byUUID := make(map[string]*items.Note) + + for _, item := range all { + if note, isNote := item.(*items.Note); isNote { + byUUID[note.GetUUID()] = note + } + } + + for _, dup := range FindDuplicateNotes(all) { + if dup.OriginalPresent { + out.Deleted = append(out.Deleted, dup) + + continue + } + + out.KeptOriginalMissing = append(out.KeptOriginalMissing, dup) + } + + if i.DryRun || len(out.Deleted) == 0 { + _ = i.Session.CacheDB.Close() + + return out, nil + } + + var notesToDelete items.Notes + + for _, dup := range out.Deleted { + note := byUUID[dup.UUID] + if note == nil || note.GetContent() == nil { + continue + } + + note.Content.SetText("") + note.SetDeleted(true) + notesToDelete = append(notesToDelete, *note) + } + + if err = cache.SaveNotes(i.Session, so.DB, notesToDelete, true); err != nil { + return out, err + } + + if _, err = Sync(cache.SyncInput{Session: i.Session, Close: true}, true); err != nil { + return out, err + } + + return out, nil +} diff --git a/internal/sncli/duplicates_test.go b/internal/sncli/duplicates_test.go new file mode 100644 index 0000000..f4a648e --- /dev/null +++ b/internal/sncli/duplicates_test.go @@ -0,0 +1,99 @@ +package sncli + +import ( + "testing" + + "github.com/jonhadfield/gosn-v2/common" + "github.com/jonhadfield/gosn-v2/items" + "github.com/stretchr/testify/require" +) + +func testNote(uuid, title, duplicateOf string, deleted bool) *items.Note { + note := &items.Note{ + ItemCommon: items.ItemCommon{ + UUID: uuid, + ContentType: common.SNItemTypeNote, + DuplicateOf: duplicateOf, + Deleted: deleted, + }, + } + note.Content.SetTitle(title) + + return note +} + +func TestFindDuplicateNotesMarkedOnly(t *testing.T) { + in := items.Items{ + testNote("original", "Shopping", "", false), + testNote("copy", "Shopping copy", "original", false), + testNote("unrelated", "Ideas", "", false), + } + + dups := FindDuplicateNotes(in) + + require.Len(t, dups, 1) + require.Equal(t, "copy", dups[0].UUID) + require.Equal(t, "Shopping copy", dups[0].Title) + require.Equal(t, "original", dups[0].OriginalUUID) + require.True(t, dups[0].OriginalPresent) +} + +// A duplicate whose original is gone is the only copy of that content left, +// so it must not be treated as safe to delete. +func TestFindDuplicateNotesOriginalMissing(t *testing.T) { + in := items.Items{ + testNote("copy", "Shopping copy", "gone", false), + } + + dups := FindDuplicateNotes(in) + + require.Len(t, dups, 1) + require.False(t, dups[0].OriginalPresent) +} + +func TestFindDuplicateNotesOriginalDeleted(t *testing.T) { + in := items.Items{ + testNote("original", "Shopping", "", true), + testNote("copy", "Shopping copy", "original", false), + } + + dups := FindDuplicateNotes(in) + + require.Len(t, dups, 1) + require.False(t, dups[0].OriginalPresent, "a deleted original should not count as present") +} + +func TestFindDuplicateNotesSkipsDeletedDuplicates(t *testing.T) { + in := items.Items{ + testNote("original", "Shopping", "", false), + testNote("copy", "Shopping copy", "original", true), + } + + require.Empty(t, FindDuplicateNotes(in)) +} + +func TestFindDuplicateNotesNone(t *testing.T) { + in := items.Items{ + testNote("a", "One", "", false), + testNote("b", "Two", "", false), + } + + require.Empty(t, FindDuplicateNotes(in)) +} + +func TestFindDuplicateNotesChain(t *testing.T) { + // a copy of a copy: both are duplicates, and both originals are present + in := items.Items{ + testNote("original", "Shopping", "", false), + testNote("copy", "Shopping copy", "original", false), + testNote("copy2", "Shopping copy 2", "copy", false), + } + + dups := FindDuplicateNotes(in) + + require.Len(t, dups, 2) + + for _, dup := range dups { + require.True(t, dup.OriginalPresent) + } +}