Add sn delete duplicates

README.md

@@ -175,7 +175,7 @@ Run `sn <command> --help` for the flags of any command.
 | `add` | Add notes or tags |
 | `backup` | Create, inspect and restore zip backups (alias: `bak`) |
 | `debug` | Debugging tools, such as decrypting a single string |
-| `delete` | Delete notes, tags or arbitrary items by title or UUID |
+| `delete` | Delete notes, tags or items by title or UUID, or clear duplicates |
 | `edit` | Edit a note or tag in your `$EDITOR` |
 | `editor` | Show or set the editor Standard Notes associates with new notes |
 | `export` | Export notes to Markdown, HTML or JSON (alias: `exp`) |
@@ -224,6 +224,10 @@ sn get notes --title "Meeting Notes" --rich
 # Edit a note in $EDITOR
 sn edit note --title "Meeting Notes"
 
+# See which notes the app marked as copies, then delete them
+sn delete duplicates --dry-run
+sn delete duplicates
+
 # Tag every note whose title matches, in one go
 sn tag --find-title "invoice" --title finance --ignore-case
 

cmd/sncli/delete.go

@@ -11,7 +11,7 @@ func cmdDelete() *cli.Command {
 		Name:  "delete",
 		Usage: "delete items",
 		BashComplete: func(c *cli.Context) {
-			addTasks := []string{"tag", "note"}
+			addTasks := []string{"tag", "note", "duplicates"}
 			if c.NArg() > 0 {
 				return
 			}
@@ -100,6 +100,34 @@ func cmdDelete() *cli.Command {
 					return processDeleteItems(c, opts)
 				},
 			},
+			{
+				Name:  "duplicates",
+				Usage: "delete notes that Standard Notes marked as copies of another note",
+				BashComplete: func(c *cli.Context) {
+					delDupeOpts := []string{"--dry-run", "--yes"}
+					if c.NArg() > 0 {
+						return
+					}
+					for _, t := range delDupeOpts {
+						fmt.Println(t)
+					}
+				},
+				Flags: []cli.Flag{
+					&cli.BoolFlag{
+						Name:  "dry-run",
+						Usage: "list the duplicates without deleting them",
+					},
+					&cli.BoolFlag{
+						Name:  "yes",
+						Usage: "delete without asking for confirmation",
+					},
+				},
+				Action: func(c *cli.Context) error {
+					opts := getOpts(c)
+
+					return processDeleteDuplicates(c, opts)
+				},
+			},
 		},
 	}
 }

cmd/sncli/duplicates.go

@@ -0,0 +1,116 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/gookit/color"
+	"github.com/jonhadfield/gosn-v2/cache"
+	"github.com/jonhadfield/gosn-v2/common"
+	sncli "github.com/jonhadfield/sn-cli/internal/sncli"
+	"github.com/urfave/cli/v2"
+)
+
+// maxDuplicatesListed caps how many duplicates are listed before the rest are
+// summarised, so a large account does not fill the terminal.
+const maxDuplicatesListed = 20
+
+func processDeleteDuplicates(c *cli.Context, opts configOptsOutput) error {
+	sess, _, err := cache.GetSession(common.NewHTTPClient(), opts.useSession, opts.sessKey, opts.server, opts.debug)
+	if err != nil {
+		return err
+	}
+
+	if sess.CacheDBPath, err = cache.GenCacheDBPath(sess, opts.cacheDBDir, snAppName); err != nil {
+		return err
+	}
+
+	dryRun := c.Bool("dry-run")
+
+	// find the duplicates first, so they can be shown before anything is
+	// deleted
+	find := sncli.DeleteDuplicateNotesConfig{Session: &sess, DryRun: true, Debug: opts.debug}
+
+	found, err := find.Run()
+	if err != nil {
+		return fmt.Errorf("failed to find duplicate notes: %w", err)
+	}
+
+	writeDuplicatesKept(c, found.KeptOriginalMissing)
+
+	if len(found.Deleted) == 0 {
+		_, _ = fmt.Fprintln(c.App.Writer, "no duplicate notes found")
+
+		return nil
+	}
+
+	writeDuplicatesFound(c, found.Deleted, dryRun)
+
+	if dryRun {
+		return nil
+	}
+
+	if !c.Bool("yes") && !confirmDeleteDuplicates(c, len(found.Deleted)) {
+		return nil
+	}
+
+	del := sncli.DeleteDuplicateNotesConfig{Session: &sess, Debug: opts.debug}
+
+	out, err := del.Run()
+	if err != nil {
+		return fmt.Errorf("failed to delete duplicate notes: %w", err)
+	}
+
+	_, _ = fmt.Fprintln(c.App.Writer, color.Green.Sprintf("%d duplicate notes deleted", len(out.Deleted)))
+
+	return nil
+}
+
+func writeDuplicatesFound(c *cli.Context, dups []sncli.DuplicateNote, dryRun bool) {
+	heading := fmt.Sprintf("%d duplicate notes to delete:", len(dups))
+	if dryRun {
+		heading = fmt.Sprintf("%d duplicate notes would be deleted:", len(dups))
+	}
+
+	_, _ = fmt.Fprintln(c.App.Writer, heading)
+
+	for x, dup := range dups {
+		if x == maxDuplicatesListed {
+			_, _ = fmt.Fprintln(c.App.Writer, color.Gray.Sprintf("  ... and %d more", len(dups)-maxDuplicatesListed))
+
+			break
+		}
+
+		_, _ = fmt.Fprintf(c.App.Writer, "  %s  %s\n", dup.UUID, dup.Title)
+	}
+}
+
+func writeDuplicatesKept(c *cli.Context, kept []sncli.DuplicateNote) {
+	if len(kept) == 0 {
+		return
+	}
+
+	_, _ = fmt.Fprintln(c.App.Writer, color.Yellow.Sprintf(
+		"%d duplicates kept, as the note each was copied from is no longer in the account:", len(kept)))
+
+	for x, dup := range kept {
+		if x == maxDuplicatesListed {
+			_, _ = fmt.Fprintln(c.App.Writer, color.Gray.Sprintf("  ... and %d more", len(kept)-maxDuplicatesListed))
+
+			break
+		}
+
+		_, _ = fmt.Fprintf(c.App.Writer, "  %s  %s\n", dup.UUID, dup.Title)
+	}
+}
+
+func confirmDeleteDuplicates(c *cli.Context, num int) bool {
+	_, _ = fmt.Fprintf(c.App.Writer, "delete %d duplicate notes? ", num)
+
+	var input string
+
+	if _, err := fmt.Scanln(&input); err != nil {
+		return false
+	}
+
+	return sncli.StringInSlice(input, []string{"y", "yes"}, false)
+}

internal/sncli/duplicates.go

@@ -0,0 +1,144 @@
+package sncli
+
+import (
+	"fmt"
+
+	"github.com/jonhadfield/gosn-v2/cache"
+	"github.com/jonhadfield/gosn-v2/common"
+	"github.com/jonhadfield/gosn-v2/items"
+)
+
+// DuplicateNote is a note that Standard Notes marked as a copy of another,
+// by setting duplicate_of when the note was duplicated.
+type DuplicateNote struct {
+	UUID string
+	// Title of the duplicate itself
+	Title string
+	// OriginalUUID is the note this one was copied from
+	OriginalUUID string
+	// OriginalPresent reports whether that original is still in the account.
+	// When it is not, deleting the duplicate would lose the content, so the
+	// duplicate is kept.
+	OriginalPresent bool
+}
+
+// FindDuplicateNotes returns the notes marked as duplicates, and whether the
+// note each was copied from still exists.
+func FindDuplicateNotes(in items.Items) []DuplicateNote {
+	live := make(map[string]struct{})
+
+	for _, item := range in {
+		if item.GetContentType() == common.SNItemTypeNote && !item.IsDeleted() {
+			live[item.GetUUID()] = struct{}{}
+		}
+	}
+
+	var duplicates []DuplicateNote
+
+	for _, item := range in {
+		note, isNote := item.(*items.Note)
+		if !isNote || note.IsDeleted() || note.GetDuplicateOf() == "" {
+			continue
+		}
+
+		original := note.GetDuplicateOf()
+		_, originalPresent := live[original]
+
+		duplicates = append(duplicates, DuplicateNote{
+			UUID:            note.GetUUID(),
+			Title:           note.Content.GetTitle(),
+			OriginalUUID:    original,
+			OriginalPresent: originalPresent,
+		})
+	}
+
+	return duplicates
+}
+
+// DeleteDuplicateNotesConfig deletes notes marked as duplicates of another
+// note.
+type DeleteDuplicateNotesConfig struct {
+	Session *cache.Session
+	// DryRun reports what would be deleted without deleting anything.
+	DryRun bool
+	Debug  bool
+}
+
+// DeleteDuplicateNotesOutput describes what was, or would be, deleted.
+type DeleteDuplicateNotesOutput struct {
+	// Deleted holds the duplicates removed, or, for a dry run, those that
+	// would be removed.
+	Deleted []DuplicateNote
+	// KeptOriginalMissing holds duplicates left alone because the note they
+	// were copied from is no longer in the account.
+	KeptOriginalMissing []DuplicateNote
+}
+
+// Run finds duplicates and, unless this is a dry run, deletes them.
+func (i *DeleteDuplicateNotesConfig) Run() (DeleteDuplicateNotesOutput, error) {
+	var out DeleteDuplicateNotesOutput
+
+	so, err := Sync(cache.SyncInput{Session: i.Session}, true)
+	if err != nil {
+		return out, err
+	}
+
+	var allPersistedItems cache.Items
+
+	if err = so.DB.All(&allPersistedItems); err != nil {
+		return out, fmt.Errorf("getting items from db: %w", err)
+	}
+
+	var all items.Items
+
+	if all, err = allPersistedItems.ToItems(i.Session); err != nil {
+		return out, err
+	}
+
+	byUUID := make(map[string]*items.Note)
+
+	for _, item := range all {
+		if note, isNote := item.(*items.Note); isNote {
+			byUUID[note.GetUUID()] = note
+		}
+	}
+
+	for _, dup := range FindDuplicateNotes(all) {
+		if dup.OriginalPresent {
+			out.Deleted = append(out.Deleted, dup)
+
+			continue
+		}
+
+		out.KeptOriginalMissing = append(out.KeptOriginalMissing, dup)
+	}
+
+	if i.DryRun || len(out.Deleted) == 0 {
+		_ = i.Session.CacheDB.Close()
+
+		return out, nil
+	}
+
+	var notesToDelete items.Notes
+
+	for _, dup := range out.Deleted {
+		note := byUUID[dup.UUID]
+		if note == nil || note.GetContent() == nil {
+			continue
+		}
+
+		note.Content.SetText("")
+		note.SetDeleted(true)
+		notesToDelete = append(notesToDelete, *note)
+	}
+
+	if err = cache.SaveNotes(i.Session, so.DB, notesToDelete, true); err != nil {
+		return out, err
+	}
+
+	if _, err = Sync(cache.SyncInput{Session: i.Session, Close: true}, true); err != nil {
+		return out, err
+	}
+
+	return out, nil
+}