diff --git a/FileDeduplicator.Test/DeduplicatorTests.cs b/FileDeduplicator.Test/DeduplicatorTests.cs index 8888eb9..bf87cf2 100644 --- a/FileDeduplicator.Test/DeduplicatorTests.cs +++ b/FileDeduplicator.Test/DeduplicatorTests.cs @@ -193,7 +193,7 @@ public void DeleteDuplicatesRemovesEveryCopyExceptTheKeeper() AbsoluteFilePath keeper = Deduplicator.SelectFileToKeep(duplicates[0].Files); // Act - DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates); + DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates, tree.Root); // Assert Assert.AreEqual(2, result.DeletedCount); @@ -223,7 +223,7 @@ public void EmptyFilesAreNeverDeleted() IReadOnlyList duplicates = Duplicates(FileHasher.HashFiles([initA, initB, copy, keeper])); // Act - DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates); + DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates, tree.Root); // Assert Assert.HasCount(2, duplicates, "The empty files are still reported as a group."); @@ -253,7 +253,7 @@ public void ReclaimedBytesCountsOnlyTheDeletedCopies() IReadOnlyList duplicates = Duplicates(hashes); // Act - DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates); + DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates, tree.Root); // Assert -- three copies, two deleted, 100 bytes apiece Assert.AreEqual(2, result.DeletedCount); @@ -280,7 +280,7 @@ public void AFileThatChangedAfterGroupingIsPreservedAndReported() _ = tree.Write("bb.txt", "no longer the same content"); // Act - DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates); + DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates, tree.Root); // Assert Assert.IsTrue(TempTree.Exists(changed), "A file that is no longer a duplicate must not be deleted."); @@ -311,7 +311,7 @@ public void AChangedKeeperPreservesTheWholeGroup() _ = tree.Write("a.txt", "the keeper was overwritten"); // Act - DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates); + DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates, tree.Root); // Assert Assert.AreEqual(0, result.DeletedCount); @@ -340,6 +340,7 @@ public void AKeeperChangedDuringTheDeleteLoopPreservesTheRemainingCopies() // Act -- the keeper is rewritten right after the first deletion, as a sync client would DeduplicationResult result = Deduplicator.DeleteDuplicates( duplicates, + tree.Root, deleted => { if (deleted == first) @@ -375,6 +376,7 @@ public void AKeeperRewrittenWithoutChangingSizeOrTimeIsReportedAfterTheGroup() // Act -- same length, timestamp put back DeduplicationResult result = Deduplicator.DeleteDuplicates( duplicates, + tree.Root, _ => { File.WriteAllText(keeper.WeakString, "SHARED"); @@ -404,7 +406,7 @@ public void AFileThatVanishedBeforeDeletionIsReportedNotCounted() File.Delete(vanishing.WeakString); // Act - DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates); + DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates, tree.Root); // Assert Assert.AreEqual(0, result.DeletedCount); @@ -434,7 +436,7 @@ public void APathReplacedByADirectoryIsSkippedRatherThanDeleted() _ = Directory.CreateDirectory(replaced.WeakString); // Act - DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates); + DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates, tree.Root); // Assert Assert.AreEqual(0, result.DeletedCount); @@ -462,7 +464,7 @@ public void UntouchedDuplicatesAreDeletedWithNothingSkipped() IReadOnlyList duplicates = Duplicates(hashes); // Act - DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates); + DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates, tree.Root); // Assert Assert.AreEqual(2, result.DeletedCount); @@ -496,7 +498,7 @@ public void ACopyThatCannotBeDeletedIsReportedAndTheRunCarriesOn() } // Act - DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates); + DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates, tree.Root); // Assert -- the refusal is reported, not thrown Assert.ContainsSingle(result.Errors); @@ -517,8 +519,11 @@ public void ACopyThatCannotBeDeletedIsReportedAndTheRunCarriesOn() [TestMethod] public void DeletingAnEmptyGroupListIsANoOp() { + // Arrange + using TempTree tree = new(); + // Act - DeduplicationResult result = Deduplicator.DeleteDuplicates([]); + DeduplicationResult result = Deduplicator.DeleteDuplicates([], tree.Root); // Assert Assert.AreEqual(0, result.DeletedCount); diff --git a/FileDeduplicator.Test/DryRunEquivalenceTests.cs b/FileDeduplicator.Test/DryRunEquivalenceTests.cs index 4b06079..6317379 100644 --- a/FileDeduplicator.Test/DryRunEquivalenceTests.cs +++ b/FileDeduplicator.Test/DryRunEquivalenceTests.cs @@ -71,7 +71,7 @@ public void PredictedDeletionsMatchActualDeletionsExactly() HashSet predicted = PredictedDeletions(duplicates); // Act - DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates); + DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates, tree.Root); HashSet actuallyDeleted = [.. all.Where(f => !TempTree.Exists(f))]; // Assert @@ -104,7 +104,7 @@ public void EveryDuplicateGroupRetainsExactlyOneSurvivor() IReadOnlyList duplicates = Deduplicator.FindDuplicates(Deduplicator.GroupByHash(hashes)); // Act - _ = Deduplicator.DeleteDuplicates(duplicates); + _ = Deduplicator.DeleteDuplicates(duplicates, tree.Root); // Assert foreach (DuplicateGroup group in duplicates) @@ -131,13 +131,13 @@ public void DeduplicatingATreeTwiceDeletesNothingTheSecondTime() IReadOnlyList firstScan = FileScanner.ScanForFiles(tree.Root); IReadOnlyList firstDuplicates = Deduplicator.FindDuplicates(Deduplicator.GroupByHash(FileHasher.HashFiles(firstScan))); - DeduplicationResult first = Deduplicator.DeleteDuplicates(firstDuplicates); + DeduplicationResult first = Deduplicator.DeleteDuplicates(firstDuplicates, tree.Root); // Act -- second pass over the now-deduplicated tree IReadOnlyList secondScan = FileScanner.ScanForFiles(tree.Root); IReadOnlyList secondDuplicates = Deduplicator.FindDuplicates(Deduplicator.GroupByHash(FileHasher.HashFiles(secondScan))); - DeduplicationResult second = Deduplicator.DeleteDuplicates(secondDuplicates); + DeduplicationResult second = Deduplicator.DeleteDuplicates(secondDuplicates, tree.Root); // Assert Assert.AreEqual(1, first.DeletedCount); diff --git a/FileDeduplicator.Test/SameFileTests.cs b/FileDeduplicator.Test/SameFileTests.cs index de4aab9..b8886e9 100644 --- a/FileDeduplicator.Test/SameFileTests.cs +++ b/FileDeduplicator.Test/SameFileTests.cs @@ -181,7 +181,7 @@ public void DeletionSkipsAPathThatLeadsToTheKeeper() DuplicateGroup group = new(hash, [keeper, link], "content".Length); // Act - DeduplicationResult result = Deduplicator.DeleteDuplicates([group]); + DeduplicationResult result = Deduplicator.DeleteDuplicates([group], tree.Root); // Assert Assert.AreEqual(0, result.DeletedCount); diff --git a/FileDeduplicator.Test/ScannedRootContainmentTests.cs b/FileDeduplicator.Test/ScannedRootContainmentTests.cs new file mode 100644 index 0000000..5e85fc3 --- /dev/null +++ b/FileDeduplicator.Test/ScannedRootContainmentTests.cs @@ -0,0 +1,180 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.FileDeduplicator.Test; + +using ktsu.Semantics.Paths; +using ktsu.Semantics.Strings; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Deletion goes through path strings recorded at scan time. These tests check that a path which +/// no longer leads inside the scanned folder is never deleted through. +/// +[TestClass] +public sealed class ScannedRootContainmentTests +{ + /// + /// A subdirectory replaced by a link to a folder outside the root during the confirmation + /// prompt must not let the delete reach an identical file out there. + /// + [TestMethod] + public void ASubdirectorySwappedForALinkOutsideTheRootIsNotDeletedThrough() + { + // Arrange -- a.cfg is kept, sub/settings.cfg is up for deletion + using TempTree tree = new(); + using TempTree outside = new(); + AbsoluteFilePath keeper = tree.Write("a.cfg", "important config v1"); + AbsoluteFilePath copy = tree.Write("sub/settings.cfg", "important config v1"); + AbsoluteFilePath outsideFile = outside.Write("settings.cfg", "important config v1"); + IReadOnlyList duplicates = Deduplicator.FindDuplicates(Deduplicator.GroupByHash(FileHasher.HashFiles([keeper, copy]))); + + // Arrange -- while the prompt is open, sub/ becomes a link to the outside folder + string sub = Path.Combine(tree.Root.WeakString, "sub"); + Directory.Delete(sub, recursive: true); + CreateDirectoryLinkOrSkip(sub, outside.Root.WeakString); + + // Act + DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates, tree.Root); + + // Assert + Assert.IsTrue(TempTree.Exists(outsideFile), "A file outside the scanned folder must never be deleted."); + Assert.IsTrue(TempTree.Exists(keeper), "The keeper must survive."); + Assert.AreEqual(0, result.DeletedCount); + Assert.ContainsSingle(result.SkippedFiles); + Assert.AreEqual(copy, result.SkippedFiles[0].Path); + Assert.Contains("scanned folder", result.SkippedFiles[0].Reason); + } + + /// + /// The root itself may be reached through a link, as when a user passes a symlinked home + /// folder. That alone must not stop copies under it from being deleted. + /// + [TestMethod] + public void ARootReachedThroughALinkStillDeletesItsDuplicates() + { + // Arrange + using TempTree tree = new(); + using TempTree links = new(); + string linkedRoot = Path.Combine(links.Root.WeakString, "linked-root"); + CreateDirectoryLinkOrSkip(linkedRoot, tree.Root.WeakString); + AbsoluteDirectoryPath root = linkedRoot.As(); + AbsoluteFilePath keeper = Path.Combine(linkedRoot, "a.txt").As(); + AbsoluteFilePath copy = Path.Combine(linkedRoot, "nested", "bb.txt").As(); + _ = tree.Write("a.txt", "shared"); + _ = tree.Write("nested/bb.txt", "shared"); + IReadOnlyList duplicates = Deduplicator.FindDuplicates(Deduplicator.GroupByHash(FileHasher.HashFiles([keeper, copy]))); + + // Act + DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates, root); + + // Assert + Assert.AreEqual(1, result.DeletedCount); + Assert.IsEmpty(result.SkippedFiles); + Assert.IsFalse(TempTree.Exists(copy), "A copy inside the root should still be deleted."); + } + + /// + /// Resolution follows a relative link and a link inside a link's target. + /// + [TestMethod] + public void RealPathFollowsNestedRelativeLinks() + { + // Arrange -- first -> second (relative), second -> target (absolute) + using TempTree tree = new(); + _ = tree.Write("target/file.txt", "x"); + string target = Path.Combine(tree.Root.WeakString, "target"); + string second = Path.Combine(tree.Root.WeakString, "second"); + string first = Path.Combine(tree.Root.WeakString, "first"); + CreateDirectoryLinkOrSkip(second, target); + CreateDirectoryLinkOrSkip(first, "second"); + Assert.IsTrue(RealPath.TryResolve(tree.Root.WeakString, out string realRoot)); + + // Act + bool resolved = RealPath.TryResolve(Path.Combine(first, "file.txt"), out string real); + + // Assert + Assert.IsTrue(resolved); + Assert.AreEqual(Path.Combine(realRoot, "target", "file.txt"), real); + Assert.IsTrue(RealPath.IsWithin(real, realRoot)); + Assert.IsFalse(RealPath.IsWithin(realRoot + "-sibling", realRoot), "A sibling sharing the root's name as a prefix is not inside it."); + } + + /// + /// A link loop never resolves, so a scanned root reached through one leaves every copy on disk + /// with a reason, rather than deleting on an unchecked path. + /// + [TestMethod] + public void AnUnresolvableRootSkipsEveryCopy() + { + // Arrange -- loop-a -> loop-b -> loop-a + using TempTree tree = new(); + AbsoluteFilePath keeper = tree.Write("a.txt", "shared"); + AbsoluteFilePath copy = tree.Write("bb.txt", "shared"); + IReadOnlyList duplicates = Deduplicator.FindDuplicates(Deduplicator.GroupByHash(FileHasher.HashFiles([keeper, copy]))); + string loopA = Path.Combine(tree.Root.WeakString, "loop-a"); + string loopB = Path.Combine(tree.Root.WeakString, "loop-b"); + CreateDirectoryLinkOrSkip(loopA, loopB); + CreateDirectoryLinkOrSkip(loopB, loopA); + + // Act + DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates, loopA.As()); + + // Assert + Assert.AreEqual(0, result.DeletedCount); + Assert.IsTrue(TempTree.Exists(copy), "A copy must not be deleted when the root cannot be resolved."); + Assert.ContainsSingle(result.SkippedFiles); + Assert.Contains("could not be resolved", result.SkippedFiles[0].Reason); + } + + /// + /// Resolution refuses a relative path, gives up on a link loop, and handles "." and ".." + /// segments without leaving the real directory tree. + /// + [TestMethod] + public void RealPathHandlesDotSegmentsLoopsAndRelativeInput() + { + // Arrange + using TempTree tree = new(); + _ = tree.Write("dir/file.txt", "x"); + Assert.IsTrue(RealPath.TryResolve(tree.Root.WeakString, out string realRoot)); + string loopA = Path.Combine(tree.Root.WeakString, "loop-a"); + string loopB = Path.Combine(tree.Root.WeakString, "loop-b"); + CreateDirectoryLinkOrSkip(loopA, loopB); + CreateDirectoryLinkOrSkip(loopB, loopA); + string dotted = tree.Root.WeakString + $"{Path.DirectorySeparatorChar}.{Path.DirectorySeparatorChar}dir{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}dir{Path.DirectorySeparatorChar}file.txt"; + + // Act + Assert + Assert.IsFalse(RealPath.TryResolve(Path.Combine("relative", "path"), out _), "A relative path has no single meaning to resolve."); + Assert.IsFalse(RealPath.TryResolve(Path.Combine(loopA, "file.txt"), out _), "A link loop never resolves."); + Assert.IsTrue(RealPath.TryResolve(dotted, out string real)); + Assert.AreEqual(Path.Combine(realRoot, "dir", "file.txt"), real); + } + + /// + /// A file-system root contains everything on it, including paths that do not repeat its + /// trailing separator. + /// + [TestMethod] + public void EverythingIsWithinAFileSystemRoot() + { + // Arrange + string root = Path.GetPathRoot(Path.GetTempPath())!; + + // Act + Assert + Assert.IsTrue(RealPath.IsWithin(Path.Combine(root, "any", "where"), root)); + Assert.IsTrue(RealPath.IsWithin(root, root)); + } + + private static void CreateDirectoryLinkOrSkip(string path, string target) + { + try + { + _ = Directory.CreateSymbolicLink(path, target); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Windows without Developer Mode or elevation refuses to create symbolic links. + Assert.Inconclusive($"Cannot create a symbolic link here: {ex.Message}"); + } + } +} diff --git a/FileDeduplicator/Deduplicator.cs b/FileDeduplicator/Deduplicator.cs index b01dc19..a7f066d 100644 --- a/FileDeduplicator/Deduplicator.cs +++ b/FileDeduplicator/Deduplicator.cs @@ -139,18 +139,24 @@ private static IOrderedEnumerable OrderByPreference(IEnumerabl /// Deletes every copy in each deletable group except the one keeps. /// /// The groups to deduplicate. + /// + /// The directory that was scanned. A copy whose path no longer leads inside it is not deleted. + /// /// /// Called after each copy is deleted. Production code passes nothing; tests use it to change the /// filesystem between two deletions in one group. /// /// What was deleted, skipped and failed. - internal static DeduplicationResult DeleteDuplicates(IReadOnlyList duplicateGroups, Action? afterDelete = null) + internal static DeduplicationResult DeleteDuplicates(IReadOnlyList duplicateGroups, AbsoluteDirectoryPath root, Action? afterDelete = null) { int deletedCount = 0; long bytesReclaimed = 0; List errors = []; List skipped = []; + // Resolved now, not at scan time: the paths it is compared with are resolved now too. + string? realRoot = RealPath.TryResolve(root.WeakString, out string resolvedRoot) ? resolvedRoot : null; + foreach (DuplicateGroup group in duplicateGroups.Where(IsDeletable)) { AbsoluteFilePath keeper = SelectFileToKeep(group.Files); @@ -208,6 +214,17 @@ internal static DeduplicationResult DeleteDuplicates(IReadOnlyList + /// Reports whether a recorded path still leads to a file inside the scanned root. + /// + /// The path recorded at scan time. + /// The scanned root with every link resolved, or if it could not be. + /// When the answer is no, why -- phrased to follow the file's name. + /// if the file's directory resolves to the root or beneath it. + private static bool LeadsInsideRoot(AbsoluteFilePath file, string? realRoot, out string? reason) + { + string directory = Path.GetDirectoryName(file.WeakString)!; + + if (realRoot is null || !RealPath.TryResolve(directory, out string realDirectory)) + { + reason = "could not be resolved to confirm it is still inside the scanned folder"; + return false; + } + + if (!RealPath.IsWithin(realDirectory, realRoot)) + { + reason = $"no longer leads inside the scanned folder: its directory now resolves to {realDirectory}"; + return false; + } + + reason = null; + return true; + } + /// /// Deletes one confirmed duplicate, recording a failure instead of letting it end the run. /// diff --git a/FileDeduplicator/RealPath.cs b/FileDeduplicator/RealPath.cs new file mode 100644 index 0000000..c17b830 --- /dev/null +++ b/FileDeduplicator/RealPath.cs @@ -0,0 +1,136 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.FileDeduplicator; + +using System.IO; + +/// +/// Resolves a path to the location it leads to now, following every symbolic link and junction +/// along it. +/// +/// +/// The scan records path strings, and a path string names whatever its directories lead to at the +/// moment it is used. A directory under the root that has since been replaced by a link sends a +/// recorded path somewhere else entirely, so before deleting through one the deletion path asks +/// where it actually leads. +/// +internal static class RealPath +{ + /// + /// The most links followed while resolving one path, the same bound Linux puts on a lookup. + /// + private const int MaxLinks = 40; + + private static readonly char[] Separators = [Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar]; + + /// + /// Gets the comparison that matches how this platform's default file system compares names. + /// + internal static StringComparison Comparison { get; } = + OperatingSystem.IsWindows() || OperatingSystem.IsMacOS() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + + /// + /// Resolves an absolute path, following every link along it. + /// + /// An absolute path. + /// The path with no links left in it, when it could be resolved. + /// + /// when a link could not be read, the links nest too deeply, or a + /// component could not be inspected. + /// + internal static bool TryResolve(string path, out string realPath) + { + int linksFollowed = 0; + string? resolved = Resolve(path, ref linksFollowed); + realPath = resolved ?? string.Empty; + return resolved is not null; + } + + /// + /// Reports whether is or lies beneath it, with + /// both already resolved by . + /// + /// The resolved path to check. + /// The resolved root. + /// if is inside . + internal static bool IsWithin(string path, string root) + { + string trimmedRoot = Path.TrimEndingDirectorySeparator(root); + string trimmedPath = Path.TrimEndingDirectorySeparator(path); + + return string.Equals(trimmedPath, trimmedRoot, Comparison) + || trimmedPath.StartsWith(trimmedRoot + Path.DirectorySeparatorChar, Comparison) + || (trimmedRoot.EndsWith(Path.DirectorySeparatorChar) && trimmedPath.StartsWith(trimmedRoot, Comparison)); + } + + private static string? Resolve(string path, ref int linksFollowed) + { + if (!Path.IsPathFullyQualified(path)) + { + return null; + } + + string current = Path.GetPathRoot(path)!; + + foreach (string part in path[current.Length..].Split(Separators, StringSplitOptions.RemoveEmptyEntries)) + { + if (part == ".") + { + continue; + } + + // current never holds a link, so going up from it lexically is going up for real. + if (part == "..") + { + current = Path.GetDirectoryName(current) ?? current; + continue; + } + + // part comes from splitting on separators, so it is never rooted; Join cannot re-root either way. + string next = Path.Join(current, part); + string? target; + + try + { + target = new FileInfo(next).LinkTarget; + } + catch (IOException) + { + return null; + } + catch (UnauthorizedAccessException) + { + return null; + } + + if (target is null) + { + current = next; + continue; + } + + if (++linksFollowed > MaxLinks) + { + return null; + } + + // A target rooted without being fully qualified (a Windows "\dir" or "C:dir") depends on a + // current drive or directory this process does not share with the link, so it is refused. + if (!Path.IsPathFullyQualified(target) && Path.IsPathRooted(target)) + { + return null; + } + + // A relative target is relative to the directory holding the link, which is current. + string? resolvedTarget = Resolve(Path.IsPathFullyQualified(target) ? target : Path.Join(current, target), ref linksFollowed); + if (resolvedTarget is null) + { + return null; + } + + current = resolvedTarget; + } + + return current; + } +} diff --git a/FileDeduplicator/Verbs/Deduplicate.cs b/FileDeduplicator/Verbs/Deduplicate.cs index 21bdc39..e75be29 100644 --- a/FileDeduplicator/Verbs/Deduplicate.cs +++ b/FileDeduplicator/Verbs/Deduplicate.cs @@ -69,7 +69,7 @@ internal override void Run(Deduplicate options) // Step 6: Delete duplicates Console.WriteLine("Deleting duplicates..."); - DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates); + DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates, options.Path); Console.WriteLine(); Console.WriteLine($"Deleted {result.DeletedCount} file(s)."); diff --git a/README.md b/README.md index 8c833e3..c8e784a 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,8 @@ One file reached at two paths is not a duplicate of itself. A hardlink, a bind m The copy being kept is checked again before each deletion. If a sync client or an editor rewrites it while the other copies are being deleted, the rest of the group is kept and reported as skipped, because each of them may now be the last copy of the content. After a group's last deletion the kept copy is re-hashed, and a change the quick check could not see is reported as an error. +Deletion never leaves the scanned folder. Each path is resolved, following every link along it, immediately before it is deleted. If a directory under the root has been replaced by a link to somewhere else since the scan, for example while the confirmation prompt was open, the file is kept and reported as skipped. + For example, given these duplicates: - `C:\photos\vacation\IMG_20240101_123456.jpg` - `C:\photos\beach.jpg`