Skip .git, .svn, .hg and .bzr directories when scanning [patch]

FileDeduplicator.Test/VersionControlMetadataTests.cs

@@ -0,0 +1,114 @@
+// Copyright (c) 2023-2026 ktsu-dev contributors
+
+namespace ktsu.FileDeduplicator.Test;
+
+using ktsu.FileDeduplicator.Verbs;
+using ktsu.Semantics.Paths;
+
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+/// <summary>
+/// Tests that version-control metadata under the root is never scanned, reported or deleted.
+/// </summary>
+/// <remarks>
+/// Every branch ref in <c>.git/refs/heads</c> that points at the same commit has the same 41 bytes,
+/// so a deduplicator that walks into <c>.git</c> deletes branches (ktsu-dev/FileDeduplicator#192).
+/// </remarks>
+[TestClass]
+[DoNotParallelize]
+public sealed class VersionControlMetadataTests
+{
+	private const string BranchTip = "0123456789abcdef0123456789abcdef01234567\n";
+
+	[TestMethod]
+	[DataRow(".git")]
+	[DataRow(".svn")]
+	[DataRow(".hg")]
+	[DataRow(".bzr")]
+	[DataRow(".GIT")]
+	public void TheScanSkipsVersionControlMetadataDirectories(string metadata)
+	{
+		// Arrange
+		using TempTree tree = new();
+		AbsoluteFilePath outside = tree.Write("proj/readme.txt", "content");
+		_ = tree.Write($"proj/{metadata}/refs/heads/a", BranchTip);
+		_ = tree.Write($"proj/{metadata}/refs/heads/b", BranchTip);
+
+		// Act
+		IReadOnlyList<AbsoluteFilePath> files = FileScanner.ScanForFiles(tree.Root);
+
+		// Assert
+		CollectionAssert.AreEquivalent(new[] { outside }, files.ToArray());
+	}
+
+	[TestMethod]
+	public void TheScanStillReadsFilesWhoseNamesOnlyResembleMetadata()
+	{
+		// Arrange -- a worktree's ".git" is a file, and ".github" or "x.git" are ordinary directories
+		using TempTree tree = new();
+		AbsoluteFilePath[] expected =
+		[
+			tree.Write("worktree/.git", "gitdir: /elsewhere/.git/worktrees/w\n"),
+			tree.Write(".github/workflows/ci.yml", "on: push\n"),
+			tree.Write("mirror.git/HEAD", "ref: refs/heads/main\n"),
+			tree.Write(".gitignore", "bin/\n"),
+		];
+
+		// Act
+		IReadOnlyList<AbsoluteFilePath> files = FileScanner.ScanForFiles(tree.Root);
+
+		// Assert
+		CollectionAssert.AreEquivalent(expected, files.ToArray());
+	}
+
+	[TestMethod]
+	public void DeduplicateDeletesOnlyTheDuplicatesOutsideGitMetadata()
+	{
+		// Arrange
+		using TempTree tree = new();
+		AbsoluteFilePath[] refs =
+		[
+			tree.Write("proj/.git/refs/heads/main", BranchTip),
+			tree.Write("proj/.git/refs/heads/feature", BranchTip),
+			tree.Write("proj/.git/refs/heads/release", BranchTip),
+			tree.Write("proj/.git/logs/HEAD", "reflog line\n"),
+			tree.Write("proj/.git/logs/refs/heads/main", "reflog line\n"),
+		];
+		AbsoluteFilePath keeper = tree.Write("proj/a.txt", "a real duplicate");
+		AbsoluteFilePath copy = tree.Write("proj/copy-of-a.txt", "a real duplicate");
+
+		// Act
+		string output = ConsoleCapture.Normalize(ConsoleCapture.Run(new Deduplicate { PathString = tree.Root.WeakString }, "y"));
+
+		// Assert
+		Assert.IsTrue(TempTree.Exists(keeper), $"The kept copy was deleted. Output was:\n{output}");
+		Assert.IsFalse(TempTree.Exists(copy), $"The real duplicate outside .git was not deleted. Output was:\n{output}");
+		foreach (AbsoluteFilePath file in refs)
+		{
+			Assert.IsTrue(TempTree.Exists(file), $"{file} was deleted. Output was:\n{output}");
+		}
+
+		Assert.DoesNotContain(".git", output, $"The run reported a path inside .git. Output was:\n{output}");
+	}
+
+	[TestMethod]
+	public void ScanAndDryRunDoNotReportGitMetadata()
+	{
+		// Arrange
+		using TempTree tree = new();
+		_ = tree.Write("proj/.git/refs/heads/main", BranchTip);
+		_ = tree.Write("proj/.git/refs/heads/feature", BranchTip);
+		AbsoluteFilePath copy = tree.Write("proj/copy-of-a.txt", "a real duplicate");
+		_ = tree.Write("proj/a.txt", "a real duplicate");
+
+		// Act
+		string scan = ConsoleCapture.Normalize(ConsoleCapture.Run(new Scan { PathString = tree.Root.WeakString }, string.Empty));
+		string dryRun = ConsoleCapture.Normalize(ConsoleCapture.Run(new DryRun { PathString = tree.Root.WeakString }, string.Empty));
+
+		// Assert
+		Assert.Contains(copy.WeakString, scan, $"Scan missed the real duplicate. Output was:\n{scan}");
+		Assert.Contains(copy.WeakString, dryRun, $"DryRun missed the real duplicate. Output was:\n{dryRun}");
+		Assert.DoesNotContain(".git", scan, $"Scan reported a path inside .git. Output was:\n{scan}");
+		Assert.DoesNotContain(".git", dryRun, $"DryRun reported a path inside .git. Output was:\n{dryRun}");
+	}
+}

FileDeduplicator/FileScanner.cs

@@ -4,6 +4,7 @@ namespace ktsu.FileDeduplicator;
 
 using System.Collections.Generic;
 using System.IO;
+using System.IO.Enumeration;
 
 using ktsu.Semantics.Paths;
 using ktsu.Semantics.Strings;
@@ -42,6 +43,24 @@ internal static class FileScanner
 		AttributesToSkip = FileAttributes.ReparsePoint,
 	};
 
+	/// <summary>
+	/// Version-control metadata directories the walk never descends into.
+	/// </summary>
+	/// <remarks>
+	/// Inside <c>.git</c>, every branch ref pointing at the same commit has identical content, and so
+	/// do the reflogs of branches created together. Treating those as duplicates deleted branches to
+	/// reclaim a few hundred bytes (ktsu-dev/FileDeduplicator#192). Matched on the directory's own
+	/// name, ignoring case, so a file whose path merely contains ".git" is still scanned, and so is a
+	/// <c>.git</c> <em>file</em> (the pointer a worktree or submodule leaves), which is not a directory.
+	/// </remarks>
+	private static readonly HashSet<string> VersionControlDirectories = new(StringComparer.OrdinalIgnoreCase)
+	{
+		".git",
+		".svn",
+		".hg",
+		".bzr",
+	};
+
 	/// <summary>
 	/// Checks that <paramref name="path"/> is a directory that can be scanned, and says why in one
 	/// line when it is not.
@@ -75,8 +94,17 @@ internal static IReadOnlyList<AbsoluteFilePath> ScanForFiles(AbsoluteDirectoryPa
 			return [];
 		}
 
+		FileSystemEnumerable<string> walk = new(
+			path.WeakString,
+			(ref entry) => entry.ToFullPath(),
+			WalkOptions)
+		{
+			ShouldIncludePredicate = (ref entry) => !entry.IsDirectory,
+			ShouldRecursePredicate = (ref entry) => !VersionControlDirectories.Contains(entry.FileName.ToString()),
+		};
+
 		List<AbsoluteFilePath> files = [];
-		foreach (string file in Directory.EnumerateFiles(path.WeakString, "*", WalkOptions))
+		foreach (string file in walk)
 		{
 			// On Unix the enumeration also yields named pipes, sockets and device nodes. Opening a
 			// pipe to hash it blocks until a writer appears, and none of them is a copy of anything.

README.md

@@ -63,6 +63,8 @@ When duplicates are found, the file with the **shortest filename** is kept. If t
 
 Empty files are reported as a group but never deleted. Deleting one frees nothing, and an empty file such as `__init__.py`, `.gitkeep` or `py.typed` usually matters because it exists.
 
+Version-control metadata is never scanned. The `.git`, `.svn`, `.hg` and `.bzr` directories under the root are skipped, because their small files are routinely identical by design: every branch ref pointing at the same commit has the same content, and deleting one deletes the branch.
+
 One file reached at two paths is not a duplicate of itself. A hardlink, a bind mount, or a share mounted twice gives one file several paths, and deleting one of them either deletes the copy being kept or frees nothing. Such paths are counted as one file, and deletion never removes a path that leads to the kept file.
 
 For example, given these duplicates: