From 022b380264da6cd65a21aea932d052634be84a14 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 07:31:43 +0000 Subject: [PATCH] Skip .git, .svn, .hg and .bzr directories when scanning [patch] Inside .git, every branch ref pointing at the same commit has identical content, as do the reflogs of branches created together. The scan walked into .git like any other folder, so Deduplicate grouped those files as duplicates and deleted all but one, deleting branches and their reflogs in every repository under the root. The walk now prunes version-control metadata directories by name, so their contents are never scanned, reported or deleted. A .git file (a worktree or submodule pointer) and directories such as .github are still scanned. The README notes the exclusion next to the empty-file rule. Fixes #192 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017vfJW34mg5yez39UfTV7yf --- .../VersionControlMetadataTests.cs | 114 ++++++++++++++++++ FileDeduplicator/FileScanner.cs | 30 ++++- README.md | 2 + 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 FileDeduplicator.Test/VersionControlMetadataTests.cs diff --git a/FileDeduplicator.Test/VersionControlMetadataTests.cs b/FileDeduplicator.Test/VersionControlMetadataTests.cs new file mode 100644 index 0000000..1f5a2a8 --- /dev/null +++ b/FileDeduplicator.Test/VersionControlMetadataTests.cs @@ -0,0 +1,114 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.FileDeduplicator.Test; + +using ktsu.FileDeduplicator.Verbs; +using ktsu.Semantics.Paths; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Tests that version-control metadata under the root is never scanned, reported or deleted. +/// +/// +/// Every branch ref in .git/refs/heads that points at the same commit has the same 41 bytes, +/// so a deduplicator that walks into .git deletes branches (ktsu-dev/FileDeduplicator#192). +/// +[TestClass] +[DoNotParallelize] +public sealed class VersionControlMetadataTests +{ + private const string BranchTip = "0123456789abcdef0123456789abcdef01234567\n"; + + [TestMethod] + [DataRow(".git")] + [DataRow(".svn")] + [DataRow(".hg")] + [DataRow(".bzr")] + [DataRow(".GIT")] + public void TheScanSkipsVersionControlMetadataDirectories(string metadata) + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath outside = tree.Write("proj/readme.txt", "content"); + _ = tree.Write($"proj/{metadata}/refs/heads/a", BranchTip); + _ = tree.Write($"proj/{metadata}/refs/heads/b", BranchTip); + + // Act + IReadOnlyList files = FileScanner.ScanForFiles(tree.Root); + + // Assert + CollectionAssert.AreEquivalent(new[] { outside }, files.ToArray()); + } + + [TestMethod] + public void TheScanStillReadsFilesWhoseNamesOnlyResembleMetadata() + { + // Arrange -- a worktree's ".git" is a file, and ".github" or "x.git" are ordinary directories + using TempTree tree = new(); + AbsoluteFilePath[] expected = + [ + tree.Write("worktree/.git", "gitdir: /elsewhere/.git/worktrees/w\n"), + tree.Write(".github/workflows/ci.yml", "on: push\n"), + tree.Write("mirror.git/HEAD", "ref: refs/heads/main\n"), + tree.Write(".gitignore", "bin/\n"), + ]; + + // Act + IReadOnlyList files = FileScanner.ScanForFiles(tree.Root); + + // Assert + CollectionAssert.AreEquivalent(expected, files.ToArray()); + } + + [TestMethod] + public void DeduplicateDeletesOnlyTheDuplicatesOutsideGitMetadata() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath[] refs = + [ + tree.Write("proj/.git/refs/heads/main", BranchTip), + tree.Write("proj/.git/refs/heads/feature", BranchTip), + tree.Write("proj/.git/refs/heads/release", BranchTip), + tree.Write("proj/.git/logs/HEAD", "reflog line\n"), + tree.Write("proj/.git/logs/refs/heads/main", "reflog line\n"), + ]; + AbsoluteFilePath keeper = tree.Write("proj/a.txt", "a real duplicate"); + AbsoluteFilePath copy = tree.Write("proj/copy-of-a.txt", "a real duplicate"); + + // Act + string output = ConsoleCapture.Normalize(ConsoleCapture.Run(new Deduplicate { PathString = tree.Root.WeakString }, "y")); + + // Assert + Assert.IsTrue(TempTree.Exists(keeper), $"The kept copy was deleted. Output was:\n{output}"); + Assert.IsFalse(TempTree.Exists(copy), $"The real duplicate outside .git was not deleted. Output was:\n{output}"); + foreach (AbsoluteFilePath file in refs) + { + Assert.IsTrue(TempTree.Exists(file), $"{file} was deleted. Output was:\n{output}"); + } + + Assert.DoesNotContain(".git", output, $"The run reported a path inside .git. Output was:\n{output}"); + } + + [TestMethod] + public void ScanAndDryRunDoNotReportGitMetadata() + { + // Arrange + using TempTree tree = new(); + _ = tree.Write("proj/.git/refs/heads/main", BranchTip); + _ = tree.Write("proj/.git/refs/heads/feature", BranchTip); + AbsoluteFilePath copy = tree.Write("proj/copy-of-a.txt", "a real duplicate"); + _ = tree.Write("proj/a.txt", "a real duplicate"); + + // Act + string scan = ConsoleCapture.Normalize(ConsoleCapture.Run(new Scan { PathString = tree.Root.WeakString }, string.Empty)); + string dryRun = ConsoleCapture.Normalize(ConsoleCapture.Run(new DryRun { PathString = tree.Root.WeakString }, string.Empty)); + + // Assert + Assert.Contains(copy.WeakString, scan, $"Scan missed the real duplicate. Output was:\n{scan}"); + Assert.Contains(copy.WeakString, dryRun, $"DryRun missed the real duplicate. Output was:\n{dryRun}"); + Assert.DoesNotContain(".git", scan, $"Scan reported a path inside .git. Output was:\n{scan}"); + Assert.DoesNotContain(".git", dryRun, $"DryRun reported a path inside .git. Output was:\n{dryRun}"); + } +} diff --git a/FileDeduplicator/FileScanner.cs b/FileDeduplicator/FileScanner.cs index ab49978..b84fe56 100644 --- a/FileDeduplicator/FileScanner.cs +++ b/FileDeduplicator/FileScanner.cs @@ -4,6 +4,7 @@ namespace ktsu.FileDeduplicator; using System.Collections.Generic; using System.IO; +using System.IO.Enumeration; using ktsu.Semantics.Paths; using ktsu.Semantics.Strings; @@ -42,6 +43,24 @@ internal static class FileScanner AttributesToSkip = FileAttributes.ReparsePoint, }; + /// + /// Version-control metadata directories the walk never descends into. + /// + /// + /// Inside .git, every branch ref pointing at the same commit has identical content, and so + /// do the reflogs of branches created together. Treating those as duplicates deleted branches to + /// reclaim a few hundred bytes (ktsu-dev/FileDeduplicator#192). Matched on the directory's own + /// name, ignoring case, so a file whose path merely contains ".git" is still scanned, and so is a + /// .git file (the pointer a worktree or submodule leaves), which is not a directory. + /// + private static readonly HashSet VersionControlDirectories = new(StringComparer.OrdinalIgnoreCase) + { + ".git", + ".svn", + ".hg", + ".bzr", + }; + /// /// Checks that is a directory that can be scanned, and says why in one /// line when it is not. @@ -75,8 +94,17 @@ internal static IReadOnlyList ScanForFiles(AbsoluteDirectoryPa return []; } + FileSystemEnumerable walk = new( + path.WeakString, + (ref entry) => entry.ToFullPath(), + WalkOptions) + { + ShouldIncludePredicate = (ref entry) => !entry.IsDirectory, + ShouldRecursePredicate = (ref entry) => !VersionControlDirectories.Contains(entry.FileName.ToString()), + }; + List files = []; - foreach (string file in Directory.EnumerateFiles(path.WeakString, "*", WalkOptions)) + foreach (string file in walk) { // On Unix the enumeration also yields named pipes, sockets and device nodes. Opening a // pipe to hash it blocks until a writer appears, and none of them is a copy of anything. diff --git a/README.md b/README.md index d133e20..6782ce3 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,8 @@ When duplicates are found, the file with the **shortest filename** is kept. If t Empty files are reported as a group but never deleted. Deleting one frees nothing, and an empty file such as `__init__.py`, `.gitkeep` or `py.typed` usually matters because it exists. +Version-control metadata is never scanned. The `.git`, `.svn`, `.hg` and `.bzr` directories under the root are skipped, because their small files are routinely identical by design: every branch ref pointing at the same commit has the same content, and deleting one deletes the branch. + One file reached at two paths is not a duplicate of itself. A hardlink, a bind mount, or a share mounted twice gives one file several paths, and deleting one of them either deletes the copy being kept or frees nothing. Such paths are counted as one file, and deletion never removes a path that leads to the kept file. For example, given these duplicates: