Replace the Git-based tarball cache by a SQLite database

src/libfetchers/git-utils.cc

@@ -1632,32 +1632,6 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
     return result;
 }
 
-namespace fetchers {
-
-ref<GitRepo> Settings::getTarballCache() const
-{
-    /* v1: Had either only loose objects or thin packfiles referring to loose objects
-     * v2: Must have only packfiles with no loose objects. Should get repacked periodically
-     * for optimal packfiles.
-     */
-    static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache-v2";
-    auto tarballCache(_tarballCache.lock());
-    if (!*tarballCache)
-        *tarballCache = GitRepo::openRepo(
-            repoDir,
-            {
-                .create = true,
-                .bare = true,
-                .packfilesOnly = true,
-                /* Tarball unpacking is not expected to benefit from deltas much,
-                   compared to how much CPU times it takes to find. */
-                .dontFindDeltas = true,
-            });
-    return ref<GitRepo>(*tarballCache);
-}
-
-} // namespace fetchers
-
 static Sync<std::map<std::filesystem::path, GitRepo::WorkdirInfo>> workdirInfoCache_;
 
 GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path)

src/libfetchers/github.cc

@@ -9,6 +9,7 @@
 #include "nix/fetchers/tarball.hh"
 #include "nix/util/tarfile.hh"
 #include "nix/fetchers/git-utils.hh"
+#include "nix/fetchers/tarball-cache.hh"
 
 #include <optional>
 #include <nlohmann/json.hpp>
@@ -287,7 +288,7 @@ struct GitArchiveInputScheme : InputScheme
             if (auto lastModifiedAttrs = cache->lookup(lastModifiedKey)) {
                 auto treeHash = getRevAttr(*treeHashAttrs, "treeHash");
                 auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified");
-                if (settings.getTarballCache()->hasObject(treeHash))
+                if (settings.getTarballCache()->hasTree(treeHash))
                     return {
                         {std::move(input), TarballInfo{.treeHash = treeHash, .lastModified = (time_t) lastModified}}};
                 else
@@ -308,7 +309,7 @@ struct GitArchiveInputScheme : InputScheme
         });
 
         auto act = std::make_unique<Activity>(
-            *logger, lvlInfo, actUnknown, fmt("unpacking '%s' into the Git cache", input.to_string()));
+            *logger, lvlInfo, actUnknown, fmt("unpacking '%s' into the tarball cache", input.to_string()));
 
         TarArchive archive{*source};
         auto tarballCache = settings.getTarballCache();
@@ -351,7 +352,7 @@ struct GitArchiveInputScheme : InputScheme
         input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified));
 
         auto accessor =
-            settings.getTarballCache()->getAccessor(tarballInfo.treeHash, {}, "«" + input.to_string(true) + "»");
+            settings.getTarballCache()->getAccessor(tarballInfo.treeHash, "«" + input.to_string(true) + "»");
 
         if (!settings.trustTarballsFromGitForges)
             // FIXME: computing the NAR hash here is wasteful if

src/libfetchers/include/nix/fetchers/fetch-settings.hh

@@ -13,14 +13,14 @@
 
 namespace nix {
 
-struct GitRepo;
 struct SrcToStore;
 
 } // namespace nix
 
 namespace nix::fetchers {
 
 struct Cache;
+struct TarballCache;
 
 struct Settings : public Config
 {
@@ -155,7 +155,7 @@ struct Settings : public Config
 
     ref<Cache> getCache() const;
 
-    ref<GitRepo> getTarballCache() const;
+    ref<TarballCache> getTarballCache() const;
 
     /**
      * In-memory cache for calls to fetchToStore(); maps source paths to their store
@@ -171,7 +171,7 @@ private:
 
     mutable Sync<std::shared_ptr<Cache>> _cache;
 
-    mutable Sync<std::shared_ptr<GitRepo>> _tarballCache;
+    mutable Sync<std::shared_ptr<TarballCache>> _tarballCache;
 };
 
 } // namespace nix::fetchers

src/libfetchers/include/nix/fetchers/meson.build

@@ -12,5 +12,6 @@ headers = files(
   'input-cache.hh',
   'provenance.hh',
   'registry.hh',
+  'tarball-cache.hh',
   'tarball.hh',
 )

src/libfetchers/include/nix/fetchers/tarball-cache.hh

@@ -0,0 +1,59 @@
+#pragma once
+///@file
+
+#include "nix/fetchers/git-utils.hh"
+#include "nix/util/hash.hh"
+#include "nix/util/ref.hh"
+#include "nix/util/source-accessor.hh"
+
+#include <filesystem>
+
+namespace nix::fetchers {
+
+struct Settings;
+
+/**
+ * A content-addressed cache of unpacked tarballs, stored in a SQLite
+ * database. Files and directories are identified by their Git (SHA-1)
+ * blob and tree hashes.
+ */
+struct TarballCache
+{
+    virtual ~TarballCache() = default;
+
+    /**
+     * Open (and create if necessary) the tarball cache stored in the
+     * SQLite database `dbPath`.
+     */
+    static ref<TarballCache> open(const std::filesystem::path & dbPath);
+
+    /**
+     * Whether the cache contains the tree `treeHash`. If so, then
+     * everything reachable from that tree is present as well.
+     */
+    virtual bool hasTree(const Hash & treeHash) = 0;
+
+    virtual ref<SourceAccessor> getAccessor(const Hash & treeHash, std::string displayPrefix) = 0;
+
+    /**
+     * Return a sink that imports a file system tree into the cache.
+     * Calling `flush()` on the sink returns the hash of the root
+     * tree.
+     */
+    virtual ref<GitFileSystemObjectSink> getFileSystemObjectSink() = 0;
+
+    /**
+     * Given a Git tree hash, compute the hash of its NAR
+     * serialisation. This is memoised on-disk.
+     */
+    virtual Hash treeHashToNarHash(const Settings & settings, const Hash & treeHash) = 0;
+
+    /**
+     * If the specified tree is a directory with a single entry that
+     * is a directory, return the hash of that entry. Otherwise return
+     * the passed hash unchanged.
+     */
+    virtual Hash dereferenceSingletonDirectory(const Hash & treeHash) = 0;
+};
+
+} // namespace nix::fetchers

src/libfetchers/meson.build

@@ -31,6 +31,9 @@ deps_public += nlohmann_json
 libgit2 = dependency('libgit2', version : '>= 1.9')
 deps_private += libgit2
 
+zstd = dependency('libzstd', version : '>= 1.4.0')
+deps_private += zstd
+
 subdir('nix-meson-build-support/common')
 
 sources = files(
@@ -51,6 +54,7 @@ sources = files(
   'path.cc',
   'provenance.cc',
   'registry.cc',
+  'tarball-cache.cc',
   'tarball.cc',
 )
 

src/libfetchers/package.nix

@@ -6,6 +6,7 @@
   nix-store,
   nlohmann_json,
   libgit2,
+  zstd,
 
   # Configuration Options
 
@@ -35,6 +36,7 @@ mkMesonLibrary (finalAttrs: {
 
   buildInputs = [
     libgit2
+    zstd
   ];
 
   propagatedBuildInputs = [