From 64827ee2c35f076ae89b2d805bbeda2c19f8825a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Oct 2026 16:43:05 +0200 Subject: [PATCH 01/13] nix nario export: Support binary diffs against a base closure `nix nario export --format 2 --base ...` assumes that the destination already has the closure of the given installables (the "base closure"). Paths in the base closure are not exported, and other paths are exported as binary diffs against a path in the base closure where possible. This makes it cheap to deploy closure updates, similar to the bsdiff-based patches that Nix supported in the manifest era (removed in 867967265b80946dfe1db72d40324b4f9af988ed). Diffs are computed using zstd's "patch-from" mode (i.e. the base NAR is used as a prefix dictionary, with long-distance matching and a window covering both NARs), using the libzstd we already link against. On import, the patched NAR is streamed into the store, so only the base NAR and the patch are held in memory. The base for a path is selected by matching the store path name without the version (and with the output name kept separate, so `openssl-*-dev` only matches `-dev`), skipping bases that differ in NAR size by more than a factor of 3. Among candidates, the one with the longest common version prefix wins, then the one closest in size. A diff is only used if it's smaller than 0.9 times the standalone zstd compression of the target NAR. NARs larger than 1 GiB are not diffed. Diffs are computed in parallel. Format: no version bump. NARIO2 entries are tagged; besides the existing tag 1 (full NAR), there are now: * Tag 2: diff entry: the target's ValidPathInfo (so narHash/narSize and signatures describe the reconstructed NAR), the base path, the base's NAR hash and the zstd patch. * Tag 3: the ValidPathInfo of a path that is expected to already be present at the destination. Import fails if it's missing or has a different NAR hash. Import also fails with a clear error if a diff base is missing or doesn't have the expected NAR hash. Older versions of Nix will reject narios containing these entries. Reading narios is now done by `parseNario()` with a `NarioVisitor`, which is used by both `importPaths()` and `nix nario list`. The latter shows diff and present entries (also in `--json`). Assisted-by: Claude Opus 5.5 --- src/libstore/export-import.cc | 402 +++++++++++++++--- .../include/nix/store/export-import.hh | 43 +- src/libutil-tests/meson.build | 1 + src/libutil-tests/zstd-patch.cc | 88 ++++ src/libutil/include/nix/util/meson.build | 1 + src/libutil/include/nix/util/zstd-patch.hh | 35 ++ src/libutil/meson.build | 1 + src/libutil/zstd-patch.cc | 85 ++++ src/nix/nario-export.md | 17 + src/nix/nario-list.md | 2 + src/nix/nario.cc | 128 ++++-- tests/functional/meson.build | 1 + tests/functional/nario-diff.nix | 32 ++ tests/functional/nario-diff.sh | 68 +++ 14 files changed, 828 insertions(+), 76 deletions(-) create mode 100644 src/libutil-tests/zstd-patch.cc create mode 100644 src/libutil/include/nix/util/zstd-patch.hh create mode 100644 src/libutil/zstd-patch.cc create mode 100644 tests/functional/nario-diff.nix create mode 100755 tests/functional/nario-diff.sh diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 8dec3fcd5080..f128b2edb3b7 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -5,12 +5,43 @@ #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" #include "nix/store/worker-protocol.hh" +#include "nix/store/worker-protocol-impl.hh" +#include "nix/store/names.hh" +#include "nix/util/thread-pool.hh" +#include "nix/util/zstd-patch.hh" + +#include namespace nix { static const uint32_t exportMagicV1 = 0x4558494e; static const uint64_t exportMagicV2 = 0x324f4952414e; // = 'NARIO2' +/* Entry tags in version 2 narios. */ +static const uint64_t narioTagEnd = 0; +static const uint64_t narioTagFull = 1; +static const uint64_t narioTagDiff = 2; +static const uint64_t narioTagPresent = 3; + +/* Binary diff generation parameters. */ + +/* NARs larger than this are not diffed. */ +static const uint64_t maxDiffNarSize = 1ULL << 30; + +/* If a base differs in NAR size from the target by more than this + factor, it's not considered as a diff base. This disambiguates + e.g. between a package and a wrapper of that package. */ +static const double maxDiffSizeRatio = 3.0; + +/* A diff is only used if it's smaller than this fraction of the + standalone zstd compression of the target NAR. Applying a zstd + patch is cheap, so even a modest saving is worth it. */ +static const double maxDiffFraction = 0.9; + +static const int diffCompressionLevel = 19; + +static const int baselineCompressionLevel = 3; + static WorkerProto::Version exportProtoVersion{ .number = { @@ -19,27 +50,174 @@ static WorkerProto::Version exportProtoVersion{ }, }; -void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigned int version) +static void checkNarHash(Store & store, const ValidPathInfo & info, const Hash & hash) +{ + /* Refuse to export paths that have changed. This prevents + filesystem corruption from spreading to other machines. + Don't complain if the stored hash is zero (unknown). */ + if (hash != info.narHash && info.narHash != Hash(info.narHash.algo)) + throw Error( + "hash of path '%s' has changed from '%s' to '%s'!", + store.printStorePath(info.path), + info.narHash.to_string(HashFormat::Nix32, true), + hash.to_string(HashFormat::Nix32, true)); +} + +/** + * Return the key used to match store paths against potential diff + * bases, namely the package name without the version, and the output + * name, e.g. `openssl-3.0.13-dev` yields (`openssl`, `dev`). + */ +static std::pair getDiffKey(std::string_view name) +{ + DrvName drvName(name); + std::string outputName; + auto i = drvName.version.rfind('-'); + if (i != std::string::npos && i + 1 < drvName.version.size() && isalpha(drvName.version[i + 1])) + outputName = drvName.version.substr(i + 1); + return {drvName.name, outputName}; +} + +/** + * Return the length of the common prefix of two version strings, as + * a crude measure of their similarity. + */ +static size_t versionSimilarity(std::string_view v1, std::string_view v2) +{ + return std::mismatch(v1.begin(), v1.end(), v2.begin(), v2.end()).first - v1.begin(); +} + +static double sizeRatio(uint64_t a, uint64_t b) +{ + if (a == 0 || b == 0) + return a == b ? 1.0 : std::numeric_limits::infinity(); + return a > b ? (double) a / b : (double) b / a; +} + +struct NarioDiff +{ + StorePath basePath; + Hash baseNarHash; + std::string patch; +}; + +/** + * Select the path in `baseClosure` that is most likely to produce a + * small diff against each path in `paths` that is not in + * `baseClosure`. + */ +static std::map +selectDiffBases(Store & store, const StorePaths & paths, const StorePathSet & baseClosure) +{ + std::map, std::vector> basesByKey; + for (auto & basePath : baseClosure) + basesByKey[getDiffKey(basePath.name())].push_back(basePath); + + std::map res; + + for (auto & path : paths) { + if (baseClosure.contains(path)) + continue; + + auto i = basesByKey.find(getDiffKey(path.name())); + if (i == basesByKey.end()) + continue; + + auto info = store.queryPathInfo(path); + if (info->narSize > maxDiffNarSize) + continue; + + auto version = DrvName(path.name()).version; + + std::optional bestPath; + size_t bestVersionSimilarity = 0; + double bestSizeRatio = 0; + + for (auto & basePath : i->second) { + auto baseInfo = store.queryPathInfo(basePath); + if (baseInfo->narSize > maxDiffNarSize || baseInfo->narSize + info->narSize > maxZstdPatchWindow()) + continue; + auto ratio = sizeRatio(info->narSize, baseInfo->narSize); + if (ratio >= maxDiffSizeRatio) + continue; + auto similarity = versionSimilarity(version, DrvName(basePath.name()).version); + if (!bestPath || similarity > bestVersionSimilarity + || (similarity == bestVersionSimilarity && ratio < bestSizeRatio)) { + bestPath = basePath; + bestVersionSimilarity = similarity; + bestSizeRatio = ratio; + } + } + + if (bestPath) + res.emplace(path, *bestPath); + } + + return res; +} + +/** + * Compute a binary diff between the NARs of `basePath` and `path`. + * Returns `std::nullopt` if the diff is not small enough to be worth + * it. + */ +static std::optional makeDiff(Store & store, const StorePath & path, const StorePath & basePath) +{ + Activity act( + *logger, + lvlTalkative, + actUnknown, + fmt("computing diff of '%s' against '%s'", store.printStorePath(path), store.printStorePath(basePath))); + + auto getNar = [&](const ValidPathInfo & info) { + StringSink nar; + store.narFromPath(info.path, nar); + auto hash = hashString(HashAlgorithm::SHA256, nar.s); + checkNarHash(store, info, hash); + return std::pair{std::move(nar.s), hash}; + }; + + auto [baseNar, baseNarHash] = getNar(*store.queryPathInfo(basePath)); + auto [nar, narHash] = getNar(*store.queryPathInfo(path)); + + auto patch = makeZstdPatch(baseNar, nar, diffCompressionLevel); + auto baselineSize = makeZstdPatch("", nar, baselineCompressionLevel).size(); + + printMsg( + lvlTalkative, + "diff of '%s' against '%s': %d bytes (NAR %d bytes, compressed %d bytes)", + store.printStorePath(path), + store.printStorePath(basePath), + patch.size(), + nar.size(), + baselineSize); + + if (patch.size() >= maxDiffFraction * baselineSize) + return std::nullopt; + + return NarioDiff{ + .basePath = basePath, + .baseNarHash = baseNarHash, + .patch = std::move(patch), + }; +} + +void exportPaths( + Store & store, const StorePathSet & paths, Sink & sink, unsigned int version, const StorePathSet & basePaths) { auto sorted = store.topoSortPaths(paths); std::reverse(sorted.begin(), sorted.end()); + if (!basePaths.empty() && version != 2) + throw Error("binary diffs are only supported in nario version 2"); + auto dumpNar = [&](const ValidPathInfo & info) { HashSink hashSink(HashAlgorithm::SHA256); TeeSink teeSink(sink, hashSink); store.narFromPath(info.path, teeSink); - /* Refuse to export paths that have changed. This prevents - filesystem corruption from spreading to other machines. - Don't complain if the stored hash is zero (unknown). */ - Hash hash = hashSink.currentHash().hash; - if (hash != info.narHash && info.narHash != Hash(info.narHash.algo)) - throw Error( - "hash of path '%s' has changed from '%s' to '%s'!", - store.printStorePath(info.path), - info.narHash.to_string(HashFormat::Nix32, true), - hash.to_string(HashFormat::Nix32, true)); + checkNarHash(store, info, hashSink.currentHash().hash); }; switch (version) { @@ -56,33 +234,68 @@ void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigne sink << 0; break; - case 2: + case 2: { + StorePathSet baseClosure; + std::map diffs; + + if (!basePaths.empty()) { + store.computeFSClosure(basePaths, baseClosure); + + auto diffBases = selectDiffBases(store, sorted, baseClosure); + + Sync> diffs_; + ThreadPool pool; + for (auto & [path, basePath] : diffBases) + pool.enqueue([&, path, basePath]() { + if (auto diff = makeDiff(store, path, basePath)) + diffs_.lock()->emplace(path, std::move(*diff)); + }); + pool.process(); + diffs = std::move(*diffs_.lock()); + } + + WorkerProto::WriteConn conn{.to = sink, .version = exportProtoVersion, .shortStorePaths = true}; + sink << exportMagicV2; for (auto & path : sorted) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("exporting path '%s'", store.printStorePath(path))); - sink << 1; auto info = store.queryPathInfo(path); + + if (baseClosure.contains(path)) { + sink << narioTagPresent; + WorkerProto::write(store, conn, *info); + continue; + } + + Activity act(*logger, lvlTalkative, actUnknown, fmt("exporting path '%s'", store.printStorePath(path))); + + if (auto diff = get(diffs, path)) { + sink << narioTagDiff; + WorkerProto::write(store, conn, *info); + WorkerProto::write(store, conn, diff->basePath); + sink << diff->baseNarHash.to_string(HashFormat::SRI, true) << diff->patch; + continue; + } + + sink << narioTagFull; // FIXME: move to CommonProto? - WorkerProto::Serialise::write( - store, - WorkerProto::WriteConn{.to = sink, .version = exportProtoVersion, .shortStorePaths = true}, - *info); + WorkerProto::write(store, conn, *info); dumpNar(*info); } - sink << 0; + sink << narioTagEnd; break; + } default: throw Error("unsupported nario version %d", version); } } -StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) -{ - StorePaths res; +NarioVisitor::~NarioVisitor() = default; +void parseNario(Store & store, Source & source, NarioVisitor & visitor) +{ auto version = readNum(source); /* Note: nario version 1 lacks an explicit header. The first @@ -113,8 +326,6 @@ StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) auto path = store.parseStorePath(readString(source)); - Activity act(*logger, lvlTalkative, actUnknown, fmt("importing path '%s'", store.printStorePath(path))); - auto references = CommonProto::Serialise::read(store, CommonProto::ReadConn{.from = source}); auto deriver = readString(source); @@ -122,21 +333,17 @@ StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) if (readInt(source) == 1) readString(source); - if (!store.isValidPath(path)) { - auto narHash = hashString(HashAlgorithm::SHA256, saved.s); + auto narHash = hashString(HashAlgorithm::SHA256, saved.s); - ValidPathInfo info{path, {store, narHash}}; - if (deriver != "") - info.deriver = store.parseStorePath(deriver); - info.references = references; - info.narSize = saved.s.size(); + ValidPathInfo info{path, {store, narHash}}; + if (deriver != "") + info.deriver = store.parseStorePath(deriver); + info.references = references; + info.narSize = saved.s.size(); - // Can't use underlying source, which would have been exhausted. - auto source2 = StringSource(saved.s); - store.addToStore(info, source2, NoRepair, checkSigs); - } - - res.push_back(path); + // Can't use underlying source, which would have been exhausted. + auto source2 = StringSource(saved.s); + visitor.fullPath(info, source2); auto n = readNum(source); if (n == 0) @@ -147,33 +354,130 @@ StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) break; } - case exportMagicV2: + case exportMagicV2: { + WorkerProto::ReadConn conn{.from = source, .version = exportProtoVersion, .shortStorePaths = true}; + while (true) { - auto n = readNum(source); - if (n == 0) + auto tag = readNum(source); + if (tag == narioTagEnd) break; - if (n != 1) + + if (tag != narioTagFull && tag != narioTagDiff && tag != narioTagPresent) throw Error("input doesn't look like a nario"); - auto info = WorkerProto::Serialise::read( - store, WorkerProto::ReadConn{.from = source, .version = exportProtoVersion, .shortStorePaths = true}); + auto info = WorkerProto::Serialise::read(store, conn); - Activity act( - *logger, lvlTalkative, actUnknown, fmt("importing path '%s'", store.printStorePath(info.path))); + if (tag == narioTagFull) { + EnsureRead wrapper{source, info.narSize}; + visitor.fullPath(info, wrapper); + } - EnsureRead wrapper{source, info.narSize}; - store.addToStore(info, wrapper, NoRepair, checkSigs); + else if (tag == narioTagDiff) { + auto basePath = WorkerProto::Serialise::read(store, conn); + auto baseNarHash = Hash::parseAnyPrefixed(readString(source)); + auto patch = readString(source); + visitor.diffPath(info, basePath, baseNarHash, patch); + } - res.push_back(info.path); + else + visitor.presentPath(info); } break; + } default: throw Error("input doesn't look like a nario"); } +} - return res; +StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) +{ + struct Importer : NarioVisitor + { + Store & store; + CheckSigsFlag checkSigs; + StorePaths res; + + Importer(Store & store, CheckSigsFlag checkSigs) + : store(store) + , checkSigs(checkSigs) + { + } + + void fullPath(const ValidPathInfo & info, Source & nar) override + { + if (!store.isValidPath(info.path)) { + Activity act( + *logger, lvlTalkative, actUnknown, fmt("importing path '%s'", store.printStorePath(info.path))); + + store.addToStore(info, nar, NoRepair, checkSigs); + } + + res.push_back(info.path); + } + + void diffPath( + const ValidPathInfo & info, + const StorePath & basePath, + const Hash & baseNarHash, + std::string_view patch) override + { + if (!store.isValidPath(info.path)) { + Activity act( + *logger, + lvlTalkative, + actUnknown, + fmt("importing path '%s' by patching '%s'", + store.printStorePath(info.path), + store.printStorePath(basePath))); + + if (!store.isValidPath(basePath)) + throw Error( + "cannot import path '%s' because it's a binary diff against path '%s', which is not valid", + store.printStorePath(info.path), + store.printStorePath(basePath)); + + StringSink baseNar; + store.narFromPath(basePath, baseNar); + + auto actualBaseNarHash = hashString(baseNarHash.algo, baseNar.s); + if (actualBaseNarHash != baseNarHash) + throw Error( + "cannot import path '%s' because it's a binary diff against path '%s', which has NAR hash '%s' instead of the expected '%s'", + store.printStorePath(info.path), + store.printStorePath(basePath), + actualBaseNarHash.to_string(HashFormat::SRI, true), + baseNarHash.to_string(HashFormat::SRI, true)); + + auto nar = sinkToSource([&](Sink & sink) { applyZstdPatch(baseNar.s, patch, sink); }); + store.addToStore(info, *nar, NoRepair, checkSigs); + } + + res.push_back(info.path); + } + + void presentPath(const ValidPathInfo & info) override + { + auto localInfo = store.maybeQueryPathInfo(info.path); + if (!localInfo) + throw Error( + "nario requires path '%s' to already be valid in the store", store.printStorePath(info.path)); + + if (localInfo->narHash != info.narHash && localInfo->narHash != Hash(localInfo->narHash.algo)) + throw Error( + "path '%s' in the store has NAR hash '%s', but the nario expects '%s'", + store.printStorePath(info.path), + localInfo->narHash.to_string(HashFormat::SRI, true), + info.narHash.to_string(HashFormat::SRI, true)); + + res.push_back(info.path); + } + }; + + Importer importer(store, checkSigs); + parseNario(store, source, importer); + return std::move(importer.res); } } // namespace nix diff --git a/src/libstore/include/nix/store/export-import.hh b/src/libstore/include/nix/store/export-import.hh index 4ea696f992f9..5d71ddcd529a 100644 --- a/src/libstore/include/nix/store/export-import.hh +++ b/src/libstore/include/nix/store/export-import.hh @@ -7,8 +7,49 @@ namespace nix { /** * Export multiple paths in the format expected by `nix-store * --import`. The paths will be sorted topologically. + * + * If `basePaths` is non-empty (only supported for version 2), the + * receiver is assumed to have the closure of `basePaths`. Paths in + * that closure are not exported (but recorded as being expected to + * be present), and other paths are exported as binary diffs against + * a path in that closure if a suitable one is found. */ -void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigned int version); +void exportPaths( + Store & store, const StorePathSet & paths, Sink & sink, unsigned int version, const StorePathSet & basePaths = {}); + +/** + * Callbacks for the entries in a nario, used by `parseNario()`. + */ +struct NarioVisitor +{ + virtual ~NarioVisitor(); + + /** + * A path whose NAR is contained in the nario. The visitor must + * read exactly `info.narSize` bytes from `nar`. + */ + virtual void fullPath(const ValidPathInfo & info, Source & nar) = 0; + + /** + * A path whose NAR is given as a binary diff (see + * `makeZstdPatch()`) against the NAR of `basePath`, which is + * expected to have NAR hash `baseNarHash`. + */ + virtual void diffPath( + const ValidPathInfo & info, const StorePath & basePath, const Hash & baseNarHash, std::string_view patch) = 0; + + /** + * A path that is not contained in the nario, but that is expected + * to already be present in the destination store. + */ + virtual void presentPath(const ValidPathInfo & info) = 0; +}; + +/** + * Parse a nario created by `exportPaths()`, calling `visitor` for + * each entry. + */ +void parseNario(Store & store, Source & source, NarioVisitor & visitor); /** * Import a sequence of NAR dumps created by `exportPaths()` into the diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index f7004c1c160d..f105c1cd6e1e 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -102,6 +102,7 @@ sources = files( 'url.cc', 'util.cc', 'xml-writer.cc', + 'zstd-patch.cc', ) if have_cxa_throw diff --git a/src/libutil-tests/zstd-patch.cc b/src/libutil-tests/zstd-patch.cc new file mode 100644 index 000000000000..9f41d58fa87e --- /dev/null +++ b/src/libutil-tests/zstd-patch.cc @@ -0,0 +1,88 @@ +#include "nix/util/zstd-patch.hh" +#include "nix/util/compression.hh" + +#include + +#include + +namespace nix { + +static std::string randomString(size_t size, uint64_t seed) +{ + std::mt19937_64 rng(seed); + std::string s(size, '\0'); + for (auto & c : s) + c = (char) rng(); + return s; +} + +static std::string applyPatch(std::string_view base, std::string_view patch) +{ + StringSink sink; + applyZstdPatch(base, patch, sink); + return std::move(sink.s); +} + +TEST(zstdPatch, roundtrip) +{ + auto base = randomString(1 << 20, 1); + auto target = base; + /* Make some small changes. */ + target.replace(1000, 5, "hello"); + target.insert(500000, "inserted"); + target.erase(800000, 100); + + auto patch = makeZstdPatch(base, target, 19); + ASSERT_EQ(applyPatch(base, patch), target); + + /* The base is random, so the patch is only small because it + refers to the base. */ + ASSERT_LT(patch.size(), 1000); +} + +TEST(zstdPatch, emptyBase) +{ + auto target = randomString(10000, 2); + auto patch = makeZstdPatch("", target, 3); + ASSERT_EQ(applyPatch("", patch), target); +} + +TEST(zstdPatch, emptyTarget) +{ + auto base = randomString(10000, 3); + auto patch = makeZstdPatch(base, "", 3); + ASSERT_EQ(applyPatch(base, patch), ""); +} + +TEST(zstdPatch, wrongBase) +{ + auto base = randomString(100000, 4); + auto target = base; + target.replace(50000, 5, "hello"); + auto patch = makeZstdPatch(base, target, 19); + + /* Applying to the wrong base produces garbage (or an error), so + callers must check the base's hash. */ + auto otherBase = randomString(100000, 5); + try { + ASSERT_NE(applyPatch(otherBase, patch), target); + } catch (CompressionError &) { + } +} + +TEST(zstdPatch, truncated) +{ + auto base = randomString(100000, 6); + auto target = randomString(100000, 7); + auto patch = makeZstdPatch(base, target, 3); + ASSERT_THROW(applyPatch(base, patch.substr(0, patch.size() / 2)), CompressionError); +} + +TEST(zstdPatch, trailingGarbage) +{ + auto base = randomString(1000, 8); + auto patch = makeZstdPatch(base, base, 3); + ASSERT_THROW(applyPatch(base, patch + "garbage"), CompressionError); +} + +} // namespace nix diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 1379fd8d2f2a..0cbb36fe6958 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -109,4 +109,5 @@ headers = [ config_pub_h ] + files( 'util.hh', 'variant-wrapper.hh', 'xml-writer.hh', + 'zstd-patch.hh', ) diff --git a/src/libutil/include/nix/util/zstd-patch.hh b/src/libutil/include/nix/util/zstd-patch.hh new file mode 100644 index 000000000000..3782c6a85028 --- /dev/null +++ b/src/libutil/include/nix/util/zstd-patch.hh @@ -0,0 +1,35 @@ +#pragma once +///@file + +#include "nix/util/serialise.hh" + +#include +#include + +namespace nix { + +/** + * Compute a binary patch that transforms `base` into `target`, using + * zstd's "patch-from" mode (i.e. `base` is used as a prefix + * dictionary). The result is a single zstd frame that can be + * decompressed by `applyZstdPatch()` given the same `base`. + * + * With an empty `base`, this is just ordinary zstd compression of + * `target`. + */ +std::string makeZstdPatch(std::string_view base, std::string_view target, int level); + +/** + * Apply a patch produced by `makeZstdPatch()` to `base`, writing the + * reconstructed target to `sink`. The output is streamed, so only + * `base` and `patch` need to be held in memory. + */ +void applyZstdPatch(std::string_view base, std::string_view patch, Sink & sink); + +/** + * The largest combined size of base and target that a zstd patch can + * fully cover (i.e. the maximum zstd window size). + */ +uint64_t maxZstdPatchWindow(); + +} // namespace nix diff --git a/src/libutil/meson.build b/src/libutil/meson.build index bf3a1f3b0833..dd6594a3a396 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -201,6 +201,7 @@ sources = [ config_priv_h ] + files( 'users.cc', 'util.cc', 'xml-writer.cc', + 'zstd-patch.cc', ) subdir('include/nix/util') diff --git a/src/libutil/zstd-patch.cc b/src/libutil/zstd-patch.cc new file mode 100644 index 000000000000..ca3d748dc5ff --- /dev/null +++ b/src/libutil/zstd-patch.cc @@ -0,0 +1,85 @@ +#include "nix/util/zstd-patch.hh" +#include "nix/util/compression.hh" +#include "nix/util/signals.hh" + +#include + +#include +#include + +namespace nix { + +static size_t checkZstd(size_t ret) +{ + if (ZSTD_isError(ret)) + throw CompressionError("zstd error: %s", ZSTD_getErrorName(ret)); + return ret; +} + +static int maxWindowLog() +{ + return ZSTD_cParam_getBounds(ZSTD_c_windowLog).upperBound; +} + +uint64_t maxZstdPatchWindow() +{ + return uint64_t(1) << maxWindowLog(); +} + +std::string makeZstdPatch(std::string_view base, std::string_view target, int level) +{ + std::unique_ptr cctx{ZSTD_createCCtx(), ZSTD_freeCCtx}; + if (!cctx) + throw CompressionError("unable to initialise zstd encoder"); + + checkZstd(ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_compressionLevel, level)); + + if (!base.empty()) { + /* The window must cover both the base (prefix) and the + target, otherwise the end of the target can't refer to the + start of the base. */ + uint64_t total = base.size() + target.size(); + int windowLog = std::bit_width(total - 1); + auto bounds = ZSTD_cParam_getBounds(ZSTD_c_windowLog); + windowLog = std::clamp(windowLog, bounds.lowerBound, bounds.upperBound); + checkZstd(ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_windowLog, windowLog)); + checkZstd(ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_enableLongDistanceMatching, 1)); + checkZstd(ZSTD_CCtx_refPrefix(cctx.get(), base.data(), base.size())); + } + + std::string out(ZSTD_compressBound(target.size()), '\0'); + auto n = checkZstd(ZSTD_compress2(cctx.get(), out.data(), out.size(), target.data(), target.size())); + out.resize(n); + return out; +} + +void applyZstdPatch(std::string_view base, std::string_view patch, Sink & sink) +{ + std::unique_ptr dctx{ZSTD_createDCtx(), ZSTD_freeDCtx}; + if (!dctx) + throw CompressionError("unable to initialise zstd decoder"); + + checkZstd(ZSTD_DCtx_setParameter(dctx.get(), ZSTD_d_windowLogMax, maxWindowLog())); + if (!base.empty()) + checkZstd(ZSTD_DCtx_refPrefix(dctx.get(), base.data(), base.size())); + + std::vector outbuf(ZSTD_DStreamOutSize()); + ZSTD_inBuffer in = {patch.data(), patch.size(), 0}; + + while (true) { + checkInterrupt(); + ZSTD_outBuffer out = {outbuf.data(), outbuf.size(), 0}; + auto ret = checkZstd(ZSTD_decompressStream(dctx.get(), &out, &in)); + if (out.pos > 0) + sink({outbuf.data(), out.pos}); + if (ret == 0) { + if (in.pos != in.size) + throw CompressionError("trailing garbage after zstd patch"); + break; + } + if (in.pos == in.size && out.pos < out.size) + throw CompressionError("truncated zstd patch"); + } +} + +} // namespace nix diff --git a/src/nix/nario-export.md b/src/nix/nario-export.md index 2480733c1cae..20f9a494435a 100644 --- a/src/nix/nario-export.md +++ b/src/nix/nario-export.md @@ -14,6 +14,13 @@ R""( # nix nario import --no-check-sigs < dump.nario ``` +* Export the closure of a new version of `hello`, assuming that the destination already has the closure of an old version: + + ```console + # nix nario export --format 2 -r /nix/store/6i6xl6bmcpxqd51m8nlva40d5c1bhndx-hello-2.12.3 \ + --base /nix/store/nm7p8wxflggcwxfzayhysq4z6a1wg373-hello-2.12.3 > update.nario + ``` + # Description This command prints to standard output a serialization of the specified store paths in `nario` format. This serialization can be imported into another store using `nix nario import`. @@ -26,4 +33,14 @@ You must specify the desired `nario` version. Currently the following versions a * `2`: The latest version. Recommended. +# Binary diffs + +The `--base` flag specifies one or more *installables* whose closure (the *base closure*) is assumed to be present in the store into which the nario will be imported. This can greatly reduce the size of the nario when deploying an update to a closure: + +* Paths that are in the base closure are not included in the nario. Instead, the nario records that they're expected to be present, and `nix nario import` fails if they're missing or have a different NAR hash. + +* For other paths, `nix nario export` looks for a path in the base closure with the same name (ignoring the version), e.g. `hello-2.12.2` for `hello-2.12.3`. If one exists, the path is stored as a binary diff (computed using zstd) against the NAR of that base path, provided that the diff is sufficiently small. `nix nario import` reconstructs the path by applying the diff to the base path, after verifying that the base path has the expected NAR hash. + +Binary diffs require nario format 2. Narios that contain binary diffs cannot be imported by versions of Nix that don't support them. + )"" diff --git a/src/nix/nario-list.md b/src/nix/nario-list.md index c050457b3657..d72bed817bfe 100644 --- a/src/nix/nario-list.md +++ b/src/nix/nario-list.md @@ -40,4 +40,6 @@ R""( This command lists the contents of a nario file read from standard input. +For narios created with `nix nario export --base`, paths that are stored as a binary diff are shown with the base path of the diff and the size of the diff. Their file contents are not shown by `--recursive`, since that would require the base path. Paths that are expected to already be present at the destination are shown as such. In the JSON output, these have a `diff` or `present` attribute, respectively. + )"" diff --git a/src/nix/nario.cc b/src/nix/nario.cc index 39b18d20403c..a326530691d9 100644 --- a/src/nix/nario.cc +++ b/src/nix/nario.cc @@ -35,6 +35,7 @@ static auto rCmdNario = registerCommand("nario"); struct CmdNarioExport : StorePathsCommand { unsigned int version = 0; + std::vector baseArgs; CmdNarioExport() { @@ -45,6 +46,15 @@ struct CmdNarioExport : StorePathsCommand .handler = {&version}, .required = true, }); + + addFlag({ + .longName = "base", + .description = + "Assume that the closure of *installable* is already present at the destination. Paths in this closure are not exported, and other paths are exported as binary diffs against paths in this closure where possible. Can be specified multiple times.", + .labels = {"installable"}, + .handler = {[this](std::string s) { baseArgs.push_back(s); }}, + .completer = getCompleteInstallable(), + }); } std::string description() override @@ -65,7 +75,13 @@ struct CmdNarioExport : StorePathsCommand if (isatty(fd)) throw UsageError("refusing to write nario to a terminal"); FdSink sink(std::move(fd)); - exportPaths(*store, StorePathSet(storePaths.begin(), storePaths.end()), sink, version); + + StorePathSet basePaths; + if (!baseArgs.empty()) + basePaths = Installable::toStorePathSet( + getEvalStore(), store, Realise::Outputs, operateOn, parseInstallables(store, baseArgs)); + + exportPaths(*store, StorePathSet(storePaths.begin(), storePaths.end()), sink, version, basePaths); } }; @@ -242,12 +258,8 @@ struct CmdNarioList : Command, MixJSON, MixLongListing struct ListingStore : Store { - std::optional json; - CmdNarioList & cmd; - - ListingStore(ref config, CmdNarioList & cmd) + ListingStore(ref config) : Store{*config} - , cmd(cmd) { } @@ -270,24 +282,7 @@ struct CmdNarioList : Command, MixJSON, MixLongListing void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override { - std::optional contents; - if (cmd.listContents) - contents = listNar(source); - else - source.skip(info.narSize); - - if (json) { - // FIXME: make the JSON format configurable. - auto obj = info.toJSON(this, true, PathInfoJsonFormat::V1); - if (contents) - obj.emplace("contents", *contents); - json->emplace(printStorePath(info.path), std::move(obj)); - } else { - if (contents) - renderNarListing(CanonPath(printStorePath(info.path)), *contents, cmd.longListing); - else - logger->cout(fmt("%s: %d bytes", printStorePath(info.path), info.narSize)); - } + unsupported("addToStore"); } StorePath addToStoreFromDump( @@ -332,12 +327,93 @@ struct CmdNarioList : Command, MixJSON, MixLongListing void anchor() override {} }; + struct Lister : NarioVisitor + { + Store & store; + CmdNarioList & cmd; + std::optional json; + + Lister(Store & store, CmdNarioList & cmd) + : store(store) + , cmd(cmd) + { + } + + void + add(const ValidPathInfo & info, + std::optional contents, + std::function extendJSON = {}) + { + if (!json) + return; + // FIXME: make the JSON format configurable. + auto obj = info.toJSON(&store, true, PathInfoJsonFormat::V1); + if (contents) + obj.emplace("contents", std::move(*contents)); + if (extendJSON) + extendJSON(obj); + json->emplace(store.printStorePath(info.path), std::move(obj)); + } + + void fullPath(const ValidPathInfo & info, Source & nar) override + { + std::optional contents; + if (cmd.listContents) + contents = listNar(nar); + else + nar.skip(info.narSize); + + if (!json) { + if (contents) + renderNarListing(CanonPath(store.printStorePath(info.path)), *contents, cmd.longListing); + else + logger->cout(fmt("%s: %d bytes", store.printStorePath(info.path), info.narSize)); + } + + add(info, std::move(contents)); + } + + void diffPath( + const ValidPathInfo & info, + const StorePath & basePath, + const Hash & baseNarHash, + std::string_view patch) override + { + if (!json) + logger->cout( + fmt("%s: %d bytes, diff against %s (%d bytes)", + store.printStorePath(info.path), + info.narSize, + store.printStorePath(basePath), + patch.size())); + + add(info, std::nullopt, [&](nlohmann::json & obj) { + obj.emplace( + "diff", + nlohmann::json{ + {"base", store.printStorePath(basePath)}, + {"baseNarHash", baseNarHash.to_string(HashFormat::SRI, true)}, + {"size", patch.size()}, + }); + }); + } + + void presentPath(const ValidPathInfo & info) override + { + if (!json) + logger->cout(fmt("%s: expected to be present", store.printStorePath(info.path))); + + add(info, std::nullopt, [](nlohmann::json & obj) { obj.emplace("present", true); }); + } + }; + auto source{getNarioSource()}; auto config = make_ref(StoreConfig::Params()); - ListingStore lister(config, *this); + ListingStore store(config); + Lister lister(store, *this); if (json) lister.json = nlohmann::json::object(); - importPaths(lister, source, NoCheckSigs); + parseNario(store, source, lister); if (json) { auto j = nlohmann::json::object(); j["version"] = 1; diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 1e43c200068d..014af671603e 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -132,6 +132,7 @@ suites = [ 'multiple-outputs-substitute-failure.sh', 'multiple-outputs.sh', 'nar-access.sh', + 'nario-diff.sh', 'nars.sh', 'nested-sandboxing.sh', 'nix-build.sh', diff --git a/tests/functional/nario-diff.nix b/tests/functional/nario-diff.nix new file mode 100644 index 000000000000..4c3ee01ed296 --- /dev/null +++ b/tests/functional/nario-diff.nix @@ -0,0 +1,32 @@ +{ + version, +}: +with import ./config.nix; + +let + + # A dependency that's identical in both versions. + shared = mkDerivation { + name = "nario-shared-1.0"; + buildCommand = "mkdir $out; echo shared > $out/shared"; + }; + + # A dependency that only exists in version 2, so it has no diff base. + extra = mkDerivation { + name = "nario-extra"; + buildCommand = "mkdir $out; seq 1 10000 > $out/data"; + }; + +in + +mkDerivation { + name = "nario-pkg-${version}"; + buildCommand = '' + mkdir $out + seq 1 100000 > $out/data + echo ${version} >> $out/data + echo $out >> $out/data + ln -s ${shared} $out/shared + ${if version == "2" then "ln -s ${extra} $out/extra" else ""} + ''; +} diff --git a/tests/functional/nario-diff.sh b/tests/functional/nario-diff.sh new file mode 100755 index 000000000000..90d8197467f1 --- /dev/null +++ b/tests/functional/nario-diff.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +clearStore + +old=$(nix-build nario-diff.nix --argstr version 1 --no-out-link) +new=$(nix-build nario-diff.nix --argstr version 2 --no-out-link) +shared=$(nix-store -qR "$new" | grep nario-shared) +extra=$(nix-store -qR "$new" | grep nario-extra) + +nix key generate-secret --key-name my-key > "$TEST_ROOT"/secret +public_key=$(nix key convert-secret-to-public < "$TEST_ROOT"/secret) +nix store sign --key-file "$TEST_ROOT/secret" -r "$new" + +# Binary diffs require format 2. +expectStderr 1 nix nario export --format 1 -r "$new" --base "$old" | grepQuiet "only supported in nario version 2" + +nix nario export --format 2 -r "$old" > "$TEST_ROOT"/old.nario +nix nario export --format 2 -r "$new" > "$TEST_ROOT"/full.nario +nix nario export --format 2 -r "$new" --base "$old" > "$TEST_ROOT"/diff.nario + +# The diff should be much smaller than the full export. +(( $(stat -c %s "$TEST_ROOT"/diff.nario) * 4 < $(stat -c %s "$TEST_ROOT"/full.nario) )) + +# Test `nix nario list`. +nix nario list < "$TEST_ROOT"/diff.nario | grepQuiet "^$new: [0-9]* bytes, diff against $old ([0-9]* bytes)$" +nix nario list < "$TEST_ROOT"/diff.nario | grepQuiet "^$shared: expected to be present$" +nix nario list < "$TEST_ROOT"/diff.nario | grepQuiet "^$extra: [0-9]* bytes$" +nix nario list -R < "$TEST_ROOT"/diff.nario | grepQuiet "^$extra/data$" + +json=$(nix nario list --json < "$TEST_ROOT/diff.nario") +[[ $(printf "%s" "$json" | jq -r ".paths.\"$new\".diff.base") = "$old" ]] +[[ $(printf "%s" "$json" | jq -r ".paths.\"$new\".diff.baseNarHash") = $(nix path-info --json "$old" | jq -r ".[].narHash") ]] +[[ $(printf "%s" "$json" | jq -r ".paths.\"$shared\".present") = true ]] +[[ $(printf "%s" "$json" | jq -r ".paths.\"$extra\".diff") = null ]] + +# Importing into an empty store fails because the base closure is missing. +clearStore +expectStderr 1 nix nario import --no-check-sigs < "$TEST_ROOT"/diff.nario | grepQuiet "nario requires path '$shared' to already be valid in the store" + +# Importing into a store that lacks the diff base fails. +clearStore +nix nario import --no-check-sigs < "$TEST_ROOT"/old.nario +nix-store --delete "$old" +expectStderr 1 nix nario import --no-check-sigs < "$TEST_ROOT"/diff.nario | grepQuiet "binary diff against path '$old', which is not valid" + +# Importing into a store that has the base closure succeeds, and preserves signatures. +clearStore +nix nario import --no-check-sigs < "$TEST_ROOT"/old.nario +expectStderr 1 nix nario import < "$TEST_ROOT"/diff.nario | grepQuiet "lacks a signature" +nix nario import --trusted-public-keys "$public_key" < "$TEST_ROOT"/diff.nario +[[ $(nix path-info --json "$new" | jq -r .[].signatures[]) =~ my-key: ]] +nix store verify --no-trust -r "$new" +grepQuiet "^2$" "$new/data" +[[ $(readlink "$new/extra") = "$extra" ]] + +# Importing again is a no-op. +nix nario import --no-check-sigs < "$TEST_ROOT"/diff.nario + +# Importing fails if the base has a different NAR hash. +clearStore +nix nario import --no-check-sigs < "$TEST_ROOT"/old.nario +chmod -R u+w "$old" +echo corrupt >> "$old/data" +expectStderr 1 nix nario import --no-check-sigs < "$TEST_ROOT"/diff.nario | grepQuiet "binary diff against path '$old', which has NAR hash '.*' instead of the expected" From ab2ebe05e2850ff070e0c059a6a4d54312bb2dff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Oct 2026 19:20:36 +0200 Subject: [PATCH 02/13] makeZstdPatch(): Use the multi-threaded zstd compressor At high compression levels, the single-threaded zstd compressor makes poor use of the prefix (base). For example, for two byte-identical 65 MB NARs (noto-fonts-cjk-sans), it produced a 13 MB patch at level 19, versus 6 KB with the multi-threaded compressor (which `zstd --patch-from` uses by default, since it defaults to 1 worker). Patch sizes for other NARs also improved somewhat (e.g. nodejs-slim 24.15.0 -> 24.16.0: 5.6 MB -> 4.8 MB), though a few got slightly bigger. So set ZSTD_c_nbWorkers to 1. More workers don't improve patch size, and we already compute diffs in parallel. Assisted-by: Claude Opus 5.5 --- src/libutil/zstd-patch.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libutil/zstd-patch.cc b/src/libutil/zstd-patch.cc index ca3d748dc5ff..9b135cfd63aa 100644 --- a/src/libutil/zstd-patch.cc +++ b/src/libutil/zstd-patch.cc @@ -44,6 +44,14 @@ std::string makeZstdPatch(std::string_view base, std::string_view target, int le windowLog = std::clamp(windowLog, bounds.lowerBound, bounds.upperBound); checkZstd(ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_windowLog, windowLog)); checkZstd(ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_enableLongDistanceMatching, 1)); + /* Use the multi-threaded compressor (even with a single + worker), since the single-threaded one makes poor use of + the prefix at high compression levels (e.g. a 13 MB patch + vs 6 KB between two identical 65 MB inputs at level 19). + Don't checkZstd(): if libzstd was built without + ZSTD_MULTITHREAD this returns an error, but compression + still works. */ + ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_nbWorkers, 1); checkZstd(ZSTD_CCtx_refPrefix(cctx.get(), base.data(), base.size())); } From f8f470bef4491a1fd8b14e305f8c0764024123eb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Oct 2026 19:48:02 +0200 Subject: [PATCH 03/13] nix nario export: Add --compression flag Binary diffs in narios are already zstd-compressed, while full NARs are not, so compressing a nario as a whole (as users would previously do) is wasteful. Instead, `nix nario export --compression ` now compresses the NARs of full entries. These are stored in a new NARIO2 entry type (tag 4): like tag 1, it has the ValidPathInfo (describing the uncompressed NAR), followed by the compression method (e.g. `zstd`) and the compressed NAR as a length-prefixed string. Since the length must be written first, the exporter buffers each compressed NAR in memory. The importer streams the compressed data through a decompressor, so it uses constant memory. Any compression method supported by Nix can be used; zstd uses the multi-threaded compressor. The default is still no compression, so that the output of `nix nario export` without `--base` remains readable by older versions of Nix. `NarioVisitor::fullPath()` now gets the compression method and size of compressed NARs, which `nix nario list` shows (and includes in its JSON output as `compression`). For example, for a NixOS system closure, `--compression zstd` reduced the nario from 19.7 GB to 7.15 GB (in 112 s, with a peak RSS of 1.6 GB). Assisted-by: Claude Opus 5.5 --- src/libstore/export-import.cc | 66 ++++++++++++++++--- .../include/nix/store/export-import.hh | 30 ++++++++- src/nix/nario-export.md | 10 +++ src/nix/nario-list.md | 2 + src/nix/nario.cc | 31 ++++++++- tests/functional/nario-diff.sh | 33 ++++++++++ 6 files changed, 158 insertions(+), 14 deletions(-) diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index f128b2edb3b7..5aaf6ac841bc 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -9,6 +9,7 @@ #include "nix/store/names.hh" #include "nix/util/thread-pool.hh" #include "nix/util/zstd-patch.hh" +#include "nix/util/compression.hh" #include @@ -22,6 +23,7 @@ static const uint64_t narioTagEnd = 0; static const uint64_t narioTagFull = 1; static const uint64_t narioTagDiff = 2; static const uint64_t narioTagPresent = 3; +static const uint64_t narioTagCompressed = 4; /* Binary diff generation parameters. */ @@ -203,7 +205,12 @@ static std::optional makeDiff(Store & store, const StorePath & path, } void exportPaths( - Store & store, const StorePathSet & paths, Sink & sink, unsigned int version, const StorePathSet & basePaths) + Store & store, + const StorePathSet & paths, + Sink & sink, + unsigned int version, + const StorePathSet & basePaths, + CompressionAlgo compression) { auto sorted = store.topoSortPaths(paths); std::reverse(sorted.begin(), sorted.end()); @@ -211,7 +218,10 @@ void exportPaths( if (!basePaths.empty() && version != 2) throw Error("binary diffs are only supported in nario version 2"); - auto dumpNar = [&](const ValidPathInfo & info) { + if (compression != CompressionAlgo::none && version != 2) + throw Error("compression is only supported in nario version 2"); + + auto dumpNar = [&](const ValidPathInfo & info, Sink & sink) { HashSink hashSink(HashAlgorithm::SHA256); TeeSink teeSink(sink, hashSink); @@ -226,7 +236,7 @@ void exportPaths( for (auto & path : sorted) { sink << 1; auto info = store.queryPathInfo(path); - dumpNar(*info); + dumpNar(*info, sink); sink << exportMagicV1 << store.printStorePath(path); CommonProto::write(store, CommonProto::WriteConn{.to = sink}, info->references); sink << (info->deriver ? store.printStorePath(*info->deriver) : "") << 0; @@ -277,10 +287,24 @@ void exportPaths( continue; } + if (compression != CompressionAlgo::none) { + /* We need to know the size of the compressed NAR + before writing it, so buffer it in memory. */ + StringSink compressed; + auto compressionSink = makeCompressionSink(compression, compressed, true); + dumpNar(*info, *compressionSink); + compressionSink->finish(); + + sink << narioTagCompressed; + WorkerProto::write(store, conn, *info); + sink << showCompressionAlgo(compression) << compressed.s; + continue; + } + sink << narioTagFull; // FIXME: move to CommonProto? WorkerProto::write(store, conn, *info); - dumpNar(*info); + dumpNar(*info, sink); } sink << narioTagEnd; @@ -343,7 +367,7 @@ void parseNario(Store & store, Source & source, NarioVisitor & visitor) // Can't use underlying source, which would have been exhausted. auto source2 = StringSource(saved.s); - visitor.fullPath(info, source2); + visitor.fullPath(info, source2, std::nullopt); auto n = readNum(source); if (n == 0) @@ -362,14 +386,40 @@ void parseNario(Store & store, Source & source, NarioVisitor & visitor) if (tag == narioTagEnd) break; - if (tag != narioTagFull && tag != narioTagDiff && tag != narioTagPresent) + if (tag != narioTagFull && tag != narioTagDiff && tag != narioTagPresent && tag != narioTagCompressed) throw Error("input doesn't look like a nario"); auto info = WorkerProto::Serialise::read(store, conn); if (tag == narioTagFull) { EnsureRead wrapper{source, info.narSize}; - visitor.fullPath(info, wrapper); + visitor.fullPath(info, wrapper, std::nullopt); + } + + else if (tag == narioTagCompressed) { + auto algo = parseCompressionAlgo(readString(source)); + auto size = readNum(source); + + /* Stream the compressed data through a decompressor, + keeping track of how much is left so we can skip + it if the visitor doesn't read the whole NAR. */ + uint64_t remaining = size; + { + auto nar = sinkToSource([&](Sink & sink) { + auto decompressionSink = makeDecompressionSink(algo, sink); + std::vector buf(65536); + while (remaining) { + auto n = std::min(remaining, buf.size()); + source(buf.data(), n); + remaining -= n; + (*decompressionSink)({buf.data(), n}); + } + decompressionSink->finish(); + }); + visitor.fullPath(info, *nar, NarioCompression{.algo = algo, .size = size}); + } + source.skip(remaining); + readPadding(size, source); } else if (tag == narioTagDiff) { @@ -405,7 +455,7 @@ StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) { } - void fullPath(const ValidPathInfo & info, Source & nar) override + void fullPath(const ValidPathInfo & info, Source & nar, std::optional compression) override { if (!store.isValidPath(info.path)) { Activity act( diff --git a/src/libstore/include/nix/store/export-import.hh b/src/libstore/include/nix/store/export-import.hh index 5d71ddcd529a..7b56a58e0542 100644 --- a/src/libstore/include/nix/store/export-import.hh +++ b/src/libstore/include/nix/store/export-import.hh @@ -1,6 +1,7 @@ #pragma once #include "nix/store/store-api.hh" +#include "nix/util/compression-algo.hh" namespace nix { @@ -13,9 +14,30 @@ namespace nix { * that closure are not exported (but recorded as being expected to * be present), and other paths are exported as binary diffs against * a path in that closure if a suitable one is found. + * + * If `compression` is not `none` (only supported for version 2), + * NARs that are not exported as binary diffs are compressed. */ void exportPaths( - Store & store, const StorePathSet & paths, Sink & sink, unsigned int version, const StorePathSet & basePaths = {}); + Store & store, + const StorePathSet & paths, + Sink & sink, + unsigned int version, + const StorePathSet & basePaths = {}, + CompressionAlgo compression = CompressionAlgo::none); + +/** + * How a NAR is compressed inside a nario. + */ +struct NarioCompression +{ + CompressionAlgo algo; + + /** + * Size of the compressed NAR. + */ + uint64_t size; +}; /** * Callbacks for the entries in a nario, used by `parseNario()`. @@ -26,9 +48,11 @@ struct NarioVisitor /** * A path whose NAR is contained in the nario. The visitor must - * read exactly `info.narSize` bytes from `nar`. + * read exactly `info.narSize` bytes from `nar`. If the NAR is + * stored in compressed form, `compression` describes how; `nar` + * returns the decompressed NAR in any case. */ - virtual void fullPath(const ValidPathInfo & info, Source & nar) = 0; + virtual void fullPath(const ValidPathInfo & info, Source & nar, std::optional compression) = 0; /** * A path whose NAR is given as a binary diff (see diff --git a/src/nix/nario-export.md b/src/nix/nario-export.md index 20f9a494435a..bbf7ceb0e46d 100644 --- a/src/nix/nario-export.md +++ b/src/nix/nario-export.md @@ -43,4 +43,14 @@ The `--base` flag specifies one or more *installables* whose closure (the *base Binary diffs require nario format 2. Narios that contain binary diffs cannot be imported by versions of Nix that don't support them. +# Compression + +By default, the NARs in a nario are not compressed. The `--compression` flag specifies a compression method (such as `zstd` or `xz`) for NARs that are not exported as binary diffs. (Binary diffs are always compressed.) This is preferable to compressing the nario as a whole, since compressing binary diffs a second time is a waste of time. For example: + +```console +# nix nario export --format 2 --compression zstd -r /nix/store/6i6xl6bmcpxqd51m8nlva40d5c1bhndx-hello-2.12.3 > hello.nario +``` + +Compression requires nario format 2. Narios that contain compressed NARs cannot be imported by versions of Nix that don't support them. + )"" diff --git a/src/nix/nario-list.md b/src/nix/nario-list.md index d72bed817bfe..432d77f06591 100644 --- a/src/nix/nario-list.md +++ b/src/nix/nario-list.md @@ -42,4 +42,6 @@ This command lists the contents of a nario file read from standard input. For narios created with `nix nario export --base`, paths that are stored as a binary diff are shown with the base path of the diff and the size of the diff. Their file contents are not shown by `--recursive`, since that would require the base path. Paths that are expected to already be present at the destination are shown as such. In the JSON output, these have a `diff` or `present` attribute, respectively. +For narios created with `nix nario export --compression`, the compression method and compressed size of each NAR are shown. In the JSON output, these are in the `compression` attribute. + )"" diff --git a/src/nix/nario.cc b/src/nix/nario.cc index a326530691d9..d74a8a9c404d 100644 --- a/src/nix/nario.cc +++ b/src/nix/nario.cc @@ -36,6 +36,7 @@ struct CmdNarioExport : StorePathsCommand { unsigned int version = 0; std::vector baseArgs; + CompressionAlgo compression = CompressionAlgo::none; CmdNarioExport() { @@ -55,6 +56,14 @@ struct CmdNarioExport : StorePathsCommand .handler = {[this](std::string s) { baseArgs.push_back(s); }}, .completer = getCompleteInstallable(), }); + + addFlag({ + .longName = "compression", + .description = + "Compression method (e.g. `zstd`) to use for NARs that are not exported as binary diffs. The default is `none`. Requires nario format 2.", + .labels = {"method"}, + .handler = {[this](std::string s) { compression = parseCompressionAlgo(s, true); }}, + }); } std::string description() override @@ -81,7 +90,7 @@ struct CmdNarioExport : StorePathsCommand basePaths = Installable::toStorePathSet( getEvalStore(), store, Realise::Outputs, operateOn, parseInstallables(store, baseArgs)); - exportPaths(*store, StorePathSet(storePaths.begin(), storePaths.end()), sink, version, basePaths); + exportPaths(*store, StorePathSet(storePaths.begin(), storePaths.end()), sink, version, basePaths, compression); } }; @@ -355,7 +364,8 @@ struct CmdNarioList : Command, MixJSON, MixLongListing json->emplace(store.printStorePath(info.path), std::move(obj)); } - void fullPath(const ValidPathInfo & info, Source & nar) override + void + fullPath(const ValidPathInfo & info, Source & nar, std::optional compression) override { std::optional contents; if (cmd.listContents) @@ -366,11 +376,26 @@ struct CmdNarioList : Command, MixJSON, MixLongListing if (!json) { if (contents) renderNarListing(CanonPath(store.printStorePath(info.path)), *contents, cmd.longListing); + else if (compression) + logger->cout( + fmt("%s: %d bytes, %s-compressed (%d bytes)", + store.printStorePath(info.path), + info.narSize, + showCompressionAlgo(compression->algo), + compression->size)); else logger->cout(fmt("%s: %d bytes", store.printStorePath(info.path), info.narSize)); } - add(info, std::move(contents)); + add(info, std::move(contents), [&](nlohmann::json & obj) { + if (compression) + obj.emplace( + "compression", + nlohmann::json{ + {"method", showCompressionAlgo(compression->algo)}, + {"size", compression->size}, + }); + }); } void diffPath( diff --git a/tests/functional/nario-diff.sh b/tests/functional/nario-diff.sh index 90d8197467f1..0c13b153d3ef 100755 --- a/tests/functional/nario-diff.sh +++ b/tests/functional/nario-diff.sh @@ -25,6 +25,26 @@ nix nario export --format 2 -r "$new" --base "$old" > "$TEST_ROOT"/diff.nario # The diff should be much smaller than the full export. (( $(stat -c %s "$TEST_ROOT"/diff.nario) * 4 < $(stat -c %s "$TEST_ROOT"/full.nario) )) +# Test compression of full NARs. +expectStderr 1 nix nario export --format 1 --compression zstd -r "$new" | grepQuiet "compression is only supported in nario version 2" +expectStderr 1 nix nario export --format 2 --compression foo -r "$new" | grepQuiet "unknown compression method 'foo'" + +for method in zstd xz; do + nix nario export --format 2 --compression "$method" -r "$new" > "$TEST_ROOT/full-$method.nario" + (( $(stat -c %s "$TEST_ROOT/full-$method.nario") < $(stat -c %s "$TEST_ROOT"/full.nario) )) + nix nario list < "$TEST_ROOT/full-$method.nario" | grepQuiet "^$extra: [0-9]* bytes, $method-compressed ([0-9]* bytes)$" + nix nario list -R < "$TEST_ROOT/full-$method.nario" | grepQuiet "^$extra/data$" + nix nario list -R < "$TEST_ROOT/full-$method.nario" | grepQuiet "^$new/data$" + json=$(nix nario list --json < "$TEST_ROOT/full-$method.nario") + [[ $(printf "%s" "$json" | jq -r ".paths.\"$new\".compression.method") = "$method" ]] + (( $(printf "%s" "$json" | jq -r ".paths.\"$new\".compression.size") < $(printf "%s" "$json" | jq -r ".paths.\"$new\".narSize") )) +done + +nix nario export --format 2 --compression zstd -r "$new" --base "$old" > "$TEST_ROOT"/diff-zstd.nario +(( $(stat -c %s "$TEST_ROOT"/diff-zstd.nario) < $(stat -c %s "$TEST_ROOT"/diff.nario) )) +nix nario list < "$TEST_ROOT"/diff-zstd.nario | grepQuiet "^$new: [0-9]* bytes, diff against $old ([0-9]* bytes)$" +nix nario list < "$TEST_ROOT"/diff-zstd.nario | grepQuiet "^$extra: [0-9]* bytes, zstd-compressed ([0-9]* bytes)$" + # Test `nix nario list`. nix nario list < "$TEST_ROOT"/diff.nario | grepQuiet "^$new: [0-9]* bytes, diff against $old ([0-9]* bytes)$" nix nario list < "$TEST_ROOT"/diff.nario | grepQuiet "^$shared: expected to be present$" @@ -60,6 +80,19 @@ grepQuiet "^2$" "$new/data" # Importing again is a no-op. nix nario import --no-check-sigs < "$TEST_ROOT"/diff.nario +# Import compressed narios. +for method in zstd xz; do + clearStore + nix nario import --no-check-sigs < "$TEST_ROOT/full-$method.nario" + nix store verify --no-trust -r "$new" + [[ $(nix path-info --json "$new" | jq -r .[].signatures[]) =~ my-key: ]] +done + +clearStore +nix nario import --no-check-sigs < "$TEST_ROOT"/old.nario +nix nario import --no-check-sigs < "$TEST_ROOT"/diff-zstd.nario +nix store verify --no-trust -r "$new" + # Importing fails if the base has a different NAR hash. clearStore nix nario import --no-check-sigs < "$TEST_ROOT"/old.nario From dbaaf35a562f1481ab44dc5614cfca6a88fae397 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 8 Oct 2026 12:03:30 +0200 Subject: [PATCH 04/13] nario: Record the diff algorithm in diff entries Diff entries (tag 2) now contain the name of the diff algorithm, directly after the ValidPathInfo (like the compression method in tag 4 entries). This allows other diff algorithms to be added in the future. The current algorithm (zstd with the base NAR as a prefix dictionary) is called `zstd`. Unknown algorithms are rejected when parsing. `nix nario list` shows the algorithm, and includes it in the JSON output as `diff.algorithm`. Assisted-by: Claude Opus 5.5 --- src/libstore/export-import.cc | 31 +++++++++++++++++-- .../include/nix/store/export-import.hh | 27 +++++++++++++--- src/nix/nario-list.md | 2 +- src/nix/nario.cc | 5 ++- tests/functional/nario-diff.sh | 5 +-- 5 files changed, 60 insertions(+), 10 deletions(-) diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 5aaf6ac841bc..5a51eae02559 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -96,8 +96,25 @@ static double sizeRatio(uint64_t a, uint64_t b) return a > b ? (double) a / b : (double) b / a; } +std::string showNarioDiffAlgo(NarioDiffAlgo algo) +{ + switch (algo) { + case NarioDiffAlgo::zstd: + return "zstd"; + } + unreachable(); +} + +NarioDiffAlgo parseNarioDiffAlgo(std::string_view s) +{ + if (s == "zstd") + return NarioDiffAlgo::zstd; + throw Error("unsupported nario diff algorithm '%s'", s); +} + struct NarioDiff { + NarioDiffAlgo algo; StorePath basePath; Hash baseNarHash; std::string patch; @@ -198,6 +215,7 @@ static std::optional makeDiff(Store & store, const StorePath & path, return std::nullopt; return NarioDiff{ + .algo = NarioDiffAlgo::zstd, .basePath = basePath, .baseNarHash = baseNarHash, .patch = std::move(patch), @@ -282,6 +300,7 @@ void exportPaths( if (auto diff = get(diffs, path)) { sink << narioTagDiff; WorkerProto::write(store, conn, *info); + sink << showNarioDiffAlgo(diff->algo); WorkerProto::write(store, conn, diff->basePath); sink << diff->baseNarHash.to_string(HashFormat::SRI, true) << diff->patch; continue; @@ -423,10 +442,11 @@ void parseNario(Store & store, Source & source, NarioVisitor & visitor) } else if (tag == narioTagDiff) { + auto algo = parseNarioDiffAlgo(readString(source)); auto basePath = WorkerProto::Serialise::read(store, conn); auto baseNarHash = Hash::parseAnyPrefixed(readString(source)); auto patch = readString(source); - visitor.diffPath(info, basePath, baseNarHash, patch); + visitor.diffPath(info, algo, basePath, baseNarHash, patch); } else @@ -469,6 +489,7 @@ StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) void diffPath( const ValidPathInfo & info, + NarioDiffAlgo algo, const StorePath & basePath, const Hash & baseNarHash, std::string_view patch) override @@ -500,7 +521,13 @@ StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) actualBaseNarHash.to_string(HashFormat::SRI, true), baseNarHash.to_string(HashFormat::SRI, true)); - auto nar = sinkToSource([&](Sink & sink) { applyZstdPatch(baseNar.s, patch, sink); }); + auto nar = sinkToSource([&](Sink & sink) { + switch (algo) { + case NarioDiffAlgo::zstd: + applyZstdPatch(baseNar.s, patch, sink); + break; + } + }); store.addToStore(info, *nar, NoRepair, checkSigs); } diff --git a/src/libstore/include/nix/store/export-import.hh b/src/libstore/include/nix/store/export-import.hh index 7b56a58e0542..cd157e723b86 100644 --- a/src/libstore/include/nix/store/export-import.hh +++ b/src/libstore/include/nix/store/export-import.hh @@ -39,6 +39,21 @@ struct NarioCompression uint64_t size; }; +/** + * Binary diff algorithms supported in narios. + */ +enum class NarioDiffAlgo { + /** + * zstd compression of the target NAR, using the base NAR as a + * prefix dictionary (see `makeZstdPatch()`). + */ + zstd, +}; + +std::string showNarioDiffAlgo(NarioDiffAlgo algo); + +NarioDiffAlgo parseNarioDiffAlgo(std::string_view s); + /** * Callbacks for the entries in a nario, used by `parseNario()`. */ @@ -55,12 +70,16 @@ struct NarioVisitor virtual void fullPath(const ValidPathInfo & info, Source & nar, std::optional compression) = 0; /** - * A path whose NAR is given as a binary diff (see - * `makeZstdPatch()`) against the NAR of `basePath`, which is - * expected to have NAR hash `baseNarHash`. + * A path whose NAR is given as a binary diff (computed using + * `algo`) against the NAR of `basePath`, which is expected to + * have NAR hash `baseNarHash`. */ virtual void diffPath( - const ValidPathInfo & info, const StorePath & basePath, const Hash & baseNarHash, std::string_view patch) = 0; + const ValidPathInfo & info, + NarioDiffAlgo algo, + const StorePath & basePath, + const Hash & baseNarHash, + std::string_view patch) = 0; /** * A path that is not contained in the nario, but that is expected diff --git a/src/nix/nario-list.md b/src/nix/nario-list.md index 432d77f06591..d85f172485aa 100644 --- a/src/nix/nario-list.md +++ b/src/nix/nario-list.md @@ -40,7 +40,7 @@ R""( This command lists the contents of a nario file read from standard input. -For narios created with `nix nario export --base`, paths that are stored as a binary diff are shown with the base path of the diff and the size of the diff. Their file contents are not shown by `--recursive`, since that would require the base path. Paths that are expected to already be present at the destination are shown as such. In the JSON output, these have a `diff` or `present` attribute, respectively. +For narios created with `nix nario export --base`, paths that are stored as a binary diff are shown with the diff algorithm (currently always `zstd`), the base path of the diff and the size of the diff. Their file contents are not shown by `--recursive`, since that would require the base path. Paths that are expected to already be present at the destination are shown as such. In the JSON output, these have a `diff` or `present` attribute, respectively. For narios created with `nix nario export --compression`, the compression method and compressed size of each NAR are shown. In the JSON output, these are in the `compression` attribute. diff --git a/src/nix/nario.cc b/src/nix/nario.cc index d74a8a9c404d..1c1b1ef33968 100644 --- a/src/nix/nario.cc +++ b/src/nix/nario.cc @@ -400,15 +400,17 @@ struct CmdNarioList : Command, MixJSON, MixLongListing void diffPath( const ValidPathInfo & info, + NarioDiffAlgo algo, const StorePath & basePath, const Hash & baseNarHash, std::string_view patch) override { if (!json) logger->cout( - fmt("%s: %d bytes, diff against %s (%d bytes)", + fmt("%s: %d bytes, %s diff against %s (%d bytes)", store.printStorePath(info.path), info.narSize, + showNarioDiffAlgo(algo), store.printStorePath(basePath), patch.size())); @@ -416,6 +418,7 @@ struct CmdNarioList : Command, MixJSON, MixLongListing obj.emplace( "diff", nlohmann::json{ + {"algorithm", showNarioDiffAlgo(algo)}, {"base", store.printStorePath(basePath)}, {"baseNarHash", baseNarHash.to_string(HashFormat::SRI, true)}, {"size", patch.size()}, diff --git a/tests/functional/nario-diff.sh b/tests/functional/nario-diff.sh index 0c13b153d3ef..284e16145ccf 100755 --- a/tests/functional/nario-diff.sh +++ b/tests/functional/nario-diff.sh @@ -42,16 +42,17 @@ done nix nario export --format 2 --compression zstd -r "$new" --base "$old" > "$TEST_ROOT"/diff-zstd.nario (( $(stat -c %s "$TEST_ROOT"/diff-zstd.nario) < $(stat -c %s "$TEST_ROOT"/diff.nario) )) -nix nario list < "$TEST_ROOT"/diff-zstd.nario | grepQuiet "^$new: [0-9]* bytes, diff against $old ([0-9]* bytes)$" +nix nario list < "$TEST_ROOT"/diff-zstd.nario | grepQuiet "^$new: [0-9]* bytes, zstd diff against $old ([0-9]* bytes)$" nix nario list < "$TEST_ROOT"/diff-zstd.nario | grepQuiet "^$extra: [0-9]* bytes, zstd-compressed ([0-9]* bytes)$" # Test `nix nario list`. -nix nario list < "$TEST_ROOT"/diff.nario | grepQuiet "^$new: [0-9]* bytes, diff against $old ([0-9]* bytes)$" +nix nario list < "$TEST_ROOT"/diff.nario | grepQuiet "^$new: [0-9]* bytes, zstd diff against $old ([0-9]* bytes)$" nix nario list < "$TEST_ROOT"/diff.nario | grepQuiet "^$shared: expected to be present$" nix nario list < "$TEST_ROOT"/diff.nario | grepQuiet "^$extra: [0-9]* bytes$" nix nario list -R < "$TEST_ROOT"/diff.nario | grepQuiet "^$extra/data$" json=$(nix nario list --json < "$TEST_ROOT/diff.nario") +[[ $(printf "%s" "$json" | jq -r ".paths.\"$new\".diff.algorithm") = zstd ]] [[ $(printf "%s" "$json" | jq -r ".paths.\"$new\".diff.base") = "$old" ]] [[ $(printf "%s" "$json" | jq -r ".paths.\"$new\".diff.baseNarHash") = $(nix path-info --json "$old" | jq -r ".[].narHash") ]] [[ $(printf "%s" "$json" | jq -r ".paths.\"$shared\".present") = true ]] From 1be0ecd207b3c881e0893735a3f2948bcec294a5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 8 Oct 2026 12:27:57 +0200 Subject: [PATCH 05/13] nix nario export: Add --base-selection-method flag This prepares for adding other methods for selecting the base path against which to diff a path (e.g. based on content similarity). The only method for now is `by-name` (the default), which is the existing heuristic of matching store path names without the version. selectDiffBases() now applies the constraints that are independent of the selection method (the NAR size limit, and base + target fitting in the zstd window) and dispatches to selectDiffBasesByName() for the actual matching. Also, exportPaths() now takes a NarioExportOptions struct rather than an ever-growing list of arguments. Assisted-by: Claude Opus 5.5 --- src/libstore/export-import.cc | 88 ++++++++++++++----- .../include/nix/store/export-import.hh | 60 +++++++++---- src/nix/nario-export.md | 2 +- src/nix/nario.cc | 27 +++++- src/nix/nix-store/nix-store.cc | 2 +- tests/functional/nario-diff.sh | 5 ++ 6 files changed, 142 insertions(+), 42 deletions(-) diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 5a51eae02559..b669cee95faf 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -120,31 +120,42 @@ struct NarioDiff std::string patch; }; +std::string showBaseSelectionMethod(BaseSelectionMethod method) +{ + switch (method) { + case BaseSelectionMethod::byName: + return "by-name"; + } + unreachable(); +} + +BaseSelectionMethod parseBaseSelectionMethod(std::string_view s) +{ + if (s == "by-name") + return BaseSelectionMethod::byName; + throw Error("unknown base selection method '%s'", s); +} + /** - * Select the path in `baseClosure` that is most likely to produce a - * small diff against each path in `paths` that is not in - * `baseClosure`. + * The `by-name` base selection method: select a base path with the + * same name (ignoring the version), preferring the one with the most + * similar version and NAR size. */ static std::map -selectDiffBases(Store & store, const StorePaths & paths, const StorePathSet & baseClosure) +selectDiffBasesByName(Store & store, const StorePaths & targets, const StorePaths & bases) { std::map, std::vector> basesByKey; - for (auto & basePath : baseClosure) + for (auto & basePath : bases) basesByKey[getDiffKey(basePath.name())].push_back(basePath); std::map res; - for (auto & path : paths) { - if (baseClosure.contains(path)) - continue; - + for (auto & path : targets) { auto i = basesByKey.find(getDiffKey(path.name())); if (i == basesByKey.end()) continue; auto info = store.queryPathInfo(path); - if (info->narSize > maxDiffNarSize) - continue; auto version = DrvName(path.name()).version; @@ -154,8 +165,6 @@ selectDiffBases(Store & store, const StorePaths & paths, const StorePathSet & ba for (auto & basePath : i->second) { auto baseInfo = store.queryPathInfo(basePath); - if (baseInfo->narSize > maxDiffNarSize || baseInfo->narSize + info->narSize > maxZstdPatchWindow()) - continue; auto ratio = sizeRatio(info->narSize, baseInfo->narSize); if (ratio >= maxDiffSizeRatio) continue; @@ -175,6 +184,45 @@ selectDiffBases(Store & store, const StorePaths & paths, const StorePathSet & ba return res; } +/** + * Select the path in `baseClosure` that is most likely to produce a + * small diff against each path in `paths` that is not in + * `baseClosure`, using the base selection method `method`. + */ +static std::map +selectDiffBases(Store & store, BaseSelectionMethod method, const StorePaths & paths, const StorePathSet & baseClosure) +{ + /* Constraints that apply regardless of the selection method. */ + auto isDiffable = [&](const StorePath & path) { return store.queryPathInfo(path)->narSize <= maxDiffNarSize; }; + + StorePaths targets; + for (auto & path : paths) + if (!baseClosure.contains(path) && isDiffable(path)) + targets.push_back(path); + + StorePaths bases; + for (auto & path : baseClosure) + if (isDiffable(path)) + bases.push_back(path); + + std::map res; + + switch (method) { + case BaseSelectionMethod::byName: + res = selectDiffBasesByName(store, targets, bases); + break; + } + + /* zstd patches can only refer to the base if base and target + fit in the zstd window together. */ + std::erase_if(res, [&](auto & pair) { + return store.queryPathInfo(pair.first)->narSize + store.queryPathInfo(pair.second)->narSize + > maxZstdPatchWindow(); + }); + + return res; +} + /** * Compute a binary diff between the NARs of `basePath` and `path`. * Returns `std::nullopt` if the diff is not small enough to be worth @@ -222,14 +270,12 @@ static std::optional makeDiff(Store & store, const StorePath & path, }; } -void exportPaths( - Store & store, - const StorePathSet & paths, - Sink & sink, - unsigned int version, - const StorePathSet & basePaths, - CompressionAlgo compression) +void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, const NarioExportOptions & options) { + auto version = options.version; + auto & basePaths = options.basePaths; + auto compression = options.compression; + auto sorted = store.topoSortPaths(paths); std::reverse(sorted.begin(), sorted.end()); @@ -269,7 +315,7 @@ void exportPaths( if (!basePaths.empty()) { store.computeFSClosure(basePaths, baseClosure); - auto diffBases = selectDiffBases(store, sorted, baseClosure); + auto diffBases = selectDiffBases(store, options.baseSelectionMethod, sorted, baseClosure); Sync> diffs_; ThreadPool pool; diff --git a/src/libstore/include/nix/store/export-import.hh b/src/libstore/include/nix/store/export-import.hh index cd157e723b86..7ba096d08015 100644 --- a/src/libstore/include/nix/store/export-import.hh +++ b/src/libstore/include/nix/store/export-import.hh @@ -5,26 +5,54 @@ namespace nix { +/** + * Methods for selecting the base path against which to diff a path + * being exported. + */ +enum class BaseSelectionMethod { + /** + * Select a base path with the same name (ignoring the version). + */ + byName, +}; + +std::string showBaseSelectionMethod(BaseSelectionMethod method); + +BaseSelectionMethod parseBaseSelectionMethod(std::string_view s); + +struct NarioExportOptions +{ + /** + * The nario format version (1 or 2). + */ + unsigned int version; + + /** + * If non-empty (only supported for version 2), the receiver is + * assumed to have the closure of these paths. Paths in that + * closure are not exported (but recorded as being expected to be + * present), and other paths are exported as binary diffs against + * a path in that closure if a suitable one is found. + */ + StorePathSet basePaths; + + /** + * How to find suitable base paths. + */ + BaseSelectionMethod baseSelectionMethod = BaseSelectionMethod::byName; + + /** + * If not `none` (only supported for version 2), NARs that are not + * exported as binary diffs are compressed. + */ + CompressionAlgo compression = CompressionAlgo::none; +}; + /** * Export multiple paths in the format expected by `nix-store * --import`. The paths will be sorted topologically. - * - * If `basePaths` is non-empty (only supported for version 2), the - * receiver is assumed to have the closure of `basePaths`. Paths in - * that closure are not exported (but recorded as being expected to - * be present), and other paths are exported as binary diffs against - * a path in that closure if a suitable one is found. - * - * If `compression` is not `none` (only supported for version 2), - * NARs that are not exported as binary diffs are compressed. */ -void exportPaths( - Store & store, - const StorePathSet & paths, - Sink & sink, - unsigned int version, - const StorePathSet & basePaths = {}, - CompressionAlgo compression = CompressionAlgo::none); +void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, const NarioExportOptions & options); /** * How a NAR is compressed inside a nario. diff --git a/src/nix/nario-export.md b/src/nix/nario-export.md index bbf7ceb0e46d..8171fe29ab95 100644 --- a/src/nix/nario-export.md +++ b/src/nix/nario-export.md @@ -39,7 +39,7 @@ The `--base` flag specifies one or more *installables* whose closure (the *base * Paths that are in the base closure are not included in the nario. Instead, the nario records that they're expected to be present, and `nix nario import` fails if they're missing or have a different NAR hash. -* For other paths, `nix nario export` looks for a path in the base closure with the same name (ignoring the version), e.g. `hello-2.12.2` for `hello-2.12.3`. If one exists, the path is stored as a binary diff (computed using zstd) against the NAR of that base path, provided that the diff is sufficiently small. `nix nario import` reconstructs the path by applying the diff to the base path, after verifying that the base path has the expected NAR hash. +* For other paths, `nix nario export` looks for a suitable path in the base closure. How this path is selected is determined by the flag `--base-selection-method`. Currently, the only method is `by-name` (the default), which selects a path with the same name (ignoring the version), e.g. `hello-2.12.2` for `hello-2.12.3`. If a suitable path exists, the path is stored as a binary diff (computed using zstd) against the NAR of that base path, provided that the diff is sufficiently small. `nix nario import` reconstructs the path by applying the diff to the base path, after verifying that the base path has the expected NAR hash. Binary diffs require nario format 2. Narios that contain binary diffs cannot be imported by versions of Nix that don't support them. diff --git a/src/nix/nario.cc b/src/nix/nario.cc index 1c1b1ef33968..274bb91dd82f 100644 --- a/src/nix/nario.cc +++ b/src/nix/nario.cc @@ -36,6 +36,7 @@ struct CmdNarioExport : StorePathsCommand { unsigned int version = 0; std::vector baseArgs; + BaseSelectionMethod baseSelectionMethod = BaseSelectionMethod::byName; CompressionAlgo compression = CompressionAlgo::none; CmdNarioExport() @@ -57,6 +58,21 @@ struct CmdNarioExport : StorePathsCommand .completer = getCompleteInstallable(), }); + addFlag({ + .longName = "base-selection-method", + .description = + "How to select the path in the base closure against which to diff a path. Currently the only method is `by-name` (select a path with the same name, ignoring the version), which is the default.", + .labels = {"method"}, + .handler = {[this](std::string s) { baseSelectionMethod = parseBaseSelectionMethod(s); }}, + .completer = {[](AddCompletions & completions, size_t, std::string_view prefix) { + for (auto method : {BaseSelectionMethod::byName}) { + auto s = showBaseSelectionMethod(method); + if (s.starts_with(prefix)) + completions.add(s); + } + }}, + }); + addFlag({ .longName = "compression", .description = @@ -85,12 +101,17 @@ struct CmdNarioExport : StorePathsCommand throw UsageError("refusing to write nario to a terminal"); FdSink sink(std::move(fd)); - StorePathSet basePaths; + NarioExportOptions options{ + .version = version, + .baseSelectionMethod = baseSelectionMethod, + .compression = compression, + }; + if (!baseArgs.empty()) - basePaths = Installable::toStorePathSet( + options.basePaths = Installable::toStorePathSet( getEvalStore(), store, Realise::Outputs, operateOn, parseInstallables(store, baseArgs)); - exportPaths(*store, StorePathSet(storePaths.begin(), storePaths.end()), sink, version, basePaths, compression); + exportPaths(*store, StorePathSet(storePaths.begin(), storePaths.end()), sink, options); } }; diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index 59928479aba6..be0e6e29ed6e 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -773,7 +773,7 @@ static void opExport(Strings opFlags, Strings opArgs) paths.insert(store->followLinksToStorePath(i)); FdSink sink(getStandardOutput()); - exportPaths(*store, paths, sink, 1); + exportPaths(*store, paths, sink, {.version = 1}); sink.flush(); } diff --git a/tests/functional/nario-diff.sh b/tests/functional/nario-diff.sh index 284e16145ccf..c9f4308ffc71 100755 --- a/tests/functional/nario-diff.sh +++ b/tests/functional/nario-diff.sh @@ -22,6 +22,11 @@ nix nario export --format 2 -r "$old" > "$TEST_ROOT"/old.nario nix nario export --format 2 -r "$new" > "$TEST_ROOT"/full.nario nix nario export --format 2 -r "$new" --base "$old" > "$TEST_ROOT"/diff.nario +# `by-name` is the default base selection method. +nix nario export --format 2 -r "$new" --base "$old" --base-selection-method by-name > "$TEST_ROOT"/diff-by-name.nario +cmp "$TEST_ROOT"/diff.nario "$TEST_ROOT"/diff-by-name.nario +expectStderr 1 nix nario export --format 2 -r "$new" --base "$old" --base-selection-method foo | grepQuiet "unknown base selection method 'foo'" + # The diff should be much smaller than the full export. (( $(stat -c %s "$TEST_ROOT"/diff.nario) * 4 < $(stat -c %s "$TEST_ROOT"/full.nario) )) From f5ce5454e7cf04cf047b755b7a2ee238a13c15f0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 8 Oct 2026 12:51:08 +0200 Subject: [PATCH 06/13] nix nario export: Add --compression-level flag This sets the compression level for the method given by `--compression`. The default for zstd is now 9 rather than zstd's own default of 3, since it's still fast but compresses significantly better. Other methods keep their own defaults. The baseline against which binary diffs are compared (a diff is only used if it's sufficiently smaller than the compressed NAR) now uses the same zstd level that full NARs are compressed with, or 9 if they aren't compressed with zstd. Previously it was always level 3. Assisted-by: Claude Opus 5.5 --- src/libstore/export-import.cc | 22 +++++++++++++++---- .../include/nix/store/export-import.hh | 7 ++++++ src/nix/nario-export.md | 2 ++ src/nix/nario.cc | 10 +++++++++ tests/functional/nario-diff.sh | 7 ++++++ 5 files changed, 44 insertions(+), 4 deletions(-) diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index b669cee95faf..2e619e65021a 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -42,7 +42,10 @@ static const double maxDiffFraction = 0.9; static const int diffCompressionLevel = 19; -static const int baselineCompressionLevel = 3; +/* The default zstd compression level for full NARs. Also used to + decide whether a diff is worth it if full NARs are not compressed + using zstd. */ +static const int defaultZstdCompressionLevel = 9; static WorkerProto::Version exportProtoVersion{ .number = @@ -228,7 +231,8 @@ selectDiffBases(Store & store, BaseSelectionMethod method, const StorePaths & pa * Returns `std::nullopt` if the diff is not small enough to be worth * it. */ -static std::optional makeDiff(Store & store, const StorePath & path, const StorePath & basePath) +static std::optional +makeDiff(Store & store, const StorePath & path, const StorePath & basePath, int baselineCompressionLevel) { Activity act( *logger, @@ -275,6 +279,8 @@ void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, const N auto version = options.version; auto & basePaths = options.basePaths; auto compression = options.compression; + auto compressionLevel = + options.compressionLevel.value_or(compression == CompressionAlgo::zstd ? defaultZstdCompressionLevel : -1); auto sorted = store.topoSortPaths(paths); std::reverse(sorted.begin(), sorted.end()); @@ -285,6 +291,14 @@ void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, const N if (compression != CompressionAlgo::none && version != 2) throw Error("compression is only supported in nario version 2"); + if (options.compressionLevel && compression == CompressionAlgo::none) + throw Error("a compression level requires a compression method"); + + /* A diff must be smaller than the full NAR compressed with zstd at + the level we would otherwise use. */ + auto baselineCompressionLevel = + compression == CompressionAlgo::zstd ? compressionLevel : defaultZstdCompressionLevel; + auto dumpNar = [&](const ValidPathInfo & info, Sink & sink) { HashSink hashSink(HashAlgorithm::SHA256); TeeSink teeSink(sink, hashSink); @@ -321,7 +335,7 @@ void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, const N ThreadPool pool; for (auto & [path, basePath] : diffBases) pool.enqueue([&, path, basePath]() { - if (auto diff = makeDiff(store, path, basePath)) + if (auto diff = makeDiff(store, path, basePath, baselineCompressionLevel)) diffs_.lock()->emplace(path, std::move(*diff)); }); pool.process(); @@ -356,7 +370,7 @@ void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, const N /* We need to know the size of the compressed NAR before writing it, so buffer it in memory. */ StringSink compressed; - auto compressionSink = makeCompressionSink(compression, compressed, true); + auto compressionSink = makeCompressionSink(compression, compressed, true, compressionLevel); dumpNar(*info, *compressionSink); compressionSink->finish(); diff --git a/src/libstore/include/nix/store/export-import.hh b/src/libstore/include/nix/store/export-import.hh index 7ba096d08015..7edaea846dea 100644 --- a/src/libstore/include/nix/store/export-import.hh +++ b/src/libstore/include/nix/store/export-import.hh @@ -46,6 +46,13 @@ struct NarioExportOptions * exported as binary diffs are compressed. */ CompressionAlgo compression = CompressionAlgo::none; + + /** + * The compression level to use for `compression`. If unset, the + * default is 9 for zstd, and the compression method's default + * otherwise. + */ + std::optional compressionLevel; }; /** diff --git a/src/nix/nario-export.md b/src/nix/nario-export.md index 8171fe29ab95..9cc7e3e6766a 100644 --- a/src/nix/nario-export.md +++ b/src/nix/nario-export.md @@ -51,6 +51,8 @@ By default, the NARs in a nario are not compressed. The `--compression` flag spe # nix nario export --format 2 --compression zstd -r /nix/store/6i6xl6bmcpxqd51m8nlva40d5c1bhndx-hello-2.12.3 > hello.nario ``` +The `--compression-level` flag specifies the compression level. For `zstd`, the default is 9, which is still fast but compresses significantly better than zstd's own default of 3. + Compression requires nario format 2. Narios that contain compressed NARs cannot be imported by versions of Nix that don't support them. )"" diff --git a/src/nix/nario.cc b/src/nix/nario.cc index 274bb91dd82f..57eec3e40738 100644 --- a/src/nix/nario.cc +++ b/src/nix/nario.cc @@ -38,6 +38,7 @@ struct CmdNarioExport : StorePathsCommand std::vector baseArgs; BaseSelectionMethod baseSelectionMethod = BaseSelectionMethod::byName; CompressionAlgo compression = CompressionAlgo::none; + std::optional compressionLevel; CmdNarioExport() { @@ -80,6 +81,14 @@ struct CmdNarioExport : StorePathsCommand .labels = {"method"}, .handler = {[this](std::string s) { compression = parseCompressionAlgo(s, true); }}, }); + + addFlag({ + .longName = "compression-level", + .description = + "Compression level to use for the method specified by `--compression`. The default is 9 for `zstd`, and the compression method's own default for other methods.", + .labels = {"level"}, + .handler = {&compressionLevel}, + }); } std::string description() override @@ -105,6 +114,7 @@ struct CmdNarioExport : StorePathsCommand .version = version, .baseSelectionMethod = baseSelectionMethod, .compression = compression, + .compressionLevel = compressionLevel, }; if (!baseArgs.empty()) diff --git a/tests/functional/nario-diff.sh b/tests/functional/nario-diff.sh index c9f4308ffc71..cea3cce27997 100755 --- a/tests/functional/nario-diff.sh +++ b/tests/functional/nario-diff.sh @@ -33,6 +33,13 @@ expectStderr 1 nix nario export --format 2 -r "$new" --base "$old" --base-select # Test compression of full NARs. expectStderr 1 nix nario export --format 1 --compression zstd -r "$new" | grepQuiet "compression is only supported in nario version 2" expectStderr 1 nix nario export --format 2 --compression foo -r "$new" | grepQuiet "unknown compression method 'foo'" +expectStderr 1 nix nario export --format 2 --compression-level 5 -r "$new" | grepQuiet "a compression level requires a compression method" + +# Higher compression levels should produce smaller narios. +nix nario export --format 2 --compression zstd --compression-level 1 -r "$new" > "$TEST_ROOT/full-zstd-1.nario" +nix nario export --format 2 --compression zstd --compression-level 19 -r "$new" > "$TEST_ROOT/full-zstd-19.nario" +(( $(stat -c %s "$TEST_ROOT/full-zstd-19.nario") < $(stat -c %s "$TEST_ROOT/full-zstd-1.nario") )) +nix nario import --no-check-sigs < "$TEST_ROOT/full-zstd-19.nario" for method in zstd xz; do nix nario export --format 2 --compression "$method" -r "$new" > "$TEST_ROOT/full-$method.nario" From 9d8506f93f56908ef82dbe1158c76d5cee84c198 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 8 Oct 2026 13:04:40 +0200 Subject: [PATCH 07/13] nix nario export: Don't buffer large compressed NARs in memory Since the size of a compressed NAR must be written before its contents, we buffered the entire compressed NAR in memory, which uses unbounded memory for large NARs. This adds `SpillingStringSink`, a sink that accumulates data in memory up to a limit, and spills to an anonymous temporary file beyond that. `getSource()` returns a source that reads the data back from either. `nix nario export` now uses it with a 32 MiB limit. The nario format is unchanged. Assisted-by: Claude Opus 5.5 --- src/libstore/export-import.cc | 14 ++++- src/libutil-tests/meson.build | 1 + src/libutil-tests/spilling-sink.cc | 56 +++++++++++++++++++ src/libutil/include/nix/util/meson.build | 1 + src/libutil/include/nix/util/spilling-sink.hh | 55 ++++++++++++++++++ src/libutil/meson.build | 1 + src/libutil/spilling-sink.cc | 44 +++++++++++++++ 7 files changed, 169 insertions(+), 3 deletions(-) create mode 100644 src/libutil-tests/spilling-sink.cc create mode 100644 src/libutil/include/nix/util/spilling-sink.hh create mode 100644 src/libutil/spilling-sink.cc diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 2e619e65021a..608f9b4c692b 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -10,6 +10,7 @@ #include "nix/util/thread-pool.hh" #include "nix/util/zstd-patch.hh" #include "nix/util/compression.hh" +#include "nix/util/spilling-sink.hh" #include @@ -47,6 +48,10 @@ static const int diffCompressionLevel = 19; using zstd. */ static const int defaultZstdCompressionLevel = 9; +/* Compressed NARs larger than this are buffered on disk rather than + in memory. */ +static const size_t maxInMemoryCompressedNarSize = 32 * 1024 * 1024; + static WorkerProto::Version exportProtoVersion{ .number = { @@ -368,15 +373,18 @@ void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, const N if (compression != CompressionAlgo::none) { /* We need to know the size of the compressed NAR - before writing it, so buffer it in memory. */ - StringSink compressed; + before writing it, so buffer it in memory or on + disk. */ + SpillingStringSink compressed(maxInMemoryCompressedNarSize); auto compressionSink = makeCompressionSink(compression, compressed, true, compressionLevel); dumpNar(*info, *compressionSink); compressionSink->finish(); sink << narioTagCompressed; WorkerProto::write(store, conn, *info); - sink << showCompressionAlgo(compression) << compressed.s; + sink << showCompressionAlgo(compression) << compressed.size(); + compressed.getSource()->drainInto(sink); + writePadding(compressed.size(), sink); continue; } diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index f105c1cd6e1e..f5a2b4b49963 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -95,6 +95,7 @@ sources = files( 'sort.cc', 'source-accessor.cc', 'spawn.cc', + 'spilling-sink.cc', 'strings.cc', 'suggestions.cc', 'terminal.cc', diff --git a/src/libutil-tests/spilling-sink.cc b/src/libutil-tests/spilling-sink.cc new file mode 100644 index 000000000000..49259a9aa7c1 --- /dev/null +++ b/src/libutil-tests/spilling-sink.cc @@ -0,0 +1,56 @@ +#include "nix/util/spilling-sink.hh" + +#include + +namespace nix { + +TEST(SpillingStringSink, belowLimit) +{ + SpillingStringSink sink(16); + sink("hello"); + sink(" world"); + ASSERT_FALSE(sink.isSpilled()); + ASSERT_EQ(sink.size(), 11); + ASSERT_EQ(sink.getSource()->drain(), "hello world"); +} + +TEST(SpillingStringSink, atLimit) +{ + SpillingStringSink sink(8); + sink("0123"); + sink("4567"); + ASSERT_FALSE(sink.isSpilled()); + ASSERT_EQ(sink.getSource()->drain(), "01234567"); +} + +TEST(SpillingStringSink, empty) +{ + SpillingStringSink sink(8); + ASSERT_FALSE(sink.isSpilled()); + ASSERT_EQ(sink.size(), 0); + ASSERT_EQ(sink.getSource()->drain(), ""); +} + +TEST(SpillingStringSink, spills) +{ + SpillingStringSink sink(1000); + std::string expected; + for (int i = 0; i < 10000; ++i) { + auto s = fmt("line %d\n", i); + sink(s); + expected += s; + } + ASSERT_TRUE(sink.isSpilled()); + ASSERT_EQ(sink.size(), expected.size()); + ASSERT_EQ(sink.getSource()->drain(), expected); +} + +TEST(SpillingStringSink, spillsOnFirstWrite) +{ + SpillingStringSink sink(4); + sink("0123456789"); + ASSERT_TRUE(sink.isSpilled()); + ASSERT_EQ(sink.getSource()->drain(), "0123456789"); +} + +} // namespace nix diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 0cbb36fe6958..7308fd740ea8 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -90,6 +90,7 @@ headers = [ config_pub_h ] + files( 'sort.hh', 'source-accessor.hh', 'source-path.hh', + 'spilling-sink.hh', 'split.hh', 'std-hash.hh', 'strings-inline.hh', diff --git a/src/libutil/include/nix/util/spilling-sink.hh b/src/libutil/include/nix/util/spilling-sink.hh new file mode 100644 index 000000000000..5594c70da3c8 --- /dev/null +++ b/src/libutil/include/nix/util/spilling-sink.hh @@ -0,0 +1,55 @@ +#pragma once +///@file + +#include "nix/util/serialise.hh" + +namespace nix { + +/** + * A sink that accumulates data in memory up to `maxMemory` bytes, and + * spills to an anonymous temporary file once that limit is exceeded. + * This is useful when the size of some data must be known before it + * can be written, without using unbounded memory. + */ +struct SpillingStringSink : Sink +{ + explicit SpillingStringSink(size_t maxMemory) + : maxMemory(maxMemory) + { + } + + void operator()(std::string_view data) override; + + /** + * Total number of bytes written. + */ + uint64_t size() const + { + return written; + } + + /** + * Whether the data has been spilled to disk. + */ + bool isSpilled() const + { + return (bool) fd; + } + + /** + * Return a source that reads back everything written so far. No + * more data may be written after calling this. The source must + * not outlive this sink. + */ + std::unique_ptr getSource(); + +private: + size_t maxMemory; + uint64_t written = 0; + bool finished = false; + StringSink memory; + AutoCloseFD fd; + std::optional fileSink; +}; + +} // namespace nix diff --git a/src/libutil/meson.build b/src/libutil/meson.build index dd6594a3a396..43a805b4b514 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -188,6 +188,7 @@ sources = [ config_priv_h ] + files( 'signature/signer.cc', 'source-accessor.cc', 'source-path.cc', + 'spilling-sink.cc', 'strings.cc', 'suggestions.cc', 'table.cc', diff --git a/src/libutil/spilling-sink.cc b/src/libutil/spilling-sink.cc new file mode 100644 index 000000000000..ea59f307b22a --- /dev/null +++ b/src/libutil/spilling-sink.cc @@ -0,0 +1,44 @@ +#include "nix/util/spilling-sink.hh" +#include "nix/util/file-system.hh" + +#include + +namespace nix { + +void SpillingStringSink::operator()(std::string_view data) +{ + assert(!finished); + + written += data.size(); + + if (!fileSink && memory.s.size() + data.size() <= maxMemory) { + memory(data); + return; + } + + if (!fileSink) { + fd = createAnonymousTempFile(); + fileSink.emplace(fd.get()); + (*fileSink)(memory.s); + memory.s = {}; + } + + (*fileSink)(data); +} + +std::unique_ptr SpillingStringSink::getSource() +{ + assert(!finished); + finished = true; + + if (fileSink) { + fileSink->flush(); + auto source = std::make_unique(fd.get()); + source->restart(); + return source; + } + + return std::make_unique(std::move(memory.s)); +} + +} // namespace nix From 9bb93075474b38ab5d7e6b12e73eb81e0b21e9cb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 8 Oct 2026 15:28:25 +0200 Subject: [PATCH 08/13] nix nario export: Compute diffs and compressed NARs in parallel Previously, binary diffs were all computed up front in a thread pool before anything was written, and full NARs were compressed one at a time on the writing thread (relying only on zstd's internal multithreading, which doesn't help much for small NARs). This adds `processOrdered()`, which runs a "produce" function for a range of items in a thread pool and calls a "consume" function on the results strictly in order. At most `window` items (by default twice the number of threads) are in progress or waiting to be consumed, so memory and temporary disk usage stay bounded. `exportPaths()` now uses this to compute diffs and compressed NARs in parallel, falling back to compression if a diff isn't worth it, and writes entries as soon as they're ready. Only paths that need expensive work go through the thread pool; cheap entries (paths in the base closure and uncompressed full NARs) are written in between. Otherwise, a diff export where most paths are in the base closure would get little parallelism from the window. The output is byte-identical to before. Exporting a 6.6 GB NixOS system closure with `--compression zstd` went from 240s to 101s on a 24-core machine. Diff exports take about the same time as before, since they already computed diffs in parallel. Assisted-by: Claude Opus 5.5 --- src/libstore/export-import.cc | 127 +++++++++++++------- src/libutil-tests/meson.build | 1 + src/libutil-tests/thread-pool.cc | 97 +++++++++++++++ src/libutil/include/nix/util/thread-pool.hh | 86 +++++++++++++ 4 files changed, 269 insertions(+), 42 deletions(-) create mode 100644 src/libutil-tests/thread-pool.cc diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 608f9b4c692b..ad8cb57e516f 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -329,70 +329,113 @@ void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, const N case 2: { StorePathSet baseClosure; - std::map diffs; + std::map diffBases; if (!basePaths.empty()) { store.computeFSClosure(basePaths, baseClosure); - - auto diffBases = selectDiffBases(store, options.baseSelectionMethod, sorted, baseClosure); - - Sync> diffs_; - ThreadPool pool; - for (auto & [path, basePath] : diffBases) - pool.enqueue([&, path, basePath]() { - if (auto diff = makeDiff(store, path, basePath, baselineCompressionLevel)) - diffs_.lock()->emplace(path, std::move(*diff)); - }); - pool.process(); - diffs = std::move(*diffs_.lock()); + diffBases = selectDiffBases(store, options.baseSelectionMethod, sorted, baseClosure); } - WorkerProto::WriteConn conn{.to = sink, .version = exportProtoVersion, .shortStorePaths = true}; + /* The expensive part of an entry (a binary diff or a + compressed NAR), computed in parallel. If neither is set, + the path is either present in the base closure or written + as a full NAR. */ + struct ExportEntry + { + std::optional diff; + std::unique_ptr compressed; + }; + + /* The indices in `sorted` of the paths that need expensive + work. Only these are processed by the thread pool, so that + cheap entries (e.g. paths in the base closure) don't take + up room in its window. */ + std::vector jobs; + for (size_t i = 0; i < sorted.size(); ++i) + if (!baseClosure.contains(sorted[i]) + && (diffBases.contains(sorted[i]) || compression != CompressionAlgo::none)) + jobs.push_back(i); - sink << exportMagicV2; + WorkerProto::WriteConn conn{.to = sink, .version = exportProtoVersion, .shortStorePaths = true}; - for (auto & path : sorted) { + auto writeEntry = [&](size_t i, ExportEntry && entry) { + auto & path = sorted[i]; auto info = store.queryPathInfo(path); if (baseClosure.contains(path)) { sink << narioTagPresent; WorkerProto::write(store, conn, *info); - continue; } - Activity act(*logger, lvlTalkative, actUnknown, fmt("exporting path '%s'", store.printStorePath(path))); - - if (auto diff = get(diffs, path)) { + else if (entry.diff) { sink << narioTagDiff; WorkerProto::write(store, conn, *info); - sink << showNarioDiffAlgo(diff->algo); - WorkerProto::write(store, conn, diff->basePath); - sink << diff->baseNarHash.to_string(HashFormat::SRI, true) << diff->patch; - continue; + sink << showNarioDiffAlgo(entry.diff->algo); + WorkerProto::write(store, conn, entry.diff->basePath); + sink << entry.diff->baseNarHash.to_string(HashFormat::SRI, true) << entry.diff->patch; } - if (compression != CompressionAlgo::none) { - /* We need to know the size of the compressed NAR - before writing it, so buffer it in memory or on - disk. */ - SpillingStringSink compressed(maxInMemoryCompressedNarSize); - auto compressionSink = makeCompressionSink(compression, compressed, true, compressionLevel); - dumpNar(*info, *compressionSink); - compressionSink->finish(); - + else if (entry.compressed) { sink << narioTagCompressed; WorkerProto::write(store, conn, *info); - sink << showCompressionAlgo(compression) << compressed.size(); - compressed.getSource()->drainInto(sink); - writePadding(compressed.size(), sink); - continue; + sink << showCompressionAlgo(compression) << entry.compressed->size(); + entry.compressed->getSource()->drainInto(sink); + writePadding(entry.compressed->size(), sink); } - sink << narioTagFull; - // FIXME: move to CommonProto? - WorkerProto::write(store, conn, *info); - dumpNar(*info, sink); - } + else { + Activity act(*logger, lvlTalkative, actUnknown, fmt("exporting path '%s'", store.printStorePath(path))); + sink << narioTagFull; + // FIXME: move to CommonProto? + WorkerProto::write(store, conn, *info); + dumpNar(*info, sink); + } + }; + + /* Write the cheap entries preceding index `end`. */ + size_t nextToWrite = 0; + auto writeCheapEntries = [&](size_t end) { + for (; nextToWrite < end; ++nextToWrite) + writeEntry(nextToWrite, {}); + }; + + sink << exportMagicV2; + + processOrdered( + jobs.size(), + [&](size_t j) { + auto & path = sorted[jobs[j]]; + ExportEntry entry; + + auto basePath = get(diffBases, path); + + Activity act(*logger, lvlTalkative, actUnknown, fmt("exporting path '%s'", store.printStorePath(path))); + + if (basePath) { + entry.diff = makeDiff(store, path, *basePath, baselineCompressionLevel); + if (entry.diff) + return entry; + } + + if (compression != CompressionAlgo::none) { + /* We need to know the size of the compressed NAR + before writing it, so buffer it in memory or on + disk. */ + entry.compressed = std::make_unique(maxInMemoryCompressedNarSize); + auto compressionSink = makeCompressionSink(compression, *entry.compressed, true, compressionLevel); + dumpNar(*store.queryPathInfo(path), *compressionSink); + compressionSink->finish(); + } + + return entry; + }, + [&](size_t j, ExportEntry && entry) { + writeCheapEntries(jobs[j]); + writeEntry(jobs[j], std::move(entry)); + nextToWrite++; + }); + + writeCheapEntries(sorted.size()); sink << narioTagEnd; break; diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index f5a2b4b49963..2380deded55b 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -99,6 +99,7 @@ sources = files( 'strings.cc', 'suggestions.cc', 'terminal.cc', + 'thread-pool.cc', 'topo-sort.cc', 'url.cc', 'util.cc', diff --git a/src/libutil-tests/thread-pool.cc b/src/libutil-tests/thread-pool.cc new file mode 100644 index 000000000000..9d031e2b8520 --- /dev/null +++ b/src/libutil-tests/thread-pool.cc @@ -0,0 +1,97 @@ +#include "nix/util/thread-pool.hh" + +#include + +#include + +namespace nix { + +TEST(processOrdered, consumesInOrder) +{ + const size_t n = 200; + + std::vector consumed; + + processOrdered( + n, + [&](size_t i) { + /* Make items finish out of order. */ + thread_local std::mt19937 rng(std::hash{}(std::this_thread::get_id())); + std::this_thread::sleep_for(std::chrono::microseconds(rng() % 1000)); + return i * i; + }, + [&](size_t i, size_t && res) { + ASSERT_EQ(i, consumed.size()); + ASSERT_EQ(res, i * i); + consumed.push_back(i); + }, + 0, + 8); + + ASSERT_EQ(consumed.size(), n); +} + +TEST(processOrdered, respectsWindow) +{ + const size_t window = 5; + + std::atomic outstanding = 0, maxOutstanding = 0; + size_t count = 0; + + processOrdered( + 100, + [&](size_t i) { + auto cur = ++outstanding; + size_t prev = maxOutstanding; + while (cur > prev && !maxOutstanding.compare_exchange_weak(prev, cur)) + ; + /* Make the first item slow, so that other items pile up + behind it. */ + if (i == 0) + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + return i; + }, + [&](size_t i, size_t &&) { + --outstanding; + count++; + }, + window, + 8); + + ASSERT_EQ(count, 100); + ASSERT_LE(maxOutstanding, window); +} + +TEST(processOrdered, empty) +{ + processOrdered(0, [](size_t) -> int { throw Error("should not be called"); }, [](size_t, int &&) { FAIL(); }); +} + +TEST(processOrdered, produceThrows) +{ + ASSERT_THROW( + processOrdered( + 100, + [](size_t i) { + if (i == 42) + throw Error("boom"); + return i; + }, + [](size_t, size_t &&) {}), + Error); +} + +TEST(processOrdered, consumeThrows) +{ + ASSERT_THROW( + processOrdered( + 100, + [](size_t i) { return i; }, + [](size_t i, size_t &&) { + if (i == 42) + throw Error("boom"); + }), + Error); +} + +} // namespace nix diff --git a/src/libutil/include/nix/util/thread-pool.hh b/src/libutil/include/nix/util/thread-pool.hh index a6bb7cf02d1c..29d4d353f044 100644 --- a/src/libutil/include/nix/util/thread-pool.hh +++ b/src/libutil/include/nix/util/thread-pool.hh @@ -10,6 +10,8 @@ #include #include #include +#include +#include namespace nix { @@ -182,4 +184,88 @@ void processGraph( throw Error("graph processing incomplete (cyclic reference?)"); } +/** + * Compute `produce(i)` for each `i` in `[0, n)` in parallel, and call + * `consume(i, result)` for each result strictly in order of `i`, one + * at a time (though not necessarily on the same thread). At most + * `window` items are being produced or waiting to be consumed at any + * time. If `window` is 0, it defaults to twice the number of threads, + * so that a slow item doesn't leave the other threads idle. + */ +template +void processOrdered( + size_t n, fun produce, fun consume, size_t window = 0, size_t maxThreads = 0) +{ + if (!maxThreads) + maxThreads = std::max(1U, std::thread::hardware_concurrency()); + + if (!window) + window = 2 * maxThreads; + + struct State + { + std::map ready; + size_t nextToConsume = 0; + size_t nextToEnqueue = 0; + bool consuming = false; + }; + + Sync state_; + + std::function worker; + + /* Create pool last to ensure threads are stopped before other + destructors run. */ + ThreadPool pool(maxThreads); + + worker = [&](size_t i) { + auto res = produce(i); + + { + auto state(state_.lock()); + state->ready.emplace(i, std::move(res)); + + /* If another thread is already consuming results, it will + pick up this one. */ + if (state->consuming) + return; + state->consuming = true; + } + + while (true) { + size_t j; + std::optional value; + + { + auto state(state_.lock()); + auto k = state->ready.find(state->nextToConsume); + if (k == state->ready.end()) { + state->consuming = false; + return; + } + j = state->nextToConsume; + value.emplace(std::move(k->second)); + state->ready.erase(k); + } + + consume(j, std::move(*value)); + + { + auto state(state_.lock()); + state->nextToConsume++; + if (state->nextToEnqueue < n) + pool.enqueue(std::bind(worker, state->nextToEnqueue++)); + } + } + }; + + { + auto state(state_.lock()); + while (state->nextToEnqueue < std::min(n, window)) + pool.enqueue(std::bind(worker, state->nextToEnqueue++)); + } + + pool.process(); +} + } // namespace nix From b10176e3272b63aa997c00a8f6c8dff5e5b0ef97 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 8 Oct 2026 17:21:44 +0200 Subject: [PATCH 09/13] processOrdered(): Bound buffered results by size rather than count processOrdered() previously allowed at most `window` (by default twice the number of threads) items to be in progress or waiting to be consumed. This caused head-of-line blocking: when a slow item (e.g. a binary diff of a large NAR) is the next one to be consumed, the other threads quickly finish the remaining items in the window and then sit idle. In an export of a mass rebuild of a NixOS system (2314 diffs, of which ~15 are NARs over 250 MB), the large diffs were effectively serialized, so the export took 1181s instead of 437s before the parallelism changes. Instead, processOrdered() now stops starting new items only while the total size of the buffered results (as returned by a `getSize` callback) exceeds `maxBuffered`. Since diffs are small, this allows the thread pool to work far ahead of a slow item, while still bounding the amount of compressed NAR data that piles up. `nix nario export` uses a limit of 1 GiB. Up to twice as many items as threads are active at any time, since ThreadPool only starts a new thread if there are more pending items than threads. Timings on a 12-core/24-thread machine (output is identical): mass rebuild diff: 417s before parallelism, 1181s with window, 424s now full system export: 226s before parallelism, 105s with window, 67s now Assisted-by: Claude Opus 5.5 --- src/libstore/export-import.cc | 17 +++-- src/libutil-tests/thread-pool.cc | 74 ++++++++++++++++----- src/libutil/include/nix/util/thread-pool.hh | 71 ++++++++++++++------ 3 files changed, 123 insertions(+), 39 deletions(-) diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index ad8cb57e516f..9131329ccd64 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -52,6 +52,11 @@ static const int defaultZstdCompressionLevel = 9; in memory. */ static const size_t maxInMemoryCompressedNarSize = 32 * 1024 * 1024; +/* When exporting in parallel, don't start new diffs or compressions + while the entries that have been computed but not yet written + exceed this size. */ +static const size_t maxBufferedEntriesSize = 1024 * 1024 * 1024; + static WorkerProto::Version exportProtoVersion{ .number = { @@ -347,9 +352,9 @@ void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, const N }; /* The indices in `sorted` of the paths that need expensive - work. Only these are processed by the thread pool, so that - cheap entries (e.g. paths in the base closure) don't take - up room in its window. */ + work. Only these are processed by the thread pool; cheap + entries (e.g. paths in the base closure) are written in + between. */ std::vector jobs; for (size_t i = 0; i < sorted.size(); ++i) if (!baseClosure.contains(sorted[i]) @@ -433,7 +438,11 @@ void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, const N writeCheapEntries(jobs[j]); writeEntry(jobs[j], std::move(entry)); nextToWrite++; - }); + }, + [](const ExportEntry & entry) -> size_t { + return entry.diff ? entry.diff->patch.size() : entry.compressed ? entry.compressed->size() : 0; + }, + maxBufferedEntriesSize); writeCheapEntries(sorted.size()); diff --git a/src/libutil-tests/thread-pool.cc b/src/libutil-tests/thread-pool.cc index 9d031e2b8520..c336c66dcdbc 100644 --- a/src/libutil-tests/thread-pool.cc +++ b/src/libutil-tests/thread-pool.cc @@ -6,6 +6,11 @@ namespace nix { +static size_t unitSize(const size_t &) +{ + return 1; +} + TEST(processOrdered, consumesInOrder) { const size_t n = 200; @@ -25,46 +30,81 @@ TEST(processOrdered, consumesInOrder) ASSERT_EQ(res, i * i); consumed.push_back(i); }, - 0, + unitSize, + 10, 8); ASSERT_EQ(consumed.size(), n); } -TEST(processOrdered, respectsWindow) +TEST(processOrdered, respectsBufferLimit) { - const size_t window = 5; + const size_t maxBuffered = 5, maxThreads = 4; - std::atomic outstanding = 0, maxOutstanding = 0; + std::atomic buffered = 0, maxSeen = 0; size_t count = 0; processOrdered( 100, [&](size_t i) { - auto cur = ++outstanding; - size_t prev = maxOutstanding; - while (cur > prev && !maxOutstanding.compare_exchange_weak(prev, cur)) - ; /* Make the first item slow, so that other items pile up behind it. */ if (i == 0) std::this_thread::sleep_for(std::chrono::milliseconds(50)); + auto cur = ++buffered; + size_t prev = maxSeen; + while (cur > prev && !maxSeen.compare_exchange_weak(prev, cur)) + ; return i; }, [&](size_t i, size_t &&) { - --outstanding; + --buffered; count++; }, - window, - 8); + unitSize, + maxBuffered, + maxThreads); ASSERT_EQ(count, 100); - ASSERT_LE(maxOutstanding, window); + /* The limit can be exceeded by the items that were already + started when it was reached. */ + ASSERT_LE(maxSeen, maxBuffered + 2 * maxThreads); +} + +TEST(processOrdered, slowItemDoesNotBlockOthers) +{ + const size_t n = 100; + + std::atomic produced = 0; + + processOrdered( + n, + [&](size_t i) { + if (i == 0) + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + produced++; + return i; + }, + [&](size_t i, size_t &&) { + /* Since results have size 0, all other items should have + been produced while the first one was in progress. */ + if (i == 0) { + ASSERT_EQ(produced, n); + } + }, + [](const size_t &) -> size_t { return 0; }, + 0, + 4); } TEST(processOrdered, empty) { - processOrdered(0, [](size_t) -> int { throw Error("should not be called"); }, [](size_t, int &&) { FAIL(); }); + processOrdered( + 0, + [](size_t) -> int { throw Error("should not be called"); }, + [](size_t, int &&) { FAIL(); }, + [](const int &) -> size_t { return 0; }, + 10); } TEST(processOrdered, produceThrows) @@ -77,7 +117,9 @@ TEST(processOrdered, produceThrows) throw Error("boom"); return i; }, - [](size_t, size_t &&) {}), + [](size_t, size_t &&) {}, + unitSize, + 10), Error); } @@ -90,7 +132,9 @@ TEST(processOrdered, consumeThrows) [](size_t i, size_t &&) { if (i == 42) throw Error("boom"); - }), + }, + unitSize, + 10), Error); } diff --git a/src/libutil/include/nix/util/thread-pool.hh b/src/libutil/include/nix/util/thread-pool.hh index 29d4d353f044..26d426b124ae 100644 --- a/src/libutil/include/nix/util/thread-pool.hh +++ b/src/libutil/include/nix/util/thread-pool.hh @@ -187,26 +187,43 @@ void processGraph( /** * Compute `produce(i)` for each `i` in `[0, n)` in parallel, and call * `consume(i, result)` for each result strictly in order of `i`, one - * at a time (though not necessarily on the same thread). At most - * `window` items are being produced or waiting to be consumed at any - * time. If `window` is 0, it defaults to twice the number of threads, - * so that a slow item doesn't leave the other threads idle. + * at a time (though not necessarily on the same thread). + * + * Results that have been produced but not yet consumed are buffered. + * To bound the amount of buffered data, no new items are started + * while the total `getSize()` of the buffered results exceeds + * `maxBuffered`. (The limit can be exceeded by the results of the + * items that are already being produced.) Thus, if results are small, + * a slow item doesn't prevent the other threads from making progress + * on subsequent items. */ template void processOrdered( - size_t n, fun produce, fun consume, size_t window = 0, size_t maxThreads = 0) + size_t n, + fun produce, + fun consume, + fun getSize, + size_t maxBuffered, + size_t maxThreads = 0) { if (!maxThreads) maxThreads = std::max(1U, std::thread::hardware_concurrency()); - if (!window) - window = 2 * maxThreads; + struct Result + { + T value; + size_t size; + }; struct State { - std::map ready; + std::map ready; size_t nextToConsume = 0; size_t nextToEnqueue = 0; + /* Number of items enqueued but not yet produced. */ + size_t active = 0; + /* Total size of the results in `ready`. */ + size_t buffered = 0; bool consuming = false; }; @@ -218,12 +235,30 @@ void processOrdered( destructors run. */ ThreadPool pool(maxThreads); + /* Start new items, as long as the buffer limit hasn't been + reached. We keep up to twice as many items active as there are + threads, since `ThreadPool` only starts a new thread if there + are more pending items than threads. Note that this always + makes progress: if nothing is active, the next item to be + consumed is either in `ready` or hasn't been enqueued yet, in + which case `ready` is empty and so `buffered` is 0. */ + auto enqueueMore = [&](State & state) { + while (state.nextToEnqueue < n && state.active < 2 * maxThreads && state.buffered <= maxBuffered) { + pool.enqueue(std::bind(worker, state.nextToEnqueue++)); + state.active++; + } + }; + worker = [&](size_t i) { - auto res = produce(i); + auto value = produce(i); + auto size = getSize(value); { auto state(state_.lock()); - state->ready.emplace(i, std::move(res)); + state->ready.emplace(i, Result{std::move(value), size}); + state->buffered += size; + state->active--; + enqueueMore(*state); /* If another thread is already consuming results, it will pick up this one. */ @@ -234,7 +269,7 @@ void processOrdered( while (true) { size_t j; - std::optional value; + std::optional result; { auto state(state_.lock()); @@ -244,26 +279,22 @@ void processOrdered( return; } j = state->nextToConsume; - value.emplace(std::move(k->second)); + result.emplace(std::move(k->second)); state->ready.erase(k); } - consume(j, std::move(*value)); + consume(j, std::move(result->value)); { auto state(state_.lock()); state->nextToConsume++; - if (state->nextToEnqueue < n) - pool.enqueue(std::bind(worker, state->nextToEnqueue++)); + state->buffered -= result->size; + enqueueMore(*state); } } }; - { - auto state(state_.lock()); - while (state->nextToEnqueue < std::min(n, window)) - pool.enqueue(std::bind(worker, state->nextToEnqueue++)); - } + enqueueMore(*state_.lock()); pool.process(); } From bb971837796e6623b4d06938ce18076f913cd389 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 11 Oct 2026 15:43:51 +0200 Subject: [PATCH 10/13] nix nario export: Add experimental hdiffpatch diff algorithm This adds `--diff-algorithm {zstd,hdiffpatch}`. With `hdiffpatch`, binary diffs are computed using the external `hdiffz` program from HDiffPatch (https://github.com/sisong/HDiffPatch) and applied using `hpatchz`. These must be in `PATH` on export and import. The nario format is unchanged, since diff entries already record the diff algorithm. hdiffz is run with the options used by HDiffPatch's own zstd benchmark (`-m-6 -SD`), but with zstd level 19 to match our zstd diffs, single-threaded since diffs are already computed in parallel, and without its patch self-check since the NAR hash is verified on import. HDiffPatch isn't in Nixpkgs, so this adds a package for it (packaging/hdiffpatch.nix), which is used by the functional tests and exposed by the nix-make flake for experimentation. This is an experiment to compare against zstd diffs. (A similar experiment with bsdiff produced slightly smaller diffs than zstd but used much more memory.) Assisted-by: Claude Opus 5.5 --- packaging/dependencies.nix | 2 + packaging/hdiffpatch.nix | 58 ++++++++++ packaging/nix-make/flake.nix | 6 +- src/libstore/export-import.cc | 101 +++++++++++++++--- .../include/nix/store/export-import.hh | 42 +++++--- src/nix/nario-export.md | 8 +- src/nix/nario-list.md | 2 +- src/nix/nario.cc | 17 +++ tests/functional/nario-diff.sh | 15 +++ tests/functional/package.nix | 3 + tests/functional/test-runner.nix | 3 + 11 files changed, 226 insertions(+), 31 deletions(-) create mode 100644 packaging/hdiffpatch.nix diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 169c68ffe266..3e34b3ff45cc 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -254,6 +254,8 @@ scope: { wasmtime = pkgs.callPackage ./wasmtime.nix { }; + hdiffpatch = pkgs.callPackage ./hdiffpatch.nix { }; + sentry-native = (pkgs.callPackage ./sentry-native.nix { }).override { # Avoid having two curls in our closure. inherit (scope) curl; diff --git a/packaging/hdiffpatch.nix b/packaging/hdiffpatch.nix new file mode 100644 index 000000000000..3d0f37a1d44a --- /dev/null +++ b/packaging/hdiffpatch.nix @@ -0,0 +1,58 @@ +# HDiffPatch (https://github.com/sisong/HDiffPatch), used for +# experimental `nix nario export --diff-algorithm hdiffpatch`. +{ + lib, + stdenv, + fetchFromGitHub, + zstd, + zlib, +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "hdiffpatch"; + version = "5.1.3"; + + src = fetchFromGitHub { + owner = "sisong"; + repo = "HDiffPatch"; + tag = "v${finalAttrs.version}"; + hash = "sha256-8bDefNdKO9bLJzENY1OEap7tTSzI7C/oUECgXBJSiEE="; + }; + + buildInputs = [ + zstd + zlib + ]; + + # Disable everything that requires source checkouts of other + # projects next to the HDiffPatch source tree, and link against zstd + # and zlib from Nixpkgs. + makeFlags = [ + "ZSTD=2" + "LDEF=0" + "ZLIB=2" + "LZMA=0" + "MD5=0" + "XXH=0" + "VCD=0" + "BSD=0" + "BZIP2=0" + "DIR_DIFF=0" + ]; + + enableParallelBuilding = true; + + installPhase = '' + runHook preInstall + install -D -m 755 hdiffz $out/bin/hdiffz + install -D -m 755 hpatchz $out/bin/hpatchz + runHook postInstall + ''; + + meta = { + description = "Binary diff and patch tools for large files and directories"; + homepage = "https://github.com/sisong/HDiffPatch"; + license = lib.licenses.mit; + mainProgram = "hdiffz"; + }; +}) diff --git a/packaging/nix-make/flake.nix b/packaging/nix-make/flake.nix index 290b83c48cd6..3c4f6857f013 100644 --- a/packaging/nix-make/flake.nix +++ b/packaging/nix-make/flake.nix @@ -73,7 +73,9 @@ nix-flake-tests = self.callPackage ../../src/libflake-tests/make.nix { }; nix-flake-tests-run = self.nix-flake-tests.tests.run; - test-runner = self.callPackage ../../tests/functional/test-runner.nix { }; + test-runner = self.callPackage ../../tests/functional/test-runner.nix { + inherit (pkgs.nixDependencies2) hdiffpatch; + }; functional-tests = self.callPackage ../../tests/functional/make.nix { }; # Everything that runs tests; building it runs them all. @@ -91,6 +93,8 @@ rec { packages.${system} = { inherit (make.${system}.release) test-runner; + # For experimenting with `nix nario export --diff-algorithm hdiffpatch`. + inherit (pkgs.nixDependencies2) hdiffpatch; }; # `nix flake check` runs all tests against the release variant. diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 9131329ccd64..da0d5cf45b3e 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -11,6 +11,8 @@ #include "nix/util/zstd-patch.hh" #include "nix/util/compression.hh" #include "nix/util/spilling-sink.hh" +#include "nix/util/file-system.hh" +#include "nix/util/processes.hh" #include @@ -114,6 +116,8 @@ std::string showNarioDiffAlgo(NarioDiffAlgo algo) switch (algo) { case NarioDiffAlgo::zstd: return "zstd"; + case NarioDiffAlgo::hdiffpatch: + return "hdiffpatch"; } unreachable(); } @@ -122,9 +126,58 @@ NarioDiffAlgo parseNarioDiffAlgo(std::string_view s) { if (s == "zstd") return NarioDiffAlgo::zstd; + if (s == "hdiffpatch") + return NarioDiffAlgo::hdiffpatch; throw Error("unsupported nario diff algorithm '%s'", s); } +/** + * Compute a patch from `base` to `target` using the external `hdiffz` + * program from HDiffPatch, compressed using zstd at level `level`. + */ +static std::string makeHdiffPatch(std::string_view base, std::string_view target, int level) +{ + AutoDelete tmpDir(createTempDir("", "nix-hdiffz"), true); + auto basePath = tmpDir.path() / "base"; + auto targetPath = tmpDir.path() / "target"; + auto patchPath = tmpDir.path() / "patch"; + writeFile(basePath, base); + writeFile(targetPath, target); + /* These are the options that HDiffPatch's own benchmarks use for + zstd. Run single-threaded (`-p-1`), since we already compute + diffs in parallel, and don't verify the patch (`-d`), since + the NAR hash is checked on import. */ + runProgram( + "hdiffz", + true, + {"-m-6", + "-SD", + fmt("-c-zstd-%d-24", level), + "-p-1", + "-d", + "-f", + basePath.native(), + targetPath.native(), + patchPath.native()}); + return readFile(patchPath); +} + +/** + * Apply a patch created by `makeHdiffPatch()` to `base` using the + * external `hpatchz` program, writing the result to `sink`. + */ +static void applyHdiffPatch(std::string_view base, std::string_view patch, Sink & sink) +{ + AutoDelete tmpDir(createTempDir("", "nix-hpatchz"), true); + auto basePath = tmpDir.path() / "base"; + auto patchPath = tmpDir.path() / "patch"; + auto outPath = tmpDir.path() / "out"; + writeFile(basePath, base); + writeFile(patchPath, patch); + runProgram("hpatchz", true, {"-f", basePath.native(), patchPath.native(), outPath.native()}); + readFile(outPath, sink); +} + struct NarioDiff { NarioDiffAlgo algo; @@ -202,8 +255,12 @@ selectDiffBasesByName(Store & store, const StorePaths & targets, const StorePath * small diff against each path in `paths` that is not in * `baseClosure`, using the base selection method `method`. */ -static std::map -selectDiffBases(Store & store, BaseSelectionMethod method, const StorePaths & paths, const StorePathSet & baseClosure) +static std::map selectDiffBases( + Store & store, + BaseSelectionMethod method, + NarioDiffAlgo diffAlgo, + const StorePaths & paths, + const StorePathSet & baseClosure) { /* Constraints that apply regardless of the selection method. */ auto isDiffable = [&](const StorePath & path) { return store.queryPathInfo(path)->narSize <= maxDiffNarSize; }; @@ -228,10 +285,11 @@ selectDiffBases(Store & store, BaseSelectionMethod method, const StorePaths & pa /* zstd patches can only refer to the base if base and target fit in the zstd window together. */ - std::erase_if(res, [&](auto & pair) { - return store.queryPathInfo(pair.first)->narSize + store.queryPathInfo(pair.second)->narSize - > maxZstdPatchWindow(); - }); + if (diffAlgo == NarioDiffAlgo::zstd) + std::erase_if(res, [&](auto & pair) { + return store.queryPathInfo(pair.first)->narSize + store.queryPathInfo(pair.second)->narSize + > maxZstdPatchWindow(); + }); return res; } @@ -241,8 +299,12 @@ selectDiffBases(Store & store, BaseSelectionMethod method, const StorePaths & pa * Returns `std::nullopt` if the diff is not small enough to be worth * it. */ -static std::optional -makeDiff(Store & store, const StorePath & path, const StorePath & basePath, int baselineCompressionLevel) +static std::optional makeDiff( + Store & store, + const StorePath & path, + const StorePath & basePath, + NarioDiffAlgo diffAlgo, + int baselineCompressionLevel) { Activity act( *logger, @@ -261,7 +323,16 @@ makeDiff(Store & store, const StorePath & path, const StorePath & basePath, int auto [baseNar, baseNarHash] = getNar(*store.queryPathInfo(basePath)); auto [nar, narHash] = getNar(*store.queryPathInfo(path)); - auto patch = makeZstdPatch(baseNar, nar, diffCompressionLevel); + std::string patch; + switch (diffAlgo) { + case NarioDiffAlgo::zstd: + patch = makeZstdPatch(baseNar, nar, diffCompressionLevel); + break; + case NarioDiffAlgo::hdiffpatch: + patch = makeHdiffPatch(baseNar, nar, diffCompressionLevel); + break; + } + auto baselineSize = makeZstdPatch("", nar, baselineCompressionLevel).size(); printMsg( @@ -277,7 +348,7 @@ makeDiff(Store & store, const StorePath & path, const StorePath & basePath, int return std::nullopt; return NarioDiff{ - .algo = NarioDiffAlgo::zstd, + .algo = diffAlgo, .basePath = basePath, .baseNarHash = baseNarHash, .patch = std::move(patch), @@ -338,7 +409,7 @@ void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, const N if (!basePaths.empty()) { store.computeFSClosure(basePaths, baseClosure); - diffBases = selectDiffBases(store, options.baseSelectionMethod, sorted, baseClosure); + diffBases = selectDiffBases(store, options.baseSelectionMethod, options.diffAlgo, sorted, baseClosure); } /* The expensive part of an entry (a binary diff or a @@ -417,7 +488,7 @@ void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, const N Activity act(*logger, lvlTalkative, actUnknown, fmt("exporting path '%s'", store.printStorePath(path))); if (basePath) { - entry.diff = makeDiff(store, path, *basePath, baselineCompressionLevel); + entry.diff = makeDiff(store, path, *basePath, options.diffAlgo, baselineCompressionLevel); if (entry.diff) return entry; } @@ -442,7 +513,8 @@ void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, const N [](const ExportEntry & entry) -> size_t { return entry.diff ? entry.diff->patch.size() : entry.compressed ? entry.compressed->size() : 0; }, - maxBufferedEntriesSize); + maxBufferedEntriesSize, + 12); writeCheapEntries(sorted.size()); @@ -646,6 +718,9 @@ StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) case NarioDiffAlgo::zstd: applyZstdPatch(baseNar.s, patch, sink); break; + case NarioDiffAlgo::hdiffpatch: + applyHdiffPatch(baseNar.s, patch, sink); + break; } }); store.addToStore(info, *nar, NoRepair, checkSigs); diff --git a/src/libstore/include/nix/store/export-import.hh b/src/libstore/include/nix/store/export-import.hh index 7edaea846dea..3df266564943 100644 --- a/src/libstore/include/nix/store/export-import.hh +++ b/src/libstore/include/nix/store/export-import.hh @@ -20,6 +20,28 @@ std::string showBaseSelectionMethod(BaseSelectionMethod method); BaseSelectionMethod parseBaseSelectionMethod(std::string_view s); +/** + * Binary diff algorithms supported in narios. + */ +enum class NarioDiffAlgo { + /** + * zstd compression of the target NAR, using the base NAR as a + * prefix dictionary (see `makeZstdPatch()`). + */ + zstd, + + /** + * A zstd-compressed patch in HDiffPatch's single-stream format, + * computed and applied by the external `hdiffz` and `hpatchz` + * programs. Experimental. + */ + hdiffpatch, +}; + +std::string showNarioDiffAlgo(NarioDiffAlgo algo); + +NarioDiffAlgo parseNarioDiffAlgo(std::string_view s); + struct NarioExportOptions { /** @@ -41,6 +63,11 @@ struct NarioExportOptions */ BaseSelectionMethod baseSelectionMethod = BaseSelectionMethod::byName; + /** + * The algorithm used to compute binary diffs. + */ + NarioDiffAlgo diffAlgo = NarioDiffAlgo::zstd; + /** * If not `none` (only supported for version 2), NARs that are not * exported as binary diffs are compressed. @@ -74,21 +101,6 @@ struct NarioCompression uint64_t size; }; -/** - * Binary diff algorithms supported in narios. - */ -enum class NarioDiffAlgo { - /** - * zstd compression of the target NAR, using the base NAR as a - * prefix dictionary (see `makeZstdPatch()`). - */ - zstd, -}; - -std::string showNarioDiffAlgo(NarioDiffAlgo algo); - -NarioDiffAlgo parseNarioDiffAlgo(std::string_view s); - /** * Callbacks for the entries in a nario, used by `parseNario()`. */ diff --git a/src/nix/nario-export.md b/src/nix/nario-export.md index 9cc7e3e6766a..d35c31519a15 100644 --- a/src/nix/nario-export.md +++ b/src/nix/nario-export.md @@ -39,7 +39,13 @@ The `--base` flag specifies one or more *installables* whose closure (the *base * Paths that are in the base closure are not included in the nario. Instead, the nario records that they're expected to be present, and `nix nario import` fails if they're missing or have a different NAR hash. -* For other paths, `nix nario export` looks for a suitable path in the base closure. How this path is selected is determined by the flag `--base-selection-method`. Currently, the only method is `by-name` (the default), which selects a path with the same name (ignoring the version), e.g. `hello-2.12.2` for `hello-2.12.3`. If a suitable path exists, the path is stored as a binary diff (computed using zstd) against the NAR of that base path, provided that the diff is sufficiently small. `nix nario import` reconstructs the path by applying the diff to the base path, after verifying that the base path has the expected NAR hash. +* For other paths, `nix nario export` looks for a suitable path in the base closure. How this path is selected is determined by the flag `--base-selection-method`. Currently, the only method is `by-name` (the default), which selects a path with the same name (ignoring the version), e.g. `hello-2.12.2` for `hello-2.12.3`. If a suitable path exists, the path is stored as a binary diff against the NAR of that base path, provided that the diff is sufficiently small. `nix nario import` reconstructs the path by applying the diff to the base path, after verifying that the base path has the expected NAR hash. + +The flag `--diff-algorithm` determines how binary diffs are computed: + +* `zstd` (the default): The NAR is compressed using zstd, with the NAR of the base path as a dictionary. + +* `hdiffpatch`: The diff is computed using the external `hdiffz` program from [HDiffPatch](https://github.com/sisong/HDiffPatch), and applied using `hpatchz`. These programs must be in `PATH` both when exporting and importing. This is experimental. Binary diffs require nario format 2. Narios that contain binary diffs cannot be imported by versions of Nix that don't support them. diff --git a/src/nix/nario-list.md b/src/nix/nario-list.md index d85f172485aa..f6d12bf88357 100644 --- a/src/nix/nario-list.md +++ b/src/nix/nario-list.md @@ -40,7 +40,7 @@ R""( This command lists the contents of a nario file read from standard input. -For narios created with `nix nario export --base`, paths that are stored as a binary diff are shown with the diff algorithm (currently always `zstd`), the base path of the diff and the size of the diff. Their file contents are not shown by `--recursive`, since that would require the base path. Paths that are expected to already be present at the destination are shown as such. In the JSON output, these have a `diff` or `present` attribute, respectively. +For narios created with `nix nario export --base`, paths that are stored as a binary diff are shown with the diff algorithm (`zstd` or `hdiffpatch`), the base path of the diff and the size of the diff. Their file contents are not shown by `--recursive`, since that would require the base path. Paths that are expected to already be present at the destination are shown as such. In the JSON output, these have a `diff` or `present` attribute, respectively. For narios created with `nix nario export --compression`, the compression method and compressed size of each NAR are shown. In the JSON output, these are in the `compression` attribute. diff --git a/src/nix/nario.cc b/src/nix/nario.cc index 57eec3e40738..d1711540e80a 100644 --- a/src/nix/nario.cc +++ b/src/nix/nario.cc @@ -37,6 +37,7 @@ struct CmdNarioExport : StorePathsCommand unsigned int version = 0; std::vector baseArgs; BaseSelectionMethod baseSelectionMethod = BaseSelectionMethod::byName; + NarioDiffAlgo diffAlgo = NarioDiffAlgo::zstd; CompressionAlgo compression = CompressionAlgo::none; std::optional compressionLevel; @@ -74,6 +75,21 @@ struct CmdNarioExport : StorePathsCommand }}, }); + addFlag({ + .longName = "diff-algorithm", + .description = + "Algorithm to use for computing binary diffs. Must be `zstd` (the default) or `hdiffpatch`. `hdiffpatch` is experimental and requires the `hdiffz` and `hpatchz` programs from HDiffPatch to be in `PATH` on export and import.", + .labels = {"algorithm"}, + .handler = {[this](std::string s) { diffAlgo = parseNarioDiffAlgo(s); }}, + .completer = {[](AddCompletions & completions, size_t, std::string_view prefix) { + for (auto algo : {NarioDiffAlgo::zstd, NarioDiffAlgo::hdiffpatch}) { + auto s = showNarioDiffAlgo(algo); + if (s.starts_with(prefix)) + completions.add(s); + } + }}, + }); + addFlag({ .longName = "compression", .description = @@ -113,6 +129,7 @@ struct CmdNarioExport : StorePathsCommand NarioExportOptions options{ .version = version, .baseSelectionMethod = baseSelectionMethod, + .diffAlgo = diffAlgo, .compression = compression, .compressionLevel = compressionLevel, }; diff --git a/tests/functional/nario-diff.sh b/tests/functional/nario-diff.sh index cea3cce27997..f10b375258a1 100755 --- a/tests/functional/nario-diff.sh +++ b/tests/functional/nario-diff.sh @@ -27,6 +27,15 @@ nix nario export --format 2 -r "$new" --base "$old" --base-selection-method by-n cmp "$TEST_ROOT"/diff.nario "$TEST_ROOT"/diff-by-name.nario expectStderr 1 nix nario export --format 2 -r "$new" --base "$old" --base-selection-method foo | grepQuiet "unknown base selection method 'foo'" +# `zstd` is the default diff algorithm. +nix nario export --format 2 -r "$new" --base "$old" --diff-algorithm zstd > "$TEST_ROOT"/diff-zstd-algo.nario +cmp "$TEST_ROOT"/diff.nario "$TEST_ROOT"/diff-zstd-algo.nario +expectStderr 1 nix nario export --format 2 -r "$new" --base "$old" --diff-algorithm foo | grepQuiet "unsupported nario diff algorithm 'foo'" + +nix nario export --format 2 -r "$new" --base "$old" --diff-algorithm hdiffpatch > "$TEST_ROOT"/diff-hdiffpatch.nario +nix nario list < "$TEST_ROOT"/diff-hdiffpatch.nario | grepQuiet "^$new: [0-9]* bytes, hdiffpatch diff against $old ([0-9]* bytes)$" +[[ $(nix nario list --json < "$TEST_ROOT"/diff-hdiffpatch.nario | jq -r ".paths.\"$new\".diff.algorithm") = hdiffpatch ]] + # The diff should be much smaller than the full export. (( $(stat -c %s "$TEST_ROOT"/diff.nario) * 4 < $(stat -c %s "$TEST_ROOT"/full.nario) )) @@ -106,6 +115,12 @@ nix nario import --no-check-sigs < "$TEST_ROOT"/old.nario nix nario import --no-check-sigs < "$TEST_ROOT"/diff-zstd.nario nix store verify --no-trust -r "$new" +clearStore +nix nario import --no-check-sigs < "$TEST_ROOT"/old.nario +nix nario import --no-check-sigs < "$TEST_ROOT"/diff-hdiffpatch.nario +nix store verify --no-trust -r "$new" +grepQuiet "^2$" "$new/data" + # Importing fails if the base has a different NAR hash. clearStore nix nario import --no-check-sigs < "$TEST_ROOT"/old.nario diff --git a/tests/functional/package.nix b/tests/functional/package.nix index 5204a52c0ff1..6e94a8d48f5e 100644 --- a/tests/functional/package.nix +++ b/tests/functional/package.nix @@ -8,6 +8,7 @@ pkg-config, jq, + hdiffpatch, git, mercurial, python3, @@ -56,6 +57,8 @@ mkMesonDerivation ( pkg-config jq + # For `nix nario export --diff-algorithm hdiffpatch`. + hdiffpatch git mercurial # For the OpenTelemetry collector in `otel.sh`. diff --git a/tests/functional/test-runner.nix b/tests/functional/test-runner.nix index c9ea0684632f..a5a5e8e8c030 100644 --- a/tests/functional/test-runner.nix +++ b/tests/functional/test-runner.nix @@ -9,6 +9,7 @@ bash, coreutils, jq, + hdiffpatch, git, mercurial, unixtools, @@ -30,6 +31,8 @@ runCommand "nix-test-runner" bash coreutils jq + # For `nix nario export --diff-algorithm hdiffpatch`. + hdiffpatch git mercurial unixtools.script From 4f905997da4ac072a17d9c104890b8c0be535cf4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 11 Oct 2026 16:02:05 +0200 Subject: [PATCH 11/13] nix nario export: Use HDiffPatch as a library Instead of running the external `hdiffz` and `hpatchz` programs (which required them to be in `PATH` and went through temporary files), link HDiffPatch into libnixutil and call it directly via the new `makeHdiffPatch()` and `applyHdiffPatch()`, analogous to `makeZstdPatch()` and `applyZstdPatch()`. Upstream doesn't install a library, so the HDiffPatch package now installs the static library built by its Makefile (single-threaded and with -fPIC), its headers, and a pkg-config file that includes the configuration macros that affect the headers. The settings are the same as before (match score 6, fast block matching, zstd level 19 with a 2^24 dictionary, single-threaded). `applyHdiffPatch()` streams the output to a sink and validates the patch header (base size, patch length and memory requirements) before applying it. Since HDiffPatch overwrites its inputs during matching, `makeDiff()` now moves the NARs into `makeHdiffPatch()` rather than copying them. Assisted-by: Claude Opus 5.5 --- packaging/hdiffpatch.nix | 57 ++++++- packaging/nix-make/deps.nix | 8 + packaging/nix-make/flake.nix | 6 +- src/libstore/export-import.cc | 60 +------ .../include/nix/store/export-import.hh | 5 +- src/libutil-tests/hdiff-patch.cc | 96 +++++++++++ src/libutil-tests/meson.build | 1 + src/libutil/hdiff-patch.cc | 161 ++++++++++++++++++ src/libutil/include/nix/util/hdiff-patch.hh | 30 ++++ src/libutil/include/nix/util/meson.build | 1 + src/libutil/meson.build | 5 + src/libutil/package.nix | 2 + src/nix/nario-export.md | 2 +- src/nix/nario.cc | 2 +- tests/functional/package.nix | 3 - tests/functional/test-runner.nix | 3 - 16 files changed, 364 insertions(+), 78 deletions(-) create mode 100644 src/libutil-tests/hdiff-patch.cc create mode 100644 src/libutil/hdiff-patch.cc create mode 100644 src/libutil/include/nix/util/hdiff-patch.hh diff --git a/packaging/hdiffpatch.nix b/packaging/hdiffpatch.nix index 3d0f37a1d44a..14135b914ee9 100644 --- a/packaging/hdiffpatch.nix +++ b/packaging/hdiffpatch.nix @@ -1,11 +1,13 @@ # HDiffPatch (https://github.com/sisong/HDiffPatch), used for # experimental `nix nario export --diff-algorithm hdiffpatch`. +# +# Upstream doesn't install a library, so we install the static library +# that its Makefile builds, its headers, and a pkg-config file. { lib, stdenv, fetchFromGitHub, zstd, - zlib, }: stdenv.mkDerivation (finalAttrs: { @@ -19,18 +21,16 @@ stdenv.mkDerivation (finalAttrs: { hash = "sha256-8bDefNdKO9bLJzENY1OEap7tTSzI7C/oUECgXBJSiEE="; }; - buildInputs = [ - zstd - zlib - ]; + buildInputs = [ zstd ]; # Disable everything that requires source checkouts of other # projects next to the HDiffPatch source tree, and link against zstd - # and zlib from Nixpkgs. + # from Nixpkgs. Disable multi-threading, since Nix already computes + # diffs in parallel. makeFlags = [ "ZSTD=2" "LDEF=0" - "ZLIB=2" + "ZLIB=0" "LZMA=0" "MD5=0" "XXH=0" @@ -38,21 +38,60 @@ stdenv.mkDerivation (finalAttrs: { "BSD=0" "BZIP2=0" "DIR_DIFF=0" + "MT=0" ]; + # The static library gets linked into a shared library. + env.NIX_CFLAGS_COMPILE = "-fPIC"; + enableParallelBuilding = true; + # These must match the configuration selected by `makeFlags`, since + # they affect the headers. + cflags = lib.concatStringsSep " " [ + "-D_IS_USED_MULTITHREAD=0" + "-DZ7_ST" + "-D_IS_NEED_BSDIFF=0" + "-D_IS_NEED_VCDIFF=0" + "-D_IS_NEED_DIR_DIFF_PATCH=0" + "-D_IS_NEED_ALL_CompressPlugin=0" + "-D_IS_NEED_DEFAULT_CompressPlugin=0" + "-D_IS_NEED_ALL_ChecksumPlugin=0" + "-D_IS_NEED_DEFAULT_ChecksumPlugin=0" + "-D_CompressPlugin_zstd" + ]; + installPhase = '' runHook preInstall + + install -D -m 644 libhdiffpatch.a $out/lib/libhdiffpatch.a install -D -m 755 hdiffz $out/bin/hdiffz install -D -m 755 hpatchz $out/bin/hpatchz + + # The headers refer to each other relative to the root of the + # source tree, so keep that structure. + find . -name '*.h' -exec install -D -m 644 {} $out/include/hdiffpatch/{} \; + + mkdir -p $out/lib/pkgconfig + cat > $out/lib/pkgconfig/hdiffpatch.pc < @@ -131,53 +130,6 @@ NarioDiffAlgo parseNarioDiffAlgo(std::string_view s) throw Error("unsupported nario diff algorithm '%s'", s); } -/** - * Compute a patch from `base` to `target` using the external `hdiffz` - * program from HDiffPatch, compressed using zstd at level `level`. - */ -static std::string makeHdiffPatch(std::string_view base, std::string_view target, int level) -{ - AutoDelete tmpDir(createTempDir("", "nix-hdiffz"), true); - auto basePath = tmpDir.path() / "base"; - auto targetPath = tmpDir.path() / "target"; - auto patchPath = tmpDir.path() / "patch"; - writeFile(basePath, base); - writeFile(targetPath, target); - /* These are the options that HDiffPatch's own benchmarks use for - zstd. Run single-threaded (`-p-1`), since we already compute - diffs in parallel, and don't verify the patch (`-d`), since - the NAR hash is checked on import. */ - runProgram( - "hdiffz", - true, - {"-m-6", - "-SD", - fmt("-c-zstd-%d-24", level), - "-p-1", - "-d", - "-f", - basePath.native(), - targetPath.native(), - patchPath.native()}); - return readFile(patchPath); -} - -/** - * Apply a patch created by `makeHdiffPatch()` to `base` using the - * external `hpatchz` program, writing the result to `sink`. - */ -static void applyHdiffPatch(std::string_view base, std::string_view patch, Sink & sink) -{ - AutoDelete tmpDir(createTempDir("", "nix-hpatchz"), true); - auto basePath = tmpDir.path() / "base"; - auto patchPath = tmpDir.path() / "patch"; - auto outPath = tmpDir.path() / "out"; - writeFile(basePath, base); - writeFile(patchPath, patch); - runProgram("hpatchz", true, {"-f", basePath.native(), patchPath.native(), outPath.native()}); - readFile(outPath, sink); -} - struct NarioDiff { NarioDiffAlgo algo; @@ -323,25 +275,27 @@ static std::optional makeDiff( auto [baseNar, baseNarHash] = getNar(*store.queryPathInfo(basePath)); auto [nar, narHash] = getNar(*store.queryPathInfo(path)); + auto narSize = nar.size(); + auto baselineSize = makeZstdPatch("", nar, baselineCompressionLevel).size(); + std::string patch; switch (diffAlgo) { case NarioDiffAlgo::zstd: patch = makeZstdPatch(baseNar, nar, diffCompressionLevel); break; case NarioDiffAlgo::hdiffpatch: - patch = makeHdiffPatch(baseNar, nar, diffCompressionLevel); + /* This overwrites the NARs, so we can't use them anymore. */ + patch = makeHdiffPatch(std::move(baseNar), std::move(nar), diffCompressionLevel); break; } - auto baselineSize = makeZstdPatch("", nar, baselineCompressionLevel).size(); - printMsg( lvlTalkative, "diff of '%s' against '%s': %d bytes (NAR %d bytes, compressed %d bytes)", store.printStorePath(path), store.printStorePath(basePath), patch.size(), - nar.size(), + narSize, baselineSize); if (patch.size() >= maxDiffFraction * baselineSize) diff --git a/src/libstore/include/nix/store/export-import.hh b/src/libstore/include/nix/store/export-import.hh index 3df266564943..cab9c7055981 100644 --- a/src/libstore/include/nix/store/export-import.hh +++ b/src/libstore/include/nix/store/export-import.hh @@ -31,9 +31,8 @@ enum class NarioDiffAlgo { zstd, /** - * A zstd-compressed patch in HDiffPatch's single-stream format, - * computed and applied by the external `hdiffz` and `hpatchz` - * programs. Experimental. + * A zstd-compressed patch in HDiffPatch's single-stream format + * (see `makeHdiffPatch()`). Experimental. */ hdiffpatch, }; diff --git a/src/libutil-tests/hdiff-patch.cc b/src/libutil-tests/hdiff-patch.cc new file mode 100644 index 000000000000..b6a3b57095b1 --- /dev/null +++ b/src/libutil-tests/hdiff-patch.cc @@ -0,0 +1,96 @@ +#include "nix/util/hdiff-patch.hh" +#include "nix/util/compression.hh" + +#include + +#include + +namespace nix { + +static std::string randomHdiffString(size_t size, uint64_t seed) +{ + std::mt19937_64 rng(seed); + std::string s(size, '\0'); + for (auto & c : s) + c = (char) rng(); + return s; +} + +static std::string applyHdiffPatchToString(std::string_view base, std::string_view patch) +{ + StringSink sink; + applyHdiffPatch(base, patch, sink); + return std::move(sink.s); +} + +TEST(hdiffPatch, roundtrip) +{ + auto base = randomHdiffString(1 << 20, 1); + auto target = base; + /* Make some small changes. */ + target.replace(1000, 5, "hello"); + target.insert(500000, "inserted"); + target.erase(800000, 100); + + auto patch = makeHdiffPatch(base, target, 19); + ASSERT_EQ(applyHdiffPatchToString(base, patch), target); + + /* The base is random, so the patch is only small because it + refers to the base. */ + ASSERT_LT(patch.size(), 1000); +} + +TEST(hdiffPatch, emptyBase) +{ + auto target = randomHdiffString(10000, 2); + auto patch = makeHdiffPatch("", target, 3); + ASSERT_EQ(applyHdiffPatchToString("", patch), target); +} + +TEST(hdiffPatch, emptyTarget) +{ + auto base = randomHdiffString(10000, 3); + auto patch = makeHdiffPatch(base, "", 3); + ASSERT_EQ(applyHdiffPatchToString(base, patch), ""); +} + +TEST(hdiffPatch, wrongBase) +{ + auto base = randomHdiffString(100000, 4); + auto target = base; + target.replace(50000, 5, "hello"); + auto patch = makeHdiffPatch(base, target, 19); + + /* Applying to the wrong base produces garbage (or an error), so + callers must check the base's hash. */ + auto otherBase = randomHdiffString(100000, 5); + try { + ASSERT_NE(applyHdiffPatchToString(otherBase, patch), target); + } catch (CompressionError &) { + } + + /* A base of the wrong size is rejected. */ + ASSERT_THROW(applyHdiffPatchToString(base.substr(1), patch), CompressionError); +} + +TEST(hdiffPatch, truncated) +{ + auto base = randomHdiffString(100000, 6); + auto target = randomHdiffString(100000, 7); + auto patch = makeHdiffPatch(base, target, 3); + ASSERT_THROW(applyHdiffPatchToString(base, patch.substr(0, patch.size() / 2)), CompressionError); +} + +TEST(hdiffPatch, trailingGarbage) +{ + auto base = randomHdiffString(1000, 8); + auto patch = makeHdiffPatch(base, base, 3); + ASSERT_THROW(applyHdiffPatchToString(base, patch + "garbage"), CompressionError); +} + +TEST(hdiffPatch, garbage) +{ + ASSERT_THROW(applyHdiffPatchToString("", "not a patch"), CompressionError); +} + +} // namespace nix diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index 2380deded55b..195b09ef41e6 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -76,6 +76,7 @@ sources = files( 'fun.cc', 'git.cc', 'hash.cc', + 'hdiff-patch.cc', 'hilite.cc', 'json-utils.cc', 'local-keys.cc', diff --git a/src/libutil/hdiff-patch.cc b/src/libutil/hdiff-patch.cc new file mode 100644 index 000000000000..44eb8c8067d1 --- /dev/null +++ b/src/libutil/hdiff-patch.cc @@ -0,0 +1,161 @@ +#include "nix/util/hdiff-patch.hh" +#include "nix/util/compression.hh" + +#include + +#include +#include +#include +#include + +#include +#include + +namespace nix { + +/* The zstd dictionary size (log2) used for patches. */ +static const int zstdDictBits = 24; + +/* The minimum score for matches, i.e. `hdiffz -m-6`. */ +static const int minSingleMatchScore = 6; + +/* Don't trust the memory requirements stated in a patch beyond this. */ +static const uint64_t maxStepMemSize = 64 * 1024 * 1024; + +std::string makeHdiffPatch(std::string base, std::string target, int level) +{ + TCompressPlugin_zstd compressPlugin = zstdCompressPlugin; + compressPlugin.compress_level = level; + compressPlugin.dict_bits = zstdDictBits; + compressPlugin.thread_num = 1; + + auto bytes = [](std::string & s) { return (unsigned char *) s.data(); }; + + std::vector out; + try { + create_single_compressed_diff_block( + bytes(target), + bytes(target) + target.size(), + bytes(base), + bytes(base) + base.size(), + out, + &compressPlugin.base, + kDefaultPatchStepMemSize, + kDefaultFastMatchBlockSize, + minSingleMatchScore, + /* isUseBigCacheMatch */ false, + /* threadNum */ 1); + } catch (std::exception & e) { + throw CompressionError("computing HDiffPatch patch: %s", e.what()); + } + + return std::string((const char *) out.data(), out.size()); +} + +namespace { + +/** + * An HDiffPatch output stream that writes to a `Sink`. HDiffPatch + * writes sequentially. Exceptions thrown by the sink are stored and + * rethrown after HDiffPatch returns, since they can't propagate + * through its C code. + */ +struct SinkOutputStream +{ + hpatch_TStreamOutput stream; + Sink & sink; + uint64_t written = 0; + std::exception_ptr exception; + + SinkOutputStream(Sink & sink, uint64_t size) + : sink(sink) + { + stream.streamImport = this; + stream.streamSize = size; + stream.read_writed = nullptr; + stream.write = write; + } + + static hpatch_BOOL write( + const hpatch_TStreamOutput * stream, + hpatch_StreamPos_t writeToPos, + const unsigned char * data, + const unsigned char * dataEnd) + { + auto & self = *(SinkOutputStream *) stream->streamImport; + if (writeToPos != self.written) + return hpatch_FALSE; + try { + self.sink({(const char *) data, (size_t) (dataEnd - data)}); + } catch (...) { + self.exception = std::current_exception(); + return hpatch_FALSE; + } + self.written += dataEnd - data; + return hpatch_TRUE; + } +}; + +} // namespace + +void applyHdiffPatch(std::string_view base, std::string_view patch, Sink & sink) +{ + auto patchData = (const unsigned char *) patch.data(); + auto baseData = (const unsigned char *) base.data(); + + hpatch_singleCompressedDiffInfo info; + if (!getSingleCompressedDiffInfo_mem(&info, patchData, patchData + patch.size())) + throw CompressionError("invalid HDiffPatch patch"); + + if (info.oldDataSize != base.size()) + throw CompressionError( + "HDiffPatch patch expects a base of %d bytes, but the base has %d bytes", info.oldDataSize, base.size()); + + auto dataSize = info.compressedSize ? info.compressedSize : info.uncompressedSize; + if (info.diffDataPos > patch.size() || dataSize != patch.size() - info.diffDataPos) + throw CompressionError("HDiffPatch patch has an invalid size"); + + if (info.stepMemSize > maxStepMemSize) + throw CompressionError("HDiffPatch patch requires too much memory"); + + hpatch_TDecompress * decompressPlugin = nullptr; + if (info.compressType[0]) { + if (!zstdDecompressPlugin.is_can_open(info.compressType)) + throw CompressionError("HDiffPatch patch uses unsupported compression type '%s'", info.compressType); + decompressPlugin = &zstdDecompressPlugin; + } + + std::vector cache(info.stepMemSize + hpatch_kStreamCacheSize * 3); + + hpatch_TStreamInput baseStream, patchStream; + mem_as_hStreamInput(&baseStream, baseData, baseData + base.size()); + mem_as_hStreamInput(&patchStream, patchData, patchData + patch.size()); + + SinkOutputStream out(sink, info.newDataSize); + + auto ok = patch_single_compressed_diff( + &out.stream, + &baseStream, + &patchStream, + info.diffDataPos, + info.uncompressedSize, + info.compressedSize, + decompressPlugin, + info.coverCount, + (size_t) info.stepMemSize, + cache.data(), + cache.data() + cache.size(), + /* coversListener */ nullptr, + /* threadNum */ 1); + + if (out.exception) + std::rethrow_exception(out.exception); + + if (!ok) + throw CompressionError("applying HDiffPatch patch failed"); + + if (out.written != info.newDataSize) + throw CompressionError("HDiffPatch patch produced %d bytes instead of %d", out.written, info.newDataSize); +} + +} // namespace nix diff --git a/src/libutil/include/nix/util/hdiff-patch.hh b/src/libutil/include/nix/util/hdiff-patch.hh new file mode 100644 index 000000000000..75f9e97e0437 --- /dev/null +++ b/src/libutil/include/nix/util/hdiff-patch.hh @@ -0,0 +1,30 @@ +#pragma once +///@file + +#include "nix/util/serialise.hh" + +#include +#include + +namespace nix { + +/** + * Compute a binary patch that transforms `base` into `target` using + * HDiffPatch (https://github.com/sisong/HDiffPatch), compressed using + * zstd at level `level`. The result is in HDiffPatch's "single + * compressed stream" format and can be applied by + * `applyHdiffPatch()` given the same `base`. + * + * `base` and `target` are taken by value because HDiffPatch + * overwrites them during matching. + */ +std::string makeHdiffPatch(std::string base, std::string target, int level); + +/** + * Apply a patch produced by `makeHdiffPatch()` to `base`, writing the + * reconstructed target to `sink`. The output is streamed, so only + * `base` and `patch` need to be held in memory. + */ +void applyHdiffPatch(std::string_view base, std::string_view patch, Sink & sink); + +} // namespace nix diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 7308fd740ea8..b6f184d03beb 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -56,6 +56,7 @@ headers = [ config_pub_h ] + files( 'fun.hh', 'git.hh', 'hash.hh', + 'hdiff-patch.hh', 'hilite.hh', 'json-impls.hh', 'json-non-null.hh', diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 43a805b4b514..889f9272f9fe 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -111,6 +111,10 @@ deps_private += brotli zstd = dependency('libzstd', version : '>= 1.4.0') deps_private += zstd +# For experimental HDiffPatch binary diffs. +hdiffpatch = dependency('hdiffpatch') +deps_private += hdiffpatch + cpuid_required = get_option('cpuid') if host_machine.cpu_family() != 'x86_64' and cpuid_required.enabled() warning('Force-enabling seccomp on non-x86_64 does not make sense') @@ -169,6 +173,7 @@ sources = [ config_priv_h ] + files( 'fs-sink.cc', 'git.cc', 'hash.cc', + 'hdiff-patch.cc', 'hilite.cc', 'json-utils.cc', 'logging.cc', diff --git a/src/libutil/package.nix b/src/libutil/package.nix index fe6fcb75f454..71bfc13d2b73 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -5,6 +5,7 @@ boost, brotli, + hdiffpatch, libarchive, libblake3, libcpuid, @@ -50,6 +51,7 @@ mkMesonLibrary (finalAttrs: { buildInputs = [ brotli + hdiffpatch libblake3 libsodium openssl diff --git a/src/nix/nario-export.md b/src/nix/nario-export.md index d35c31519a15..86217b89bee7 100644 --- a/src/nix/nario-export.md +++ b/src/nix/nario-export.md @@ -45,7 +45,7 @@ The flag `--diff-algorithm` determines how binary diffs are computed: * `zstd` (the default): The NAR is compressed using zstd, with the NAR of the base path as a dictionary. -* `hdiffpatch`: The diff is computed using the external `hdiffz` program from [HDiffPatch](https://github.com/sisong/HDiffPatch), and applied using `hpatchz`. These programs must be in `PATH` both when exporting and importing. This is experimental. +* `hdiffpatch`: The diff is computed using [HDiffPatch](https://github.com/sisong/HDiffPatch) and compressed using zstd. This is experimental. Binary diffs require nario format 2. Narios that contain binary diffs cannot be imported by versions of Nix that don't support them. diff --git a/src/nix/nario.cc b/src/nix/nario.cc index d1711540e80a..710af54e3a03 100644 --- a/src/nix/nario.cc +++ b/src/nix/nario.cc @@ -78,7 +78,7 @@ struct CmdNarioExport : StorePathsCommand addFlag({ .longName = "diff-algorithm", .description = - "Algorithm to use for computing binary diffs. Must be `zstd` (the default) or `hdiffpatch`. `hdiffpatch` is experimental and requires the `hdiffz` and `hpatchz` programs from HDiffPatch to be in `PATH` on export and import.", + "Algorithm to use for computing binary diffs. Must be `zstd` (the default) or `hdiffpatch`. `hdiffpatch` is experimental.", .labels = {"algorithm"}, .handler = {[this](std::string s) { diffAlgo = parseNarioDiffAlgo(s); }}, .completer = {[](AddCompletions & completions, size_t, std::string_view prefix) { diff --git a/tests/functional/package.nix b/tests/functional/package.nix index 6e94a8d48f5e..5204a52c0ff1 100644 --- a/tests/functional/package.nix +++ b/tests/functional/package.nix @@ -8,7 +8,6 @@ pkg-config, jq, - hdiffpatch, git, mercurial, python3, @@ -57,8 +56,6 @@ mkMesonDerivation ( pkg-config jq - # For `nix nario export --diff-algorithm hdiffpatch`. - hdiffpatch git mercurial # For the OpenTelemetry collector in `otel.sh`. diff --git a/tests/functional/test-runner.nix b/tests/functional/test-runner.nix index a5a5e8e8c030..c9ea0684632f 100644 --- a/tests/functional/test-runner.nix +++ b/tests/functional/test-runner.nix @@ -9,7 +9,6 @@ bash, coreutils, jq, - hdiffpatch, git, mercurial, unixtools, @@ -31,8 +30,6 @@ runCommand "nix-test-runner" bash coreutils jq - # For `nix nario export --diff-algorithm hdiffpatch`. - hdiffpatch git mercurial unixtools.script From a88bd5584a096f09ff00a8913f4f0bf04f6d97c6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 11 Oct 2026 16:56:00 +0200 Subject: [PATCH 12/13] hdiffpatch: Don't print progress messages to stdout HDiffPatch's diff code and its zstd compression plugin print progress messages (e.g. "create suffix string ...") to stdout. Since `nix nario export` writes the nario to stdout, these messages ended up in the nario. Because stdout is buffered, they were appended after the end marker when the process exited, so the narios could still be read, but they were larger than necessary (and not a multiple of 8 bytes). Build HDiffPatch with `_IS_OUT_DIFF_INFO=0` (also exported via the pkg-config file since it affects the headers), and disable the zstd plugin's notices with `IS_NOTICE_compress_canceled=0`. Assisted-by: Claude Opus 5.5 --- packaging/hdiffpatch.nix | 11 +++++++---- src/libutil/hdiff-patch.cc | 4 ++++ tests/functional/nario-diff.sh | 4 ++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/packaging/hdiffpatch.nix b/packaging/hdiffpatch.nix index 14135b914ee9..65e3847eee04 100644 --- a/packaging/hdiffpatch.nix +++ b/packaging/hdiffpatch.nix @@ -41,14 +41,17 @@ stdenv.mkDerivation (finalAttrs: { "MT=0" ]; - # The static library gets linked into a shared library. - env.NIX_CFLAGS_COMPILE = "-fPIC"; + # The static library gets linked into a shared library. Disable + # progress messages on stdout, which would end up in the output of + # programs like `nix nario export`. + env.NIX_CFLAGS_COMPILE = "-fPIC -D_IS_OUT_DIFF_INFO=0"; enableParallelBuilding = true; - # These must match the configuration selected by `makeFlags`, since - # they affect the headers. + # These must match the configuration selected by `makeFlags` and + # `NIX_CFLAGS_COMPILE`, since they affect the headers. cflags = lib.concatStringsSep " " [ + "-D_IS_OUT_DIFF_INFO=0" "-D_IS_USED_MULTITHREAD=0" "-DZ7_ST" "-D_IS_NEED_BSDIFF=0" diff --git a/src/libutil/hdiff-patch.cc b/src/libutil/hdiff-patch.cc index 44eb8c8067d1..26685746dbf1 100644 --- a/src/libutil/hdiff-patch.cc +++ b/src/libutil/hdiff-patch.cc @@ -5,6 +5,10 @@ #include #include + +/* Don't let the compression plugins print notices on stdout, which + would end up in the output of e.g. `nix nario export`. */ +#define IS_NOTICE_compress_canceled 0 #include #include diff --git a/tests/functional/nario-diff.sh b/tests/functional/nario-diff.sh index f10b375258a1..65fd227d6ebc 100755 --- a/tests/functional/nario-diff.sh +++ b/tests/functional/nario-diff.sh @@ -35,6 +35,10 @@ expectStderr 1 nix nario export --format 2 -r "$new" --base "$old" --diff-algori nix nario export --format 2 -r "$new" --base "$old" --diff-algorithm hdiffpatch > "$TEST_ROOT"/diff-hdiffpatch.nario nix nario list < "$TEST_ROOT"/diff-hdiffpatch.nario | grepQuiet "^$new: [0-9]* bytes, hdiffpatch diff against $old ([0-9]* bytes)$" [[ $(nix nario list --json < "$TEST_ROOT"/diff-hdiffpatch.nario | jq -r ".paths.\"$new\".diff.algorithm") = hdiffpatch ]] +# All fields in a nario are padded to 8 bytes, so anything else means +# that something (like HDiffPatch's progress messages) wrote junk to +# stdout. +(( $(stat -c %s "$TEST_ROOT"/diff-hdiffpatch.nario) % 8 == 0 )) # The diff should be much smaller than the full export. (( $(stat -c %s "$TEST_ROOT"/diff.nario) * 4 < $(stat -c %s "$TEST_ROOT"/full.nario) )) From ca1c83d4676f290f239c226c13b782af3d40c93c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 12 Oct 2026 12:32:16 +0200 Subject: [PATCH 13/13] nix nario export: Make hdiffpatch the default diff algorithm When exporting 35 generations of a NixOS system profile (a full export of the first generation plus a diff of each later generation against its predecessor), HDiffPatch produced 16% smaller diffs than zstd (1.26 GB vs 1.50 GB), took 1164s instead of 1957s (4112s vs 17562s CPU), and used 16.3 GB instead of 42.6 GB peak memory. zstd diffs are still available via `--diff-algorithm zstd`. Assisted-by: Claude Opus 5.5 --- .../include/nix/store/export-import.hh | 4 ++-- src/nix/nario-export.md | 4 ++-- src/nix/nario.cc | 7 +++--- tests/functional/nario-diff.sh | 23 ++++++++++--------- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/libstore/include/nix/store/export-import.hh b/src/libstore/include/nix/store/export-import.hh index cab9c7055981..84d8a3e5b7e1 100644 --- a/src/libstore/include/nix/store/export-import.hh +++ b/src/libstore/include/nix/store/export-import.hh @@ -32,7 +32,7 @@ enum class NarioDiffAlgo { /** * A zstd-compressed patch in HDiffPatch's single-stream format - * (see `makeHdiffPatch()`). Experimental. + * (see `makeHdiffPatch()`). */ hdiffpatch, }; @@ -65,7 +65,7 @@ struct NarioExportOptions /** * The algorithm used to compute binary diffs. */ - NarioDiffAlgo diffAlgo = NarioDiffAlgo::zstd; + NarioDiffAlgo diffAlgo = NarioDiffAlgo::hdiffpatch; /** * If not `none` (only supported for version 2), NARs that are not diff --git a/src/nix/nario-export.md b/src/nix/nario-export.md index 86217b89bee7..ff609cbcecb6 100644 --- a/src/nix/nario-export.md +++ b/src/nix/nario-export.md @@ -43,9 +43,9 @@ The `--base` flag specifies one or more *installables* whose closure (the *base The flag `--diff-algorithm` determines how binary diffs are computed: -* `zstd` (the default): The NAR is compressed using zstd, with the NAR of the base path as a dictionary. +* `hdiffpatch` (the default): The diff is computed using [HDiffPatch](https://github.com/sisong/HDiffPatch) and compressed using zstd. This generally produces the smallest diffs. -* `hdiffpatch`: The diff is computed using [HDiffPatch](https://github.com/sisong/HDiffPatch) and compressed using zstd. This is experimental. +* `zstd`: The NAR is compressed using zstd, with the NAR of the base path as a dictionary. Binary diffs require nario format 2. Narios that contain binary diffs cannot be imported by versions of Nix that don't support them. diff --git a/src/nix/nario.cc b/src/nix/nario.cc index 710af54e3a03..5528b3f3aef6 100644 --- a/src/nix/nario.cc +++ b/src/nix/nario.cc @@ -37,7 +37,7 @@ struct CmdNarioExport : StorePathsCommand unsigned int version = 0; std::vector baseArgs; BaseSelectionMethod baseSelectionMethod = BaseSelectionMethod::byName; - NarioDiffAlgo diffAlgo = NarioDiffAlgo::zstd; + NarioDiffAlgo diffAlgo = NarioDiffAlgo::hdiffpatch; CompressionAlgo compression = CompressionAlgo::none; std::optional compressionLevel; @@ -77,12 +77,11 @@ struct CmdNarioExport : StorePathsCommand addFlag({ .longName = "diff-algorithm", - .description = - "Algorithm to use for computing binary diffs. Must be `zstd` (the default) or `hdiffpatch`. `hdiffpatch` is experimental.", + .description = "Algorithm to use for computing binary diffs. Must be `hdiffpatch` (the default) or `zstd`.", .labels = {"algorithm"}, .handler = {[this](std::string s) { diffAlgo = parseNarioDiffAlgo(s); }}, .completer = {[](AddCompletions & completions, size_t, std::string_view prefix) { - for (auto algo : {NarioDiffAlgo::zstd, NarioDiffAlgo::hdiffpatch}) { + for (auto algo : {NarioDiffAlgo::hdiffpatch, NarioDiffAlgo::zstd}) { auto s = showNarioDiffAlgo(algo); if (s.starts_with(prefix)) completions.add(s); diff --git a/tests/functional/nario-diff.sh b/tests/functional/nario-diff.sh index 65fd227d6ebc..6996ef8d1406 100755 --- a/tests/functional/nario-diff.sh +++ b/tests/functional/nario-diff.sh @@ -27,18 +27,19 @@ nix nario export --format 2 -r "$new" --base "$old" --base-selection-method by-n cmp "$TEST_ROOT"/diff.nario "$TEST_ROOT"/diff-by-name.nario expectStderr 1 nix nario export --format 2 -r "$new" --base "$old" --base-selection-method foo | grepQuiet "unknown base selection method 'foo'" -# `zstd` is the default diff algorithm. -nix nario export --format 2 -r "$new" --base "$old" --diff-algorithm zstd > "$TEST_ROOT"/diff-zstd-algo.nario -cmp "$TEST_ROOT"/diff.nario "$TEST_ROOT"/diff-zstd-algo.nario +# `hdiffpatch` is the default diff algorithm. +nix nario export --format 2 -r "$new" --base "$old" --diff-algorithm hdiffpatch > "$TEST_ROOT"/diff-hdiffpatch.nario +cmp "$TEST_ROOT"/diff.nario "$TEST_ROOT"/diff-hdiffpatch.nario expectStderr 1 nix nario export --format 2 -r "$new" --base "$old" --diff-algorithm foo | grepQuiet "unsupported nario diff algorithm 'foo'" -nix nario export --format 2 -r "$new" --base "$old" --diff-algorithm hdiffpatch > "$TEST_ROOT"/diff-hdiffpatch.nario -nix nario list < "$TEST_ROOT"/diff-hdiffpatch.nario | grepQuiet "^$new: [0-9]* bytes, hdiffpatch diff against $old ([0-9]* bytes)$" -[[ $(nix nario list --json < "$TEST_ROOT"/diff-hdiffpatch.nario | jq -r ".paths.\"$new\".diff.algorithm") = hdiffpatch ]] # All fields in a nario are padded to 8 bytes, so anything else means # that something (like HDiffPatch's progress messages) wrote junk to # stdout. -(( $(stat -c %s "$TEST_ROOT"/diff-hdiffpatch.nario) % 8 == 0 )) +(( $(stat -c %s "$TEST_ROOT"/diff.nario) % 8 == 0 )) + +nix nario export --format 2 -r "$new" --base "$old" --diff-algorithm zstd > "$TEST_ROOT"/diff-zstd-algo.nario +nix nario list < "$TEST_ROOT"/diff-zstd-algo.nario | grepQuiet "^$new: [0-9]* bytes, zstd diff against $old ([0-9]* bytes)$" +[[ $(nix nario list --json < "$TEST_ROOT"/diff-zstd-algo.nario | jq -r ".paths.\"$new\".diff.algorithm") = zstd ]] # The diff should be much smaller than the full export. (( $(stat -c %s "$TEST_ROOT"/diff.nario) * 4 < $(stat -c %s "$TEST_ROOT"/full.nario) )) @@ -67,17 +68,17 @@ done nix nario export --format 2 --compression zstd -r "$new" --base "$old" > "$TEST_ROOT"/diff-zstd.nario (( $(stat -c %s "$TEST_ROOT"/diff-zstd.nario) < $(stat -c %s "$TEST_ROOT"/diff.nario) )) -nix nario list < "$TEST_ROOT"/diff-zstd.nario | grepQuiet "^$new: [0-9]* bytes, zstd diff against $old ([0-9]* bytes)$" +nix nario list < "$TEST_ROOT"/diff-zstd.nario | grepQuiet "^$new: [0-9]* bytes, hdiffpatch diff against $old ([0-9]* bytes)$" nix nario list < "$TEST_ROOT"/diff-zstd.nario | grepQuiet "^$extra: [0-9]* bytes, zstd-compressed ([0-9]* bytes)$" # Test `nix nario list`. -nix nario list < "$TEST_ROOT"/diff.nario | grepQuiet "^$new: [0-9]* bytes, zstd diff against $old ([0-9]* bytes)$" +nix nario list < "$TEST_ROOT"/diff.nario | grepQuiet "^$new: [0-9]* bytes, hdiffpatch diff against $old ([0-9]* bytes)$" nix nario list < "$TEST_ROOT"/diff.nario | grepQuiet "^$shared: expected to be present$" nix nario list < "$TEST_ROOT"/diff.nario | grepQuiet "^$extra: [0-9]* bytes$" nix nario list -R < "$TEST_ROOT"/diff.nario | grepQuiet "^$extra/data$" json=$(nix nario list --json < "$TEST_ROOT/diff.nario") -[[ $(printf "%s" "$json" | jq -r ".paths.\"$new\".diff.algorithm") = zstd ]] +[[ $(printf "%s" "$json" | jq -r ".paths.\"$new\".diff.algorithm") = hdiffpatch ]] [[ $(printf "%s" "$json" | jq -r ".paths.\"$new\".diff.base") = "$old" ]] [[ $(printf "%s" "$json" | jq -r ".paths.\"$new\".diff.baseNarHash") = $(nix path-info --json "$old" | jq -r ".[].narHash") ]] [[ $(printf "%s" "$json" | jq -r ".paths.\"$shared\".present") = true ]] @@ -121,7 +122,7 @@ nix store verify --no-trust -r "$new" clearStore nix nario import --no-check-sigs < "$TEST_ROOT"/old.nario -nix nario import --no-check-sigs < "$TEST_ROOT"/diff-hdiffpatch.nario +nix nario import --no-check-sigs < "$TEST_ROOT"/diff-zstd-algo.nario nix store verify --no-trust -r "$new" grepQuiet "^2$" "$new/data"