diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 169c68ffe266..3e34b3ff45cc 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -254,6 +254,8 @@ scope: { wasmtime = pkgs.callPackage ./wasmtime.nix { }; + hdiffpatch = pkgs.callPackage ./hdiffpatch.nix { }; + sentry-native = (pkgs.callPackage ./sentry-native.nix { }).override { # Avoid having two curls in our closure. inherit (scope) curl; diff --git a/packaging/hdiffpatch.nix b/packaging/hdiffpatch.nix new file mode 100644 index 000000000000..65e3847eee04 --- /dev/null +++ b/packaging/hdiffpatch.nix @@ -0,0 +1,100 @@ +# HDiffPatch (https://github.com/sisong/HDiffPatch), used for +# experimental `nix nario export --diff-algorithm hdiffpatch`. +# +# Upstream doesn't install a library, so we install the static library +# that its Makefile builds, its headers, and a pkg-config file. +{ + lib, + stdenv, + fetchFromGitHub, + zstd, +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "hdiffpatch"; + version = "5.1.3"; + + src = fetchFromGitHub { + owner = "sisong"; + repo = "HDiffPatch"; + tag = "v${finalAttrs.version}"; + hash = "sha256-8bDefNdKO9bLJzENY1OEap7tTSzI7C/oUECgXBJSiEE="; + }; + + buildInputs = [ zstd ]; + + # Disable everything that requires source checkouts of other + # projects next to the HDiffPatch source tree, and link against zstd + # from Nixpkgs. Disable multi-threading, since Nix already computes + # diffs in parallel. + makeFlags = [ + "ZSTD=2" + "LDEF=0" + "ZLIB=0" + "LZMA=0" + "MD5=0" + "XXH=0" + "VCD=0" + "BSD=0" + "BZIP2=0" + "DIR_DIFF=0" + "MT=0" + ]; + + # The static library gets linked into a shared library. Disable + # progress messages on stdout, which would end up in the output of + # programs like `nix nario export`. + env.NIX_CFLAGS_COMPILE = "-fPIC -D_IS_OUT_DIFF_INFO=0"; + + enableParallelBuilding = true; + + # These must match the configuration selected by `makeFlags` and + # `NIX_CFLAGS_COMPILE`, since they affect the headers. + cflags = lib.concatStringsSep " " [ + "-D_IS_OUT_DIFF_INFO=0" + "-D_IS_USED_MULTITHREAD=0" + "-DZ7_ST" + "-D_IS_NEED_BSDIFF=0" + "-D_IS_NEED_VCDIFF=0" + "-D_IS_NEED_DIR_DIFF_PATCH=0" + "-D_IS_NEED_ALL_CompressPlugin=0" + "-D_IS_NEED_DEFAULT_CompressPlugin=0" + "-D_IS_NEED_ALL_ChecksumPlugin=0" + "-D_IS_NEED_DEFAULT_ChecksumPlugin=0" + "-D_CompressPlugin_zstd" + ]; + + installPhase = '' + runHook preInstall + + install -D -m 644 libhdiffpatch.a $out/lib/libhdiffpatch.a + install -D -m 755 hdiffz $out/bin/hdiffz + install -D -m 755 hpatchz $out/bin/hpatchz + + # The headers refer to each other relative to the root of the + # source tree, so keep that structure. + find . -name '*.h' -exec install -D -m 644 {} $out/include/hdiffpatch/{} \; + + mkdir -p $out/lib/pkgconfig + cat > $out/lib/pkgconfig/hdiffpatch.pc < namespace nix { static const uint32_t exportMagicV1 = 0x4558494e; static const uint64_t exportMagicV2 = 0x324f4952414e; // = 'NARIO2' +/* Entry tags in version 2 narios. */ +static const uint64_t narioTagEnd = 0; +static const uint64_t narioTagFull = 1; +static const uint64_t narioTagDiff = 2; +static const uint64_t narioTagPresent = 3; +static const uint64_t narioTagCompressed = 4; + +/* Binary diff generation parameters. */ + +/* NARs larger than this are not diffed. */ +static const uint64_t maxDiffNarSize = 1ULL << 30; + +/* If a base differs in NAR size from the target by more than this + factor, it's not considered as a diff base. This disambiguates + e.g. between a package and a wrapper of that package. */ +static const double maxDiffSizeRatio = 3.0; + +/* A diff is only used if it's smaller than this fraction of the + standalone zstd compression of the target NAR. Applying a zstd + patch is cheap, so even a modest saving is worth it. */ +static const double maxDiffFraction = 0.9; + +static const int diffCompressionLevel = 19; + +/* The default zstd compression level for full NARs. Also used to + decide whether a diff is worth it if full NARs are not compressed + using zstd. */ +static const int defaultZstdCompressionLevel = 9; + +/* Compressed NARs larger than this are buffered on disk rather than + in memory. */ +static const size_t maxInMemoryCompressedNarSize = 32 * 1024 * 1024; + +/* When exporting in parallel, don't start new diffs or compressions + while the entries that have been computed but not yet written + exceed this size. */ +static const size_t maxBufferedEntriesSize = 1024 * 1024 * 1024; + static WorkerProto::Version exportProtoVersion{ .number = { @@ -19,27 +66,281 @@ static WorkerProto::Version exportProtoVersion{ }, }; -void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigned int version) +static void checkNarHash(Store & store, const ValidPathInfo & info, const Hash & hash) +{ + /* Refuse to export paths that have changed. This prevents + filesystem corruption from spreading to other machines. + Don't complain if the stored hash is zero (unknown). */ + if (hash != info.narHash && info.narHash != Hash(info.narHash.algo)) + throw Error( + "hash of path '%s' has changed from '%s' to '%s'!", + store.printStorePath(info.path), + info.narHash.to_string(HashFormat::Nix32, true), + hash.to_string(HashFormat::Nix32, true)); +} + +/** + * Return the key used to match store paths against potential diff + * bases, namely the package name without the version, and the output + * name, e.g. `openssl-3.0.13-dev` yields (`openssl`, `dev`). + */ +static std::pair getDiffKey(std::string_view name) +{ + DrvName drvName(name); + std::string outputName; + auto i = drvName.version.rfind('-'); + if (i != std::string::npos && i + 1 < drvName.version.size() && isalpha(drvName.version[i + 1])) + outputName = drvName.version.substr(i + 1); + return {drvName.name, outputName}; +} + +/** + * Return the length of the common prefix of two version strings, as + * a crude measure of their similarity. + */ +static size_t versionSimilarity(std::string_view v1, std::string_view v2) +{ + return std::mismatch(v1.begin(), v1.end(), v2.begin(), v2.end()).first - v1.begin(); +} + +static double sizeRatio(uint64_t a, uint64_t b) +{ + if (a == 0 || b == 0) + return a == b ? 1.0 : std::numeric_limits::infinity(); + return a > b ? (double) a / b : (double) b / a; +} + +std::string showNarioDiffAlgo(NarioDiffAlgo algo) +{ + switch (algo) { + case NarioDiffAlgo::zstd: + return "zstd"; + case NarioDiffAlgo::hdiffpatch: + return "hdiffpatch"; + } + unreachable(); +} + +NarioDiffAlgo parseNarioDiffAlgo(std::string_view s) +{ + if (s == "zstd") + return NarioDiffAlgo::zstd; + if (s == "hdiffpatch") + return NarioDiffAlgo::hdiffpatch; + throw Error("unsupported nario diff algorithm '%s'", s); +} + +struct NarioDiff +{ + NarioDiffAlgo algo; + StorePath basePath; + Hash baseNarHash; + std::string patch; +}; + +std::string showBaseSelectionMethod(BaseSelectionMethod method) +{ + switch (method) { + case BaseSelectionMethod::byName: + return "by-name"; + } + unreachable(); +} + +BaseSelectionMethod parseBaseSelectionMethod(std::string_view s) +{ + if (s == "by-name") + return BaseSelectionMethod::byName; + throw Error("unknown base selection method '%s'", s); +} + +/** + * The `by-name` base selection method: select a base path with the + * same name (ignoring the version), preferring the one with the most + * similar version and NAR size. + */ +static std::map +selectDiffBasesByName(Store & store, const StorePaths & targets, const StorePaths & bases) +{ + std::map, std::vector> basesByKey; + for (auto & basePath : bases) + basesByKey[getDiffKey(basePath.name())].push_back(basePath); + + std::map res; + + for (auto & path : targets) { + auto i = basesByKey.find(getDiffKey(path.name())); + if (i == basesByKey.end()) + continue; + + auto info = store.queryPathInfo(path); + + auto version = DrvName(path.name()).version; + + std::optional bestPath; + size_t bestVersionSimilarity = 0; + double bestSizeRatio = 0; + + for (auto & basePath : i->second) { + auto baseInfo = store.queryPathInfo(basePath); + auto ratio = sizeRatio(info->narSize, baseInfo->narSize); + if (ratio >= maxDiffSizeRatio) + continue; + auto similarity = versionSimilarity(version, DrvName(basePath.name()).version); + if (!bestPath || similarity > bestVersionSimilarity + || (similarity == bestVersionSimilarity && ratio < bestSizeRatio)) { + bestPath = basePath; + bestVersionSimilarity = similarity; + bestSizeRatio = ratio; + } + } + + if (bestPath) + res.emplace(path, *bestPath); + } + + return res; +} + +/** + * Select the path in `baseClosure` that is most likely to produce a + * small diff against each path in `paths` that is not in + * `baseClosure`, using the base selection method `method`. + */ +static std::map selectDiffBases( + Store & store, + BaseSelectionMethod method, + NarioDiffAlgo diffAlgo, + const StorePaths & paths, + const StorePathSet & baseClosure) +{ + /* Constraints that apply regardless of the selection method. */ + auto isDiffable = [&](const StorePath & path) { return store.queryPathInfo(path)->narSize <= maxDiffNarSize; }; + + StorePaths targets; + for (auto & path : paths) + if (!baseClosure.contains(path) && isDiffable(path)) + targets.push_back(path); + + StorePaths bases; + for (auto & path : baseClosure) + if (isDiffable(path)) + bases.push_back(path); + + std::map res; + + switch (method) { + case BaseSelectionMethod::byName: + res = selectDiffBasesByName(store, targets, bases); + break; + } + + /* zstd patches can only refer to the base if base and target + fit in the zstd window together. */ + if (diffAlgo == NarioDiffAlgo::zstd) + std::erase_if(res, [&](auto & pair) { + return store.queryPathInfo(pair.first)->narSize + store.queryPathInfo(pair.second)->narSize + > maxZstdPatchWindow(); + }); + + return res; +} + +/** + * Compute a binary diff between the NARs of `basePath` and `path`. + * Returns `std::nullopt` if the diff is not small enough to be worth + * it. + */ +static std::optional makeDiff( + Store & store, + const StorePath & path, + const StorePath & basePath, + NarioDiffAlgo diffAlgo, + int baselineCompressionLevel) +{ + Activity act( + *logger, + lvlTalkative, + actUnknown, + fmt("computing diff of '%s' against '%s'", store.printStorePath(path), store.printStorePath(basePath))); + + auto getNar = [&](const ValidPathInfo & info) { + StringSink nar; + store.narFromPath(info.path, nar); + auto hash = hashString(HashAlgorithm::SHA256, nar.s); + checkNarHash(store, info, hash); + return std::pair{std::move(nar.s), hash}; + }; + + auto [baseNar, baseNarHash] = getNar(*store.queryPathInfo(basePath)); + auto [nar, narHash] = getNar(*store.queryPathInfo(path)); + + auto narSize = nar.size(); + auto baselineSize = makeZstdPatch("", nar, baselineCompressionLevel).size(); + + std::string patch; + switch (diffAlgo) { + case NarioDiffAlgo::zstd: + patch = makeZstdPatch(baseNar, nar, diffCompressionLevel); + break; + case NarioDiffAlgo::hdiffpatch: + /* This overwrites the NARs, so we can't use them anymore. */ + patch = makeHdiffPatch(std::move(baseNar), std::move(nar), diffCompressionLevel); + break; + } + + printMsg( + lvlTalkative, + "diff of '%s' against '%s': %d bytes (NAR %d bytes, compressed %d bytes)", + store.printStorePath(path), + store.printStorePath(basePath), + patch.size(), + narSize, + baselineSize); + + if (patch.size() >= maxDiffFraction * baselineSize) + return std::nullopt; + + return NarioDiff{ + .algo = diffAlgo, + .basePath = basePath, + .baseNarHash = baseNarHash, + .patch = std::move(patch), + }; +} + +void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, const NarioExportOptions & options) { + auto version = options.version; + auto & basePaths = options.basePaths; + auto compression = options.compression; + auto compressionLevel = + options.compressionLevel.value_or(compression == CompressionAlgo::zstd ? defaultZstdCompressionLevel : -1); + auto sorted = store.topoSortPaths(paths); std::reverse(sorted.begin(), sorted.end()); - auto dumpNar = [&](const ValidPathInfo & info) { + if (!basePaths.empty() && version != 2) + throw Error("binary diffs are only supported in nario version 2"); + + if (compression != CompressionAlgo::none && version != 2) + throw Error("compression is only supported in nario version 2"); + + if (options.compressionLevel && compression == CompressionAlgo::none) + throw Error("a compression level requires a compression method"); + + /* A diff must be smaller than the full NAR compressed with zstd at + the level we would otherwise use. */ + auto baselineCompressionLevel = + compression == CompressionAlgo::zstd ? compressionLevel : defaultZstdCompressionLevel; + + auto dumpNar = [&](const ValidPathInfo & info, Sink & sink) { HashSink hashSink(HashAlgorithm::SHA256); TeeSink teeSink(sink, hashSink); store.narFromPath(info.path, teeSink); - /* Refuse to export paths that have changed. This prevents - filesystem corruption from spreading to other machines. - Don't complain if the stored hash is zero (unknown). */ - Hash hash = hashSink.currentHash().hash; - if (hash != info.narHash && info.narHash != Hash(info.narHash.algo)) - throw Error( - "hash of path '%s' has changed from '%s' to '%s'!", - store.printStorePath(info.path), - info.narHash.to_string(HashFormat::Nix32, true), - hash.to_string(HashFormat::Nix32, true)); + checkNarHash(store, info, hashSink.currentHash().hash); }; switch (version) { @@ -48,7 +349,7 @@ void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigne for (auto & path : sorted) { sink << 1; auto info = store.queryPathInfo(path); - dumpNar(*info); + dumpNar(*info, sink); sink << exportMagicV1 << store.printStorePath(path); CommonProto::write(store, CommonProto::WriteConn{.to = sink}, info->references); sink << (info->deriver ? store.printStorePath(*info->deriver) : "") << 0; @@ -56,33 +357,134 @@ void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigne sink << 0; break; - case 2: - sink << exportMagicV2; + case 2: { + StorePathSet baseClosure; + std::map diffBases; - for (auto & path : sorted) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("exporting path '%s'", store.printStorePath(path))); - sink << 1; - auto info = store.queryPathInfo(path); - // FIXME: move to CommonProto? - WorkerProto::Serialise::write( - store, - WorkerProto::WriteConn{.to = sink, .version = exportProtoVersion, .shortStorePaths = true}, - *info); - dumpNar(*info); + if (!basePaths.empty()) { + store.computeFSClosure(basePaths, baseClosure); + diffBases = selectDiffBases(store, options.baseSelectionMethod, options.diffAlgo, sorted, baseClosure); } - sink << 0; + /* The expensive part of an entry (a binary diff or a + compressed NAR), computed in parallel. If neither is set, + the path is either present in the base closure or written + as a full NAR. */ + struct ExportEntry + { + std::optional diff; + std::unique_ptr compressed; + }; + + /* The indices in `sorted` of the paths that need expensive + work. Only these are processed by the thread pool; cheap + entries (e.g. paths in the base closure) are written in + between. */ + std::vector jobs; + for (size_t i = 0; i < sorted.size(); ++i) + if (!baseClosure.contains(sorted[i]) + && (diffBases.contains(sorted[i]) || compression != CompressionAlgo::none)) + jobs.push_back(i); + + WorkerProto::WriteConn conn{.to = sink, .version = exportProtoVersion, .shortStorePaths = true}; + + auto writeEntry = [&](size_t i, ExportEntry && entry) { + auto & path = sorted[i]; + auto info = store.queryPathInfo(path); + + if (baseClosure.contains(path)) { + sink << narioTagPresent; + WorkerProto::write(store, conn, *info); + } + + else if (entry.diff) { + sink << narioTagDiff; + WorkerProto::write(store, conn, *info); + sink << showNarioDiffAlgo(entry.diff->algo); + WorkerProto::write(store, conn, entry.diff->basePath); + sink << entry.diff->baseNarHash.to_string(HashFormat::SRI, true) << entry.diff->patch; + } + + else if (entry.compressed) { + sink << narioTagCompressed; + WorkerProto::write(store, conn, *info); + sink << showCompressionAlgo(compression) << entry.compressed->size(); + entry.compressed->getSource()->drainInto(sink); + writePadding(entry.compressed->size(), sink); + } + + else { + Activity act(*logger, lvlTalkative, actUnknown, fmt("exporting path '%s'", store.printStorePath(path))); + sink << narioTagFull; + // FIXME: move to CommonProto? + WorkerProto::write(store, conn, *info); + dumpNar(*info, sink); + } + }; + + /* Write the cheap entries preceding index `end`. */ + size_t nextToWrite = 0; + auto writeCheapEntries = [&](size_t end) { + for (; nextToWrite < end; ++nextToWrite) + writeEntry(nextToWrite, {}); + }; + + sink << exportMagicV2; + + processOrdered( + jobs.size(), + [&](size_t j) { + auto & path = sorted[jobs[j]]; + ExportEntry entry; + + auto basePath = get(diffBases, path); + + Activity act(*logger, lvlTalkative, actUnknown, fmt("exporting path '%s'", store.printStorePath(path))); + + if (basePath) { + entry.diff = makeDiff(store, path, *basePath, options.diffAlgo, baselineCompressionLevel); + if (entry.diff) + return entry; + } + + if (compression != CompressionAlgo::none) { + /* We need to know the size of the compressed NAR + before writing it, so buffer it in memory or on + disk. */ + entry.compressed = std::make_unique(maxInMemoryCompressedNarSize); + auto compressionSink = makeCompressionSink(compression, *entry.compressed, true, compressionLevel); + dumpNar(*store.queryPathInfo(path), *compressionSink); + compressionSink->finish(); + } + + return entry; + }, + [&](size_t j, ExportEntry && entry) { + writeCheapEntries(jobs[j]); + writeEntry(jobs[j], std::move(entry)); + nextToWrite++; + }, + [](const ExportEntry & entry) -> size_t { + return entry.diff ? entry.diff->patch.size() : entry.compressed ? entry.compressed->size() : 0; + }, + maxBufferedEntriesSize, + 12); + + writeCheapEntries(sorted.size()); + + sink << narioTagEnd; break; + } default: throw Error("unsupported nario version %d", version); } } -StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) -{ - StorePaths res; +NarioVisitor::~NarioVisitor() = default; +void parseNario(Store & store, Source & source, NarioVisitor & visitor) +{ auto version = readNum(source); /* Note: nario version 1 lacks an explicit header. The first @@ -113,8 +515,6 @@ StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) auto path = store.parseStorePath(readString(source)); - Activity act(*logger, lvlTalkative, actUnknown, fmt("importing path '%s'", store.printStorePath(path))); - auto references = CommonProto::Serialise::read(store, CommonProto::ReadConn{.from = source}); auto deriver = readString(source); @@ -122,21 +522,17 @@ StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) if (readInt(source) == 1) readString(source); - if (!store.isValidPath(path)) { - auto narHash = hashString(HashAlgorithm::SHA256, saved.s); + auto narHash = hashString(HashAlgorithm::SHA256, saved.s); - ValidPathInfo info{path, {store, narHash}}; - if (deriver != "") - info.deriver = store.parseStorePath(deriver); - info.references = references; - info.narSize = saved.s.size(); + ValidPathInfo info{path, {store, narHash}}; + if (deriver != "") + info.deriver = store.parseStorePath(deriver); + info.references = references; + info.narSize = saved.s.size(); - // Can't use underlying source, which would have been exhausted. - auto source2 = StringSource(saved.s); - store.addToStore(info, source2, NoRepair, checkSigs); - } - - res.push_back(path); + // Can't use underlying source, which would have been exhausted. + auto source2 = StringSource(saved.s); + visitor.fullPath(info, source2, std::nullopt); auto n = readNum(source); if (n == 0) @@ -147,33 +543,167 @@ StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) break; } - case exportMagicV2: + case exportMagicV2: { + WorkerProto::ReadConn conn{.from = source, .version = exportProtoVersion, .shortStorePaths = true}; + while (true) { - auto n = readNum(source); - if (n == 0) + auto tag = readNum(source); + if (tag == narioTagEnd) break; - if (n != 1) + + if (tag != narioTagFull && tag != narioTagDiff && tag != narioTagPresent && tag != narioTagCompressed) throw Error("input doesn't look like a nario"); - auto info = WorkerProto::Serialise::read( - store, WorkerProto::ReadConn{.from = source, .version = exportProtoVersion, .shortStorePaths = true}); + auto info = WorkerProto::Serialise::read(store, conn); - Activity act( - *logger, lvlTalkative, actUnknown, fmt("importing path '%s'", store.printStorePath(info.path))); + if (tag == narioTagFull) { + EnsureRead wrapper{source, info.narSize}; + visitor.fullPath(info, wrapper, std::nullopt); + } - EnsureRead wrapper{source, info.narSize}; - store.addToStore(info, wrapper, NoRepair, checkSigs); + else if (tag == narioTagCompressed) { + auto algo = parseCompressionAlgo(readString(source)); + auto size = readNum(source); + + /* Stream the compressed data through a decompressor, + keeping track of how much is left so we can skip + it if the visitor doesn't read the whole NAR. */ + uint64_t remaining = size; + { + auto nar = sinkToSource([&](Sink & sink) { + auto decompressionSink = makeDecompressionSink(algo, sink); + std::vector buf(65536); + while (remaining) { + auto n = std::min(remaining, buf.size()); + source(buf.data(), n); + remaining -= n; + (*decompressionSink)({buf.data(), n}); + } + decompressionSink->finish(); + }); + visitor.fullPath(info, *nar, NarioCompression{.algo = algo, .size = size}); + } + source.skip(remaining); + readPadding(size, source); + } - res.push_back(info.path); + else if (tag == narioTagDiff) { + auto algo = parseNarioDiffAlgo(readString(source)); + auto basePath = WorkerProto::Serialise::read(store, conn); + auto baseNarHash = Hash::parseAnyPrefixed(readString(source)); + auto patch = readString(source); + visitor.diffPath(info, algo, basePath, baseNarHash, patch); + } + + else + visitor.presentPath(info); } break; + } default: throw Error("input doesn't look like a nario"); } +} - return res; +StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) +{ + struct Importer : NarioVisitor + { + Store & store; + CheckSigsFlag checkSigs; + StorePaths res; + + Importer(Store & store, CheckSigsFlag checkSigs) + : store(store) + , checkSigs(checkSigs) + { + } + + void fullPath(const ValidPathInfo & info, Source & nar, std::optional compression) override + { + if (!store.isValidPath(info.path)) { + Activity act( + *logger, lvlTalkative, actUnknown, fmt("importing path '%s'", store.printStorePath(info.path))); + + store.addToStore(info, nar, NoRepair, checkSigs); + } + + res.push_back(info.path); + } + + void diffPath( + const ValidPathInfo & info, + NarioDiffAlgo algo, + const StorePath & basePath, + const Hash & baseNarHash, + std::string_view patch) override + { + if (!store.isValidPath(info.path)) { + Activity act( + *logger, + lvlTalkative, + actUnknown, + fmt("importing path '%s' by patching '%s'", + store.printStorePath(info.path), + store.printStorePath(basePath))); + + if (!store.isValidPath(basePath)) + throw Error( + "cannot import path '%s' because it's a binary diff against path '%s', which is not valid", + store.printStorePath(info.path), + store.printStorePath(basePath)); + + StringSink baseNar; + store.narFromPath(basePath, baseNar); + + auto actualBaseNarHash = hashString(baseNarHash.algo, baseNar.s); + if (actualBaseNarHash != baseNarHash) + throw Error( + "cannot import path '%s' because it's a binary diff against path '%s', which has NAR hash '%s' instead of the expected '%s'", + store.printStorePath(info.path), + store.printStorePath(basePath), + actualBaseNarHash.to_string(HashFormat::SRI, true), + baseNarHash.to_string(HashFormat::SRI, true)); + + auto nar = sinkToSource([&](Sink & sink) { + switch (algo) { + case NarioDiffAlgo::zstd: + applyZstdPatch(baseNar.s, patch, sink); + break; + case NarioDiffAlgo::hdiffpatch: + applyHdiffPatch(baseNar.s, patch, sink); + break; + } + }); + store.addToStore(info, *nar, NoRepair, checkSigs); + } + + res.push_back(info.path); + } + + void presentPath(const ValidPathInfo & info) override + { + auto localInfo = store.maybeQueryPathInfo(info.path); + if (!localInfo) + throw Error( + "nario requires path '%s' to already be valid in the store", store.printStorePath(info.path)); + + if (localInfo->narHash != info.narHash && localInfo->narHash != Hash(localInfo->narHash.algo)) + throw Error( + "path '%s' in the store has NAR hash '%s', but the nario expects '%s'", + store.printStorePath(info.path), + localInfo->narHash.to_string(HashFormat::SRI, true), + info.narHash.to_string(HashFormat::SRI, true)); + + res.push_back(info.path); + } + }; + + Importer importer(store, checkSigs); + parseNario(store, source, importer); + return std::move(importer.res); } } // namespace nix diff --git a/src/libstore/include/nix/store/export-import.hh b/src/libstore/include/nix/store/export-import.hh index 4ea696f992f9..84d8a3e5b7e1 100644 --- a/src/libstore/include/nix/store/export-import.hh +++ b/src/libstore/include/nix/store/export-import.hh @@ -1,14 +1,144 @@ #pragma once #include "nix/store/store-api.hh" +#include "nix/util/compression-algo.hh" namespace nix { +/** + * Methods for selecting the base path against which to diff a path + * being exported. + */ +enum class BaseSelectionMethod { + /** + * Select a base path with the same name (ignoring the version). + */ + byName, +}; + +std::string showBaseSelectionMethod(BaseSelectionMethod method); + +BaseSelectionMethod parseBaseSelectionMethod(std::string_view s); + +/** + * Binary diff algorithms supported in narios. + */ +enum class NarioDiffAlgo { + /** + * zstd compression of the target NAR, using the base NAR as a + * prefix dictionary (see `makeZstdPatch()`). + */ + zstd, + + /** + * A zstd-compressed patch in HDiffPatch's single-stream format + * (see `makeHdiffPatch()`). + */ + hdiffpatch, +}; + +std::string showNarioDiffAlgo(NarioDiffAlgo algo); + +NarioDiffAlgo parseNarioDiffAlgo(std::string_view s); + +struct NarioExportOptions +{ + /** + * The nario format version (1 or 2). + */ + unsigned int version; + + /** + * If non-empty (only supported for version 2), the receiver is + * assumed to have the closure of these paths. Paths in that + * closure are not exported (but recorded as being expected to be + * present), and other paths are exported as binary diffs against + * a path in that closure if a suitable one is found. + */ + StorePathSet basePaths; + + /** + * How to find suitable base paths. + */ + BaseSelectionMethod baseSelectionMethod = BaseSelectionMethod::byName; + + /** + * The algorithm used to compute binary diffs. + */ + NarioDiffAlgo diffAlgo = NarioDiffAlgo::hdiffpatch; + + /** + * If not `none` (only supported for version 2), NARs that are not + * exported as binary diffs are compressed. + */ + CompressionAlgo compression = CompressionAlgo::none; + + /** + * The compression level to use for `compression`. If unset, the + * default is 9 for zstd, and the compression method's default + * otherwise. + */ + std::optional compressionLevel; +}; + /** * Export multiple paths in the format expected by `nix-store * --import`. The paths will be sorted topologically. */ -void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigned int version); +void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, const NarioExportOptions & options); + +/** + * How a NAR is compressed inside a nario. + */ +struct NarioCompression +{ + CompressionAlgo algo; + + /** + * Size of the compressed NAR. + */ + uint64_t size; +}; + +/** + * Callbacks for the entries in a nario, used by `parseNario()`. + */ +struct NarioVisitor +{ + virtual ~NarioVisitor(); + + /** + * A path whose NAR is contained in the nario. The visitor must + * read exactly `info.narSize` bytes from `nar`. If the NAR is + * stored in compressed form, `compression` describes how; `nar` + * returns the decompressed NAR in any case. + */ + virtual void fullPath(const ValidPathInfo & info, Source & nar, std::optional compression) = 0; + + /** + * A path whose NAR is given as a binary diff (computed using + * `algo`) against the NAR of `basePath`, which is expected to + * have NAR hash `baseNarHash`. + */ + virtual void diffPath( + const ValidPathInfo & info, + NarioDiffAlgo algo, + const StorePath & basePath, + const Hash & baseNarHash, + std::string_view patch) = 0; + + /** + * A path that is not contained in the nario, but that is expected + * to already be present in the destination store. + */ + virtual void presentPath(const ValidPathInfo & info) = 0; +}; + +/** + * Parse a nario created by `exportPaths()`, calling `visitor` for + * each entry. + */ +void parseNario(Store & store, Source & source, NarioVisitor & visitor); /** * Import a sequence of NAR dumps created by `exportPaths()` into the diff --git a/src/libutil-tests/hdiff-patch.cc b/src/libutil-tests/hdiff-patch.cc new file mode 100644 index 000000000000..b6a3b57095b1 --- /dev/null +++ b/src/libutil-tests/hdiff-patch.cc @@ -0,0 +1,96 @@ +#include "nix/util/hdiff-patch.hh" +#include "nix/util/compression.hh" + +#include + +#include + +namespace nix { + +static std::string randomHdiffString(size_t size, uint64_t seed) +{ + std::mt19937_64 rng(seed); + std::string s(size, '\0'); + for (auto & c : s) + c = (char) rng(); + return s; +} + +static std::string applyHdiffPatchToString(std::string_view base, std::string_view patch) +{ + StringSink sink; + applyHdiffPatch(base, patch, sink); + return std::move(sink.s); +} + +TEST(hdiffPatch, roundtrip) +{ + auto base = randomHdiffString(1 << 20, 1); + auto target = base; + /* Make some small changes. */ + target.replace(1000, 5, "hello"); + target.insert(500000, "inserted"); + target.erase(800000, 100); + + auto patch = makeHdiffPatch(base, target, 19); + ASSERT_EQ(applyHdiffPatchToString(base, patch), target); + + /* The base is random, so the patch is only small because it + refers to the base. */ + ASSERT_LT(patch.size(), 1000); +} + +TEST(hdiffPatch, emptyBase) +{ + auto target = randomHdiffString(10000, 2); + auto patch = makeHdiffPatch("", target, 3); + ASSERT_EQ(applyHdiffPatchToString("", patch), target); +} + +TEST(hdiffPatch, emptyTarget) +{ + auto base = randomHdiffString(10000, 3); + auto patch = makeHdiffPatch(base, "", 3); + ASSERT_EQ(applyHdiffPatchToString(base, patch), ""); +} + +TEST(hdiffPatch, wrongBase) +{ + auto base = randomHdiffString(100000, 4); + auto target = base; + target.replace(50000, 5, "hello"); + auto patch = makeHdiffPatch(base, target, 19); + + /* Applying to the wrong base produces garbage (or an error), so + callers must check the base's hash. */ + auto otherBase = randomHdiffString(100000, 5); + try { + ASSERT_NE(applyHdiffPatchToString(otherBase, patch), target); + } catch (CompressionError &) { + } + + /* A base of the wrong size is rejected. */ + ASSERT_THROW(applyHdiffPatchToString(base.substr(1), patch), CompressionError); +} + +TEST(hdiffPatch, truncated) +{ + auto base = randomHdiffString(100000, 6); + auto target = randomHdiffString(100000, 7); + auto patch = makeHdiffPatch(base, target, 3); + ASSERT_THROW(applyHdiffPatchToString(base, patch.substr(0, patch.size() / 2)), CompressionError); +} + +TEST(hdiffPatch, trailingGarbage) +{ + auto base = randomHdiffString(1000, 8); + auto patch = makeHdiffPatch(base, base, 3); + ASSERT_THROW(applyHdiffPatchToString(base, patch + "garbage"), CompressionError); +} + +TEST(hdiffPatch, garbage) +{ + ASSERT_THROW(applyHdiffPatchToString("", "not a patch"), CompressionError); +} + +} // namespace nix diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index f7004c1c160d..195b09ef41e6 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -76,6 +76,7 @@ sources = files( 'fun.cc', 'git.cc', 'hash.cc', + 'hdiff-patch.cc', 'hilite.cc', 'json-utils.cc', 'local-keys.cc', @@ -95,13 +96,16 @@ sources = files( 'sort.cc', 'source-accessor.cc', 'spawn.cc', + 'spilling-sink.cc', 'strings.cc', 'suggestions.cc', 'terminal.cc', + 'thread-pool.cc', 'topo-sort.cc', 'url.cc', 'util.cc', 'xml-writer.cc', + 'zstd-patch.cc', ) if have_cxa_throw diff --git a/src/libutil-tests/spilling-sink.cc b/src/libutil-tests/spilling-sink.cc new file mode 100644 index 000000000000..49259a9aa7c1 --- /dev/null +++ b/src/libutil-tests/spilling-sink.cc @@ -0,0 +1,56 @@ +#include "nix/util/spilling-sink.hh" + +#include + +namespace nix { + +TEST(SpillingStringSink, belowLimit) +{ + SpillingStringSink sink(16); + sink("hello"); + sink(" world"); + ASSERT_FALSE(sink.isSpilled()); + ASSERT_EQ(sink.size(), 11); + ASSERT_EQ(sink.getSource()->drain(), "hello world"); +} + +TEST(SpillingStringSink, atLimit) +{ + SpillingStringSink sink(8); + sink("0123"); + sink("4567"); + ASSERT_FALSE(sink.isSpilled()); + ASSERT_EQ(sink.getSource()->drain(), "01234567"); +} + +TEST(SpillingStringSink, empty) +{ + SpillingStringSink sink(8); + ASSERT_FALSE(sink.isSpilled()); + ASSERT_EQ(sink.size(), 0); + ASSERT_EQ(sink.getSource()->drain(), ""); +} + +TEST(SpillingStringSink, spills) +{ + SpillingStringSink sink(1000); + std::string expected; + for (int i = 0; i < 10000; ++i) { + auto s = fmt("line %d\n", i); + sink(s); + expected += s; + } + ASSERT_TRUE(sink.isSpilled()); + ASSERT_EQ(sink.size(), expected.size()); + ASSERT_EQ(sink.getSource()->drain(), expected); +} + +TEST(SpillingStringSink, spillsOnFirstWrite) +{ + SpillingStringSink sink(4); + sink("0123456789"); + ASSERT_TRUE(sink.isSpilled()); + ASSERT_EQ(sink.getSource()->drain(), "0123456789"); +} + +} // namespace nix diff --git a/src/libutil-tests/thread-pool.cc b/src/libutil-tests/thread-pool.cc new file mode 100644 index 000000000000..c336c66dcdbc --- /dev/null +++ b/src/libutil-tests/thread-pool.cc @@ -0,0 +1,141 @@ +#include "nix/util/thread-pool.hh" + +#include + +#include + +namespace nix { + +static size_t unitSize(const size_t &) +{ + return 1; +} + +TEST(processOrdered, consumesInOrder) +{ + const size_t n = 200; + + std::vector consumed; + + processOrdered( + n, + [&](size_t i) { + /* Make items finish out of order. */ + thread_local std::mt19937 rng(std::hash{}(std::this_thread::get_id())); + std::this_thread::sleep_for(std::chrono::microseconds(rng() % 1000)); + return i * i; + }, + [&](size_t i, size_t && res) { + ASSERT_EQ(i, consumed.size()); + ASSERT_EQ(res, i * i); + consumed.push_back(i); + }, + unitSize, + 10, + 8); + + ASSERT_EQ(consumed.size(), n); +} + +TEST(processOrdered, respectsBufferLimit) +{ + const size_t maxBuffered = 5, maxThreads = 4; + + std::atomic buffered = 0, maxSeen = 0; + size_t count = 0; + + processOrdered( + 100, + [&](size_t i) { + /* Make the first item slow, so that other items pile up + behind it. */ + if (i == 0) + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + auto cur = ++buffered; + size_t prev = maxSeen; + while (cur > prev && !maxSeen.compare_exchange_weak(prev, cur)) + ; + return i; + }, + [&](size_t i, size_t &&) { + --buffered; + count++; + }, + unitSize, + maxBuffered, + maxThreads); + + ASSERT_EQ(count, 100); + /* The limit can be exceeded by the items that were already + started when it was reached. */ + ASSERT_LE(maxSeen, maxBuffered + 2 * maxThreads); +} + +TEST(processOrdered, slowItemDoesNotBlockOthers) +{ + const size_t n = 100; + + std::atomic produced = 0; + + processOrdered( + n, + [&](size_t i) { + if (i == 0) + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + produced++; + return i; + }, + [&](size_t i, size_t &&) { + /* Since results have size 0, all other items should have + been produced while the first one was in progress. */ + if (i == 0) { + ASSERT_EQ(produced, n); + } + }, + [](const size_t &) -> size_t { return 0; }, + 0, + 4); +} + +TEST(processOrdered, empty) +{ + processOrdered( + 0, + [](size_t) -> int { throw Error("should not be called"); }, + [](size_t, int &&) { FAIL(); }, + [](const int &) -> size_t { return 0; }, + 10); +} + +TEST(processOrdered, produceThrows) +{ + ASSERT_THROW( + processOrdered( + 100, + [](size_t i) { + if (i == 42) + throw Error("boom"); + return i; + }, + [](size_t, size_t &&) {}, + unitSize, + 10), + Error); +} + +TEST(processOrdered, consumeThrows) +{ + ASSERT_THROW( + processOrdered( + 100, + [](size_t i) { return i; }, + [](size_t i, size_t &&) { + if (i == 42) + throw Error("boom"); + }, + unitSize, + 10), + Error); +} + +} // namespace nix diff --git a/src/libutil-tests/zstd-patch.cc b/src/libutil-tests/zstd-patch.cc new file mode 100644 index 000000000000..9f41d58fa87e --- /dev/null +++ b/src/libutil-tests/zstd-patch.cc @@ -0,0 +1,88 @@ +#include "nix/util/zstd-patch.hh" +#include "nix/util/compression.hh" + +#include + +#include + +namespace nix { + +static std::string randomString(size_t size, uint64_t seed) +{ + std::mt19937_64 rng(seed); + std::string s(size, '\0'); + for (auto & c : s) + c = (char) rng(); + return s; +} + +static std::string applyPatch(std::string_view base, std::string_view patch) +{ + StringSink sink; + applyZstdPatch(base, patch, sink); + return std::move(sink.s); +} + +TEST(zstdPatch, roundtrip) +{ + auto base = randomString(1 << 20, 1); + auto target = base; + /* Make some small changes. */ + target.replace(1000, 5, "hello"); + target.insert(500000, "inserted"); + target.erase(800000, 100); + + auto patch = makeZstdPatch(base, target, 19); + ASSERT_EQ(applyPatch(base, patch), target); + + /* The base is random, so the patch is only small because it + refers to the base. */ + ASSERT_LT(patch.size(), 1000); +} + +TEST(zstdPatch, emptyBase) +{ + auto target = randomString(10000, 2); + auto patch = makeZstdPatch("", target, 3); + ASSERT_EQ(applyPatch("", patch), target); +} + +TEST(zstdPatch, emptyTarget) +{ + auto base = randomString(10000, 3); + auto patch = makeZstdPatch(base, "", 3); + ASSERT_EQ(applyPatch(base, patch), ""); +} + +TEST(zstdPatch, wrongBase) +{ + auto base = randomString(100000, 4); + auto target = base; + target.replace(50000, 5, "hello"); + auto patch = makeZstdPatch(base, target, 19); + + /* Applying to the wrong base produces garbage (or an error), so + callers must check the base's hash. */ + auto otherBase = randomString(100000, 5); + try { + ASSERT_NE(applyPatch(otherBase, patch), target); + } catch (CompressionError &) { + } +} + +TEST(zstdPatch, truncated) +{ + auto base = randomString(100000, 6); + auto target = randomString(100000, 7); + auto patch = makeZstdPatch(base, target, 3); + ASSERT_THROW(applyPatch(base, patch.substr(0, patch.size() / 2)), CompressionError); +} + +TEST(zstdPatch, trailingGarbage) +{ + auto base = randomString(1000, 8); + auto patch = makeZstdPatch(base, base, 3); + ASSERT_THROW(applyPatch(base, patch + "garbage"), CompressionError); +} + +} // namespace nix diff --git a/src/libutil/hdiff-patch.cc b/src/libutil/hdiff-patch.cc new file mode 100644 index 000000000000..26685746dbf1 --- /dev/null +++ b/src/libutil/hdiff-patch.cc @@ -0,0 +1,165 @@ +#include "nix/util/hdiff-patch.hh" +#include "nix/util/compression.hh" + +#include + +#include +#include + +/* Don't let the compression plugins print notices on stdout, which + would end up in the output of e.g. `nix nario export`. */ +#define IS_NOTICE_compress_canceled 0 +#include +#include + +#include +#include + +namespace nix { + +/* The zstd dictionary size (log2) used for patches. */ +static const int zstdDictBits = 24; + +/* The minimum score for matches, i.e. `hdiffz -m-6`. */ +static const int minSingleMatchScore = 6; + +/* Don't trust the memory requirements stated in a patch beyond this. */ +static const uint64_t maxStepMemSize = 64 * 1024 * 1024; + +std::string makeHdiffPatch(std::string base, std::string target, int level) +{ + TCompressPlugin_zstd compressPlugin = zstdCompressPlugin; + compressPlugin.compress_level = level; + compressPlugin.dict_bits = zstdDictBits; + compressPlugin.thread_num = 1; + + auto bytes = [](std::string & s) { return (unsigned char *) s.data(); }; + + std::vector out; + try { + create_single_compressed_diff_block( + bytes(target), + bytes(target) + target.size(), + bytes(base), + bytes(base) + base.size(), + out, + &compressPlugin.base, + kDefaultPatchStepMemSize, + kDefaultFastMatchBlockSize, + minSingleMatchScore, + /* isUseBigCacheMatch */ false, + /* threadNum */ 1); + } catch (std::exception & e) { + throw CompressionError("computing HDiffPatch patch: %s", e.what()); + } + + return std::string((const char *) out.data(), out.size()); +} + +namespace { + +/** + * An HDiffPatch output stream that writes to a `Sink`. HDiffPatch + * writes sequentially. Exceptions thrown by the sink are stored and + * rethrown after HDiffPatch returns, since they can't propagate + * through its C code. + */ +struct SinkOutputStream +{ + hpatch_TStreamOutput stream; + Sink & sink; + uint64_t written = 0; + std::exception_ptr exception; + + SinkOutputStream(Sink & sink, uint64_t size) + : sink(sink) + { + stream.streamImport = this; + stream.streamSize = size; + stream.read_writed = nullptr; + stream.write = write; + } + + static hpatch_BOOL write( + const hpatch_TStreamOutput * stream, + hpatch_StreamPos_t writeToPos, + const unsigned char * data, + const unsigned char * dataEnd) + { + auto & self = *(SinkOutputStream *) stream->streamImport; + if (writeToPos != self.written) + return hpatch_FALSE; + try { + self.sink({(const char *) data, (size_t) (dataEnd - data)}); + } catch (...) { + self.exception = std::current_exception(); + return hpatch_FALSE; + } + self.written += dataEnd - data; + return hpatch_TRUE; + } +}; + +} // namespace + +void applyHdiffPatch(std::string_view base, std::string_view patch, Sink & sink) +{ + auto patchData = (const unsigned char *) patch.data(); + auto baseData = (const unsigned char *) base.data(); + + hpatch_singleCompressedDiffInfo info; + if (!getSingleCompressedDiffInfo_mem(&info, patchData, patchData + patch.size())) + throw CompressionError("invalid HDiffPatch patch"); + + if (info.oldDataSize != base.size()) + throw CompressionError( + "HDiffPatch patch expects a base of %d bytes, but the base has %d bytes", info.oldDataSize, base.size()); + + auto dataSize = info.compressedSize ? info.compressedSize : info.uncompressedSize; + if (info.diffDataPos > patch.size() || dataSize != patch.size() - info.diffDataPos) + throw CompressionError("HDiffPatch patch has an invalid size"); + + if (info.stepMemSize > maxStepMemSize) + throw CompressionError("HDiffPatch patch requires too much memory"); + + hpatch_TDecompress * decompressPlugin = nullptr; + if (info.compressType[0]) { + if (!zstdDecompressPlugin.is_can_open(info.compressType)) + throw CompressionError("HDiffPatch patch uses unsupported compression type '%s'", info.compressType); + decompressPlugin = &zstdDecompressPlugin; + } + + std::vector cache(info.stepMemSize + hpatch_kStreamCacheSize * 3); + + hpatch_TStreamInput baseStream, patchStream; + mem_as_hStreamInput(&baseStream, baseData, baseData + base.size()); + mem_as_hStreamInput(&patchStream, patchData, patchData + patch.size()); + + SinkOutputStream out(sink, info.newDataSize); + + auto ok = patch_single_compressed_diff( + &out.stream, + &baseStream, + &patchStream, + info.diffDataPos, + info.uncompressedSize, + info.compressedSize, + decompressPlugin, + info.coverCount, + (size_t) info.stepMemSize, + cache.data(), + cache.data() + cache.size(), + /* coversListener */ nullptr, + /* threadNum */ 1); + + if (out.exception) + std::rethrow_exception(out.exception); + + if (!ok) + throw CompressionError("applying HDiffPatch patch failed"); + + if (out.written != info.newDataSize) + throw CompressionError("HDiffPatch patch produced %d bytes instead of %d", out.written, info.newDataSize); +} + +} // namespace nix diff --git a/src/libutil/include/nix/util/hdiff-patch.hh b/src/libutil/include/nix/util/hdiff-patch.hh new file mode 100644 index 000000000000..75f9e97e0437 --- /dev/null +++ b/src/libutil/include/nix/util/hdiff-patch.hh @@ -0,0 +1,30 @@ +#pragma once +///@file + +#include "nix/util/serialise.hh" + +#include +#include + +namespace nix { + +/** + * Compute a binary patch that transforms `base` into `target` using + * HDiffPatch (https://github.com/sisong/HDiffPatch), compressed using + * zstd at level `level`. The result is in HDiffPatch's "single + * compressed stream" format and can be applied by + * `applyHdiffPatch()` given the same `base`. + * + * `base` and `target` are taken by value because HDiffPatch + * overwrites them during matching. + */ +std::string makeHdiffPatch(std::string base, std::string target, int level); + +/** + * Apply a patch produced by `makeHdiffPatch()` to `base`, writing the + * reconstructed target to `sink`. The output is streamed, so only + * `base` and `patch` need to be held in memory. + */ +void applyHdiffPatch(std::string_view base, std::string_view patch, Sink & sink); + +} // namespace nix diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 1379fd8d2f2a..b6f184d03beb 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -56,6 +56,7 @@ headers = [ config_pub_h ] + files( 'fun.hh', 'git.hh', 'hash.hh', + 'hdiff-patch.hh', 'hilite.hh', 'json-impls.hh', 'json-non-null.hh', @@ -90,6 +91,7 @@ headers = [ config_pub_h ] + files( 'sort.hh', 'source-accessor.hh', 'source-path.hh', + 'spilling-sink.hh', 'split.hh', 'std-hash.hh', 'strings-inline.hh', @@ -109,4 +111,5 @@ headers = [ config_pub_h ] + files( 'util.hh', 'variant-wrapper.hh', 'xml-writer.hh', + 'zstd-patch.hh', ) diff --git a/src/libutil/include/nix/util/spilling-sink.hh b/src/libutil/include/nix/util/spilling-sink.hh new file mode 100644 index 000000000000..5594c70da3c8 --- /dev/null +++ b/src/libutil/include/nix/util/spilling-sink.hh @@ -0,0 +1,55 @@ +#pragma once +///@file + +#include "nix/util/serialise.hh" + +namespace nix { + +/** + * A sink that accumulates data in memory up to `maxMemory` bytes, and + * spills to an anonymous temporary file once that limit is exceeded. + * This is useful when the size of some data must be known before it + * can be written, without using unbounded memory. + */ +struct SpillingStringSink : Sink +{ + explicit SpillingStringSink(size_t maxMemory) + : maxMemory(maxMemory) + { + } + + void operator()(std::string_view data) override; + + /** + * Total number of bytes written. + */ + uint64_t size() const + { + return written; + } + + /** + * Whether the data has been spilled to disk. + */ + bool isSpilled() const + { + return (bool) fd; + } + + /** + * Return a source that reads back everything written so far. No + * more data may be written after calling this. The source must + * not outlive this sink. + */ + std::unique_ptr getSource(); + +private: + size_t maxMemory; + uint64_t written = 0; + bool finished = false; + StringSink memory; + AutoCloseFD fd; + std::optional fileSink; +}; + +} // namespace nix diff --git a/src/libutil/include/nix/util/thread-pool.hh b/src/libutil/include/nix/util/thread-pool.hh index a6bb7cf02d1c..26d426b124ae 100644 --- a/src/libutil/include/nix/util/thread-pool.hh +++ b/src/libutil/include/nix/util/thread-pool.hh @@ -10,6 +10,8 @@ #include #include #include +#include +#include namespace nix { @@ -182,4 +184,119 @@ void processGraph( throw Error("graph processing incomplete (cyclic reference?)"); } +/** + * Compute `produce(i)` for each `i` in `[0, n)` in parallel, and call + * `consume(i, result)` for each result strictly in order of `i`, one + * at a time (though not necessarily on the same thread). + * + * Results that have been produced but not yet consumed are buffered. + * To bound the amount of buffered data, no new items are started + * while the total `getSize()` of the buffered results exceeds + * `maxBuffered`. (The limit can be exceeded by the results of the + * items that are already being produced.) Thus, if results are small, + * a slow item doesn't prevent the other threads from making progress + * on subsequent items. + */ +template +void processOrdered( + size_t n, + fun produce, + fun consume, + fun getSize, + size_t maxBuffered, + size_t maxThreads = 0) +{ + if (!maxThreads) + maxThreads = std::max(1U, std::thread::hardware_concurrency()); + + struct Result + { + T value; + size_t size; + }; + + struct State + { + std::map ready; + size_t nextToConsume = 0; + size_t nextToEnqueue = 0; + /* Number of items enqueued but not yet produced. */ + size_t active = 0; + /* Total size of the results in `ready`. */ + size_t buffered = 0; + bool consuming = false; + }; + + Sync state_; + + std::function worker; + + /* Create pool last to ensure threads are stopped before other + destructors run. */ + ThreadPool pool(maxThreads); + + /* Start new items, as long as the buffer limit hasn't been + reached. We keep up to twice as many items active as there are + threads, since `ThreadPool` only starts a new thread if there + are more pending items than threads. Note that this always + makes progress: if nothing is active, the next item to be + consumed is either in `ready` or hasn't been enqueued yet, in + which case `ready` is empty and so `buffered` is 0. */ + auto enqueueMore = [&](State & state) { + while (state.nextToEnqueue < n && state.active < 2 * maxThreads && state.buffered <= maxBuffered) { + pool.enqueue(std::bind(worker, state.nextToEnqueue++)); + state.active++; + } + }; + + worker = [&](size_t i) { + auto value = produce(i); + auto size = getSize(value); + + { + auto state(state_.lock()); + state->ready.emplace(i, Result{std::move(value), size}); + state->buffered += size; + state->active--; + enqueueMore(*state); + + /* If another thread is already consuming results, it will + pick up this one. */ + if (state->consuming) + return; + state->consuming = true; + } + + while (true) { + size_t j; + std::optional result; + + { + auto state(state_.lock()); + auto k = state->ready.find(state->nextToConsume); + if (k == state->ready.end()) { + state->consuming = false; + return; + } + j = state->nextToConsume; + result.emplace(std::move(k->second)); + state->ready.erase(k); + } + + consume(j, std::move(result->value)); + + { + auto state(state_.lock()); + state->nextToConsume++; + state->buffered -= result->size; + enqueueMore(*state); + } + } + }; + + enqueueMore(*state_.lock()); + + pool.process(); +} + } // namespace nix diff --git a/src/libutil/include/nix/util/zstd-patch.hh b/src/libutil/include/nix/util/zstd-patch.hh new file mode 100644 index 000000000000..3782c6a85028 --- /dev/null +++ b/src/libutil/include/nix/util/zstd-patch.hh @@ -0,0 +1,35 @@ +#pragma once +///@file + +#include "nix/util/serialise.hh" + +#include +#include + +namespace nix { + +/** + * Compute a binary patch that transforms `base` into `target`, using + * zstd's "patch-from" mode (i.e. `base` is used as a prefix + * dictionary). The result is a single zstd frame that can be + * decompressed by `applyZstdPatch()` given the same `base`. + * + * With an empty `base`, this is just ordinary zstd compression of + * `target`. + */ +std::string makeZstdPatch(std::string_view base, std::string_view target, int level); + +/** + * Apply a patch produced by `makeZstdPatch()` to `base`, writing the + * reconstructed target to `sink`. The output is streamed, so only + * `base` and `patch` need to be held in memory. + */ +void applyZstdPatch(std::string_view base, std::string_view patch, Sink & sink); + +/** + * The largest combined size of base and target that a zstd patch can + * fully cover (i.e. the maximum zstd window size). + */ +uint64_t maxZstdPatchWindow(); + +} // namespace nix diff --git a/src/libutil/meson.build b/src/libutil/meson.build index bf3a1f3b0833..889f9272f9fe 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -111,6 +111,10 @@ deps_private += brotli zstd = dependency('libzstd', version : '>= 1.4.0') deps_private += zstd +# For experimental HDiffPatch binary diffs. +hdiffpatch = dependency('hdiffpatch') +deps_private += hdiffpatch + cpuid_required = get_option('cpuid') if host_machine.cpu_family() != 'x86_64' and cpuid_required.enabled() warning('Force-enabling seccomp on non-x86_64 does not make sense') @@ -169,6 +173,7 @@ sources = [ config_priv_h ] + files( 'fs-sink.cc', 'git.cc', 'hash.cc', + 'hdiff-patch.cc', 'hilite.cc', 'json-utils.cc', 'logging.cc', @@ -188,6 +193,7 @@ sources = [ config_priv_h ] + files( 'signature/signer.cc', 'source-accessor.cc', 'source-path.cc', + 'spilling-sink.cc', 'strings.cc', 'suggestions.cc', 'table.cc', @@ -201,6 +207,7 @@ sources = [ config_priv_h ] + files( 'users.cc', 'util.cc', 'xml-writer.cc', + 'zstd-patch.cc', ) subdir('include/nix/util') diff --git a/src/libutil/package.nix b/src/libutil/package.nix index fe6fcb75f454..71bfc13d2b73 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -5,6 +5,7 @@ boost, brotli, + hdiffpatch, libarchive, libblake3, libcpuid, @@ -50,6 +51,7 @@ mkMesonLibrary (finalAttrs: { buildInputs = [ brotli + hdiffpatch libblake3 libsodium openssl diff --git a/src/libutil/spilling-sink.cc b/src/libutil/spilling-sink.cc new file mode 100644 index 000000000000..ea59f307b22a --- /dev/null +++ b/src/libutil/spilling-sink.cc @@ -0,0 +1,44 @@ +#include "nix/util/spilling-sink.hh" +#include "nix/util/file-system.hh" + +#include + +namespace nix { + +void SpillingStringSink::operator()(std::string_view data) +{ + assert(!finished); + + written += data.size(); + + if (!fileSink && memory.s.size() + data.size() <= maxMemory) { + memory(data); + return; + } + + if (!fileSink) { + fd = createAnonymousTempFile(); + fileSink.emplace(fd.get()); + (*fileSink)(memory.s); + memory.s = {}; + } + + (*fileSink)(data); +} + +std::unique_ptr SpillingStringSink::getSource() +{ + assert(!finished); + finished = true; + + if (fileSink) { + fileSink->flush(); + auto source = std::make_unique(fd.get()); + source->restart(); + return source; + } + + return std::make_unique(std::move(memory.s)); +} + +} // namespace nix diff --git a/src/libutil/zstd-patch.cc b/src/libutil/zstd-patch.cc new file mode 100644 index 000000000000..9b135cfd63aa --- /dev/null +++ b/src/libutil/zstd-patch.cc @@ -0,0 +1,93 @@ +#include "nix/util/zstd-patch.hh" +#include "nix/util/compression.hh" +#include "nix/util/signals.hh" + +#include + +#include +#include + +namespace nix { + +static size_t checkZstd(size_t ret) +{ + if (ZSTD_isError(ret)) + throw CompressionError("zstd error: %s", ZSTD_getErrorName(ret)); + return ret; +} + +static int maxWindowLog() +{ + return ZSTD_cParam_getBounds(ZSTD_c_windowLog).upperBound; +} + +uint64_t maxZstdPatchWindow() +{ + return uint64_t(1) << maxWindowLog(); +} + +std::string makeZstdPatch(std::string_view base, std::string_view target, int level) +{ + std::unique_ptr cctx{ZSTD_createCCtx(), ZSTD_freeCCtx}; + if (!cctx) + throw CompressionError("unable to initialise zstd encoder"); + + checkZstd(ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_compressionLevel, level)); + + if (!base.empty()) { + /* The window must cover both the base (prefix) and the + target, otherwise the end of the target can't refer to the + start of the base. */ + uint64_t total = base.size() + target.size(); + int windowLog = std::bit_width(total - 1); + auto bounds = ZSTD_cParam_getBounds(ZSTD_c_windowLog); + windowLog = std::clamp(windowLog, bounds.lowerBound, bounds.upperBound); + checkZstd(ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_windowLog, windowLog)); + checkZstd(ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_enableLongDistanceMatching, 1)); + /* Use the multi-threaded compressor (even with a single + worker), since the single-threaded one makes poor use of + the prefix at high compression levels (e.g. a 13 MB patch + vs 6 KB between two identical 65 MB inputs at level 19). + Don't checkZstd(): if libzstd was built without + ZSTD_MULTITHREAD this returns an error, but compression + still works. */ + ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_nbWorkers, 1); + checkZstd(ZSTD_CCtx_refPrefix(cctx.get(), base.data(), base.size())); + } + + std::string out(ZSTD_compressBound(target.size()), '\0'); + auto n = checkZstd(ZSTD_compress2(cctx.get(), out.data(), out.size(), target.data(), target.size())); + out.resize(n); + return out; +} + +void applyZstdPatch(std::string_view base, std::string_view patch, Sink & sink) +{ + std::unique_ptr dctx{ZSTD_createDCtx(), ZSTD_freeDCtx}; + if (!dctx) + throw CompressionError("unable to initialise zstd decoder"); + + checkZstd(ZSTD_DCtx_setParameter(dctx.get(), ZSTD_d_windowLogMax, maxWindowLog())); + if (!base.empty()) + checkZstd(ZSTD_DCtx_refPrefix(dctx.get(), base.data(), base.size())); + + std::vector outbuf(ZSTD_DStreamOutSize()); + ZSTD_inBuffer in = {patch.data(), patch.size(), 0}; + + while (true) { + checkInterrupt(); + ZSTD_outBuffer out = {outbuf.data(), outbuf.size(), 0}; + auto ret = checkZstd(ZSTD_decompressStream(dctx.get(), &out, &in)); + if (out.pos > 0) + sink({outbuf.data(), out.pos}); + if (ret == 0) { + if (in.pos != in.size) + throw CompressionError("trailing garbage after zstd patch"); + break; + } + if (in.pos == in.size && out.pos < out.size) + throw CompressionError("truncated zstd patch"); + } +} + +} // namespace nix diff --git a/src/nix/nario-export.md b/src/nix/nario-export.md index 2480733c1cae..ff609cbcecb6 100644 --- a/src/nix/nario-export.md +++ b/src/nix/nario-export.md @@ -14,6 +14,13 @@ R""( # nix nario import --no-check-sigs < dump.nario ``` +* Export the closure of a new version of `hello`, assuming that the destination already has the closure of an old version: + + ```console + # nix nario export --format 2 -r /nix/store/6i6xl6bmcpxqd51m8nlva40d5c1bhndx-hello-2.12.3 \ + --base /nix/store/nm7p8wxflggcwxfzayhysq4z6a1wg373-hello-2.12.3 > update.nario + ``` + # Description This command prints to standard output a serialization of the specified store paths in `nario` format. This serialization can be imported into another store using `nix nario import`. @@ -26,4 +33,32 @@ You must specify the desired `nario` version. Currently the following versions a * `2`: The latest version. Recommended. +# Binary diffs + +The `--base` flag specifies one or more *installables* whose closure (the *base closure*) is assumed to be present in the store into which the nario will be imported. This can greatly reduce the size of the nario when deploying an update to a closure: + +* Paths that are in the base closure are not included in the nario. Instead, the nario records that they're expected to be present, and `nix nario import` fails if they're missing or have a different NAR hash. + +* For other paths, `nix nario export` looks for a suitable path in the base closure. How this path is selected is determined by the flag `--base-selection-method`. Currently, the only method is `by-name` (the default), which selects a path with the same name (ignoring the version), e.g. `hello-2.12.2` for `hello-2.12.3`. If a suitable path exists, the path is stored as a binary diff against the NAR of that base path, provided that the diff is sufficiently small. `nix nario import` reconstructs the path by applying the diff to the base path, after verifying that the base path has the expected NAR hash. + +The flag `--diff-algorithm` determines how binary diffs are computed: + +* `hdiffpatch` (the default): The diff is computed using [HDiffPatch](https://github.com/sisong/HDiffPatch) and compressed using zstd. This generally produces the smallest diffs. + +* `zstd`: The NAR is compressed using zstd, with the NAR of the base path as a dictionary. + +Binary diffs require nario format 2. Narios that contain binary diffs cannot be imported by versions of Nix that don't support them. + +# Compression + +By default, the NARs in a nario are not compressed. The `--compression` flag specifies a compression method (such as `zstd` or `xz`) for NARs that are not exported as binary diffs. (Binary diffs are always compressed.) This is preferable to compressing the nario as a whole, since compressing binary diffs a second time is a waste of time. For example: + +```console +# nix nario export --format 2 --compression zstd -r /nix/store/6i6xl6bmcpxqd51m8nlva40d5c1bhndx-hello-2.12.3 > hello.nario +``` + +The `--compression-level` flag specifies the compression level. For `zstd`, the default is 9, which is still fast but compresses significantly better than zstd's own default of 3. + +Compression requires nario format 2. Narios that contain compressed NARs cannot be imported by versions of Nix that don't support them. + )"" diff --git a/src/nix/nario-list.md b/src/nix/nario-list.md index c050457b3657..f6d12bf88357 100644 --- a/src/nix/nario-list.md +++ b/src/nix/nario-list.md @@ -40,4 +40,8 @@ R""( This command lists the contents of a nario file read from standard input. +For narios created with `nix nario export --base`, paths that are stored as a binary diff are shown with the diff algorithm (`zstd` or `hdiffpatch`), the base path of the diff and the size of the diff. Their file contents are not shown by `--recursive`, since that would require the base path. Paths that are expected to already be present at the destination are shown as such. In the JSON output, these have a `diff` or `present` attribute, respectively. + +For narios created with `nix nario export --compression`, the compression method and compressed size of each NAR are shown. In the JSON output, these are in the `compression` attribute. + )"" diff --git a/src/nix/nario.cc b/src/nix/nario.cc index 39b18d20403c..5528b3f3aef6 100644 --- a/src/nix/nario.cc +++ b/src/nix/nario.cc @@ -35,6 +35,11 @@ static auto rCmdNario = registerCommand("nario"); struct CmdNarioExport : StorePathsCommand { unsigned int version = 0; + std::vector baseArgs; + BaseSelectionMethod baseSelectionMethod = BaseSelectionMethod::byName; + NarioDiffAlgo diffAlgo = NarioDiffAlgo::hdiffpatch; + CompressionAlgo compression = CompressionAlgo::none; + std::optional compressionLevel; CmdNarioExport() { @@ -45,6 +50,60 @@ struct CmdNarioExport : StorePathsCommand .handler = {&version}, .required = true, }); + + addFlag({ + .longName = "base", + .description = + "Assume that the closure of *installable* is already present at the destination. Paths in this closure are not exported, and other paths are exported as binary diffs against paths in this closure where possible. Can be specified multiple times.", + .labels = {"installable"}, + .handler = {[this](std::string s) { baseArgs.push_back(s); }}, + .completer = getCompleteInstallable(), + }); + + addFlag({ + .longName = "base-selection-method", + .description = + "How to select the path in the base closure against which to diff a path. Currently the only method is `by-name` (select a path with the same name, ignoring the version), which is the default.", + .labels = {"method"}, + .handler = {[this](std::string s) { baseSelectionMethod = parseBaseSelectionMethod(s); }}, + .completer = {[](AddCompletions & completions, size_t, std::string_view prefix) { + for (auto method : {BaseSelectionMethod::byName}) { + auto s = showBaseSelectionMethod(method); + if (s.starts_with(prefix)) + completions.add(s); + } + }}, + }); + + addFlag({ + .longName = "diff-algorithm", + .description = "Algorithm to use for computing binary diffs. Must be `hdiffpatch` (the default) or `zstd`.", + .labels = {"algorithm"}, + .handler = {[this](std::string s) { diffAlgo = parseNarioDiffAlgo(s); }}, + .completer = {[](AddCompletions & completions, size_t, std::string_view prefix) { + for (auto algo : {NarioDiffAlgo::hdiffpatch, NarioDiffAlgo::zstd}) { + auto s = showNarioDiffAlgo(algo); + if (s.starts_with(prefix)) + completions.add(s); + } + }}, + }); + + addFlag({ + .longName = "compression", + .description = + "Compression method (e.g. `zstd`) to use for NARs that are not exported as binary diffs. The default is `none`. Requires nario format 2.", + .labels = {"method"}, + .handler = {[this](std::string s) { compression = parseCompressionAlgo(s, true); }}, + }); + + addFlag({ + .longName = "compression-level", + .description = + "Compression level to use for the method specified by `--compression`. The default is 9 for `zstd`, and the compression method's own default for other methods.", + .labels = {"level"}, + .handler = {&compressionLevel}, + }); } std::string description() override @@ -65,7 +124,20 @@ struct CmdNarioExport : StorePathsCommand if (isatty(fd)) throw UsageError("refusing to write nario to a terminal"); FdSink sink(std::move(fd)); - exportPaths(*store, StorePathSet(storePaths.begin(), storePaths.end()), sink, version); + + NarioExportOptions options{ + .version = version, + .baseSelectionMethod = baseSelectionMethod, + .diffAlgo = diffAlgo, + .compression = compression, + .compressionLevel = compressionLevel, + }; + + if (!baseArgs.empty()) + options.basePaths = Installable::toStorePathSet( + getEvalStore(), store, Realise::Outputs, operateOn, parseInstallables(store, baseArgs)); + + exportPaths(*store, StorePathSet(storePaths.begin(), storePaths.end()), sink, options); } }; @@ -242,12 +314,8 @@ struct CmdNarioList : Command, MixJSON, MixLongListing struct ListingStore : Store { - std::optional json; - CmdNarioList & cmd; - - ListingStore(ref config, CmdNarioList & cmd) + ListingStore(ref config) : Store{*config} - , cmd(cmd) { } @@ -270,24 +338,7 @@ struct CmdNarioList : Command, MixJSON, MixLongListing void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override { - std::optional contents; - if (cmd.listContents) - contents = listNar(source); - else - source.skip(info.narSize); - - if (json) { - // FIXME: make the JSON format configurable. - auto obj = info.toJSON(this, true, PathInfoJsonFormat::V1); - if (contents) - obj.emplace("contents", *contents); - json->emplace(printStorePath(info.path), std::move(obj)); - } else { - if (contents) - renderNarListing(CanonPath(printStorePath(info.path)), *contents, cmd.longListing); - else - logger->cout(fmt("%s: %d bytes", printStorePath(info.path), info.narSize)); - } + unsupported("addToStore"); } StorePath addToStoreFromDump( @@ -332,12 +383,112 @@ struct CmdNarioList : Command, MixJSON, MixLongListing void anchor() override {} }; + struct Lister : NarioVisitor + { + Store & store; + CmdNarioList & cmd; + std::optional json; + + Lister(Store & store, CmdNarioList & cmd) + : store(store) + , cmd(cmd) + { + } + + void + add(const ValidPathInfo & info, + std::optional contents, + std::function extendJSON = {}) + { + if (!json) + return; + // FIXME: make the JSON format configurable. + auto obj = info.toJSON(&store, true, PathInfoJsonFormat::V1); + if (contents) + obj.emplace("contents", std::move(*contents)); + if (extendJSON) + extendJSON(obj); + json->emplace(store.printStorePath(info.path), std::move(obj)); + } + + void + fullPath(const ValidPathInfo & info, Source & nar, std::optional compression) override + { + std::optional contents; + if (cmd.listContents) + contents = listNar(nar); + else + nar.skip(info.narSize); + + if (!json) { + if (contents) + renderNarListing(CanonPath(store.printStorePath(info.path)), *contents, cmd.longListing); + else if (compression) + logger->cout( + fmt("%s: %d bytes, %s-compressed (%d bytes)", + store.printStorePath(info.path), + info.narSize, + showCompressionAlgo(compression->algo), + compression->size)); + else + logger->cout(fmt("%s: %d bytes", store.printStorePath(info.path), info.narSize)); + } + + add(info, std::move(contents), [&](nlohmann::json & obj) { + if (compression) + obj.emplace( + "compression", + nlohmann::json{ + {"method", showCompressionAlgo(compression->algo)}, + {"size", compression->size}, + }); + }); + } + + void diffPath( + const ValidPathInfo & info, + NarioDiffAlgo algo, + const StorePath & basePath, + const Hash & baseNarHash, + std::string_view patch) override + { + if (!json) + logger->cout( + fmt("%s: %d bytes, %s diff against %s (%d bytes)", + store.printStorePath(info.path), + info.narSize, + showNarioDiffAlgo(algo), + store.printStorePath(basePath), + patch.size())); + + add(info, std::nullopt, [&](nlohmann::json & obj) { + obj.emplace( + "diff", + nlohmann::json{ + {"algorithm", showNarioDiffAlgo(algo)}, + {"base", store.printStorePath(basePath)}, + {"baseNarHash", baseNarHash.to_string(HashFormat::SRI, true)}, + {"size", patch.size()}, + }); + }); + } + + void presentPath(const ValidPathInfo & info) override + { + if (!json) + logger->cout(fmt("%s: expected to be present", store.printStorePath(info.path))); + + add(info, std::nullopt, [](nlohmann::json & obj) { obj.emplace("present", true); }); + } + }; + auto source{getNarioSource()}; auto config = make_ref(StoreConfig::Params()); - ListingStore lister(config, *this); + ListingStore store(config); + Lister lister(store, *this); if (json) lister.json = nlohmann::json::object(); - importPaths(lister, source, NoCheckSigs); + parseNario(store, source, lister); if (json) { auto j = nlohmann::json::object(); j["version"] = 1; diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index 59928479aba6..be0e6e29ed6e 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -773,7 +773,7 @@ static void opExport(Strings opFlags, Strings opArgs) paths.insert(store->followLinksToStorePath(i)); FdSink sink(getStandardOutput()); - exportPaths(*store, paths, sink, 1); + exportPaths(*store, paths, sink, {.version = 1}); sink.flush(); } diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 1e43c200068d..014af671603e 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -132,6 +132,7 @@ suites = [ 'multiple-outputs-substitute-failure.sh', 'multiple-outputs.sh', 'nar-access.sh', + 'nario-diff.sh', 'nars.sh', 'nested-sandboxing.sh', 'nix-build.sh', diff --git a/tests/functional/nario-diff.nix b/tests/functional/nario-diff.nix new file mode 100644 index 000000000000..4c3ee01ed296 --- /dev/null +++ b/tests/functional/nario-diff.nix @@ -0,0 +1,32 @@ +{ + version, +}: +with import ./config.nix; + +let + + # A dependency that's identical in both versions. + shared = mkDerivation { + name = "nario-shared-1.0"; + buildCommand = "mkdir $out; echo shared > $out/shared"; + }; + + # A dependency that only exists in version 2, so it has no diff base. + extra = mkDerivation { + name = "nario-extra"; + buildCommand = "mkdir $out; seq 1 10000 > $out/data"; + }; + +in + +mkDerivation { + name = "nario-pkg-${version}"; + buildCommand = '' + mkdir $out + seq 1 100000 > $out/data + echo ${version} >> $out/data + echo $out >> $out/data + ln -s ${shared} $out/shared + ${if version == "2" then "ln -s ${extra} $out/extra" else ""} + ''; +} diff --git a/tests/functional/nario-diff.sh b/tests/functional/nario-diff.sh new file mode 100755 index 000000000000..6996ef8d1406 --- /dev/null +++ b/tests/functional/nario-diff.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +clearStore + +old=$(nix-build nario-diff.nix --argstr version 1 --no-out-link) +new=$(nix-build nario-diff.nix --argstr version 2 --no-out-link) +shared=$(nix-store -qR "$new" | grep nario-shared) +extra=$(nix-store -qR "$new" | grep nario-extra) + +nix key generate-secret --key-name my-key > "$TEST_ROOT"/secret +public_key=$(nix key convert-secret-to-public < "$TEST_ROOT"/secret) +nix store sign --key-file "$TEST_ROOT/secret" -r "$new" + +# Binary diffs require format 2. +expectStderr 1 nix nario export --format 1 -r "$new" --base "$old" | grepQuiet "only supported in nario version 2" + +nix nario export --format 2 -r "$old" > "$TEST_ROOT"/old.nario +nix nario export --format 2 -r "$new" > "$TEST_ROOT"/full.nario +nix nario export --format 2 -r "$new" --base "$old" > "$TEST_ROOT"/diff.nario + +# `by-name` is the default base selection method. +nix nario export --format 2 -r "$new" --base "$old" --base-selection-method by-name > "$TEST_ROOT"/diff-by-name.nario +cmp "$TEST_ROOT"/diff.nario "$TEST_ROOT"/diff-by-name.nario +expectStderr 1 nix nario export --format 2 -r "$new" --base "$old" --base-selection-method foo | grepQuiet "unknown base selection method 'foo'" + +# `hdiffpatch` is the default diff algorithm. +nix nario export --format 2 -r "$new" --base "$old" --diff-algorithm hdiffpatch > "$TEST_ROOT"/diff-hdiffpatch.nario +cmp "$TEST_ROOT"/diff.nario "$TEST_ROOT"/diff-hdiffpatch.nario +expectStderr 1 nix nario export --format 2 -r "$new" --base "$old" --diff-algorithm foo | grepQuiet "unsupported nario diff algorithm 'foo'" + +# All fields in a nario are padded to 8 bytes, so anything else means +# that something (like HDiffPatch's progress messages) wrote junk to +# stdout. +(( $(stat -c %s "$TEST_ROOT"/diff.nario) % 8 == 0 )) + +nix nario export --format 2 -r "$new" --base "$old" --diff-algorithm zstd > "$TEST_ROOT"/diff-zstd-algo.nario +nix nario list < "$TEST_ROOT"/diff-zstd-algo.nario | grepQuiet "^$new: [0-9]* bytes, zstd diff against $old ([0-9]* bytes)$" +[[ $(nix nario list --json < "$TEST_ROOT"/diff-zstd-algo.nario | jq -r ".paths.\"$new\".diff.algorithm") = zstd ]] + +# The diff should be much smaller than the full export. +(( $(stat -c %s "$TEST_ROOT"/diff.nario) * 4 < $(stat -c %s "$TEST_ROOT"/full.nario) )) + +# Test compression of full NARs. +expectStderr 1 nix nario export --format 1 --compression zstd -r "$new" | grepQuiet "compression is only supported in nario version 2" +expectStderr 1 nix nario export --format 2 --compression foo -r "$new" | grepQuiet "unknown compression method 'foo'" +expectStderr 1 nix nario export --format 2 --compression-level 5 -r "$new" | grepQuiet "a compression level requires a compression method" + +# Higher compression levels should produce smaller narios. +nix nario export --format 2 --compression zstd --compression-level 1 -r "$new" > "$TEST_ROOT/full-zstd-1.nario" +nix nario export --format 2 --compression zstd --compression-level 19 -r "$new" > "$TEST_ROOT/full-zstd-19.nario" +(( $(stat -c %s "$TEST_ROOT/full-zstd-19.nario") < $(stat -c %s "$TEST_ROOT/full-zstd-1.nario") )) +nix nario import --no-check-sigs < "$TEST_ROOT/full-zstd-19.nario" + +for method in zstd xz; do + nix nario export --format 2 --compression "$method" -r "$new" > "$TEST_ROOT/full-$method.nario" + (( $(stat -c %s "$TEST_ROOT/full-$method.nario") < $(stat -c %s "$TEST_ROOT"/full.nario) )) + nix nario list < "$TEST_ROOT/full-$method.nario" | grepQuiet "^$extra: [0-9]* bytes, $method-compressed ([0-9]* bytes)$" + nix nario list -R < "$TEST_ROOT/full-$method.nario" | grepQuiet "^$extra/data$" + nix nario list -R < "$TEST_ROOT/full-$method.nario" | grepQuiet "^$new/data$" + json=$(nix nario list --json < "$TEST_ROOT/full-$method.nario") + [[ $(printf "%s" "$json" | jq -r ".paths.\"$new\".compression.method") = "$method" ]] + (( $(printf "%s" "$json" | jq -r ".paths.\"$new\".compression.size") < $(printf "%s" "$json" | jq -r ".paths.\"$new\".narSize") )) +done + +nix nario export --format 2 --compression zstd -r "$new" --base "$old" > "$TEST_ROOT"/diff-zstd.nario +(( $(stat -c %s "$TEST_ROOT"/diff-zstd.nario) < $(stat -c %s "$TEST_ROOT"/diff.nario) )) +nix nario list < "$TEST_ROOT"/diff-zstd.nario | grepQuiet "^$new: [0-9]* bytes, hdiffpatch diff against $old ([0-9]* bytes)$" +nix nario list < "$TEST_ROOT"/diff-zstd.nario | grepQuiet "^$extra: [0-9]* bytes, zstd-compressed ([0-9]* bytes)$" + +# Test `nix nario list`. +nix nario list < "$TEST_ROOT"/diff.nario | grepQuiet "^$new: [0-9]* bytes, hdiffpatch diff against $old ([0-9]* bytes)$" +nix nario list < "$TEST_ROOT"/diff.nario | grepQuiet "^$shared: expected to be present$" +nix nario list < "$TEST_ROOT"/diff.nario | grepQuiet "^$extra: [0-9]* bytes$" +nix nario list -R < "$TEST_ROOT"/diff.nario | grepQuiet "^$extra/data$" + +json=$(nix nario list --json < "$TEST_ROOT/diff.nario") +[[ $(printf "%s" "$json" | jq -r ".paths.\"$new\".diff.algorithm") = hdiffpatch ]] +[[ $(printf "%s" "$json" | jq -r ".paths.\"$new\".diff.base") = "$old" ]] +[[ $(printf "%s" "$json" | jq -r ".paths.\"$new\".diff.baseNarHash") = $(nix path-info --json "$old" | jq -r ".[].narHash") ]] +[[ $(printf "%s" "$json" | jq -r ".paths.\"$shared\".present") = true ]] +[[ $(printf "%s" "$json" | jq -r ".paths.\"$extra\".diff") = null ]] + +# Importing into an empty store fails because the base closure is missing. +clearStore +expectStderr 1 nix nario import --no-check-sigs < "$TEST_ROOT"/diff.nario | grepQuiet "nario requires path '$shared' to already be valid in the store" + +# Importing into a store that lacks the diff base fails. +clearStore +nix nario import --no-check-sigs < "$TEST_ROOT"/old.nario +nix-store --delete "$old" +expectStderr 1 nix nario import --no-check-sigs < "$TEST_ROOT"/diff.nario | grepQuiet "binary diff against path '$old', which is not valid" + +# Importing into a store that has the base closure succeeds, and preserves signatures. +clearStore +nix nario import --no-check-sigs < "$TEST_ROOT"/old.nario +expectStderr 1 nix nario import < "$TEST_ROOT"/diff.nario | grepQuiet "lacks a signature" +nix nario import --trusted-public-keys "$public_key" < "$TEST_ROOT"/diff.nario +[[ $(nix path-info --json "$new" | jq -r .[].signatures[]) =~ my-key: ]] +nix store verify --no-trust -r "$new" +grepQuiet "^2$" "$new/data" +[[ $(readlink "$new/extra") = "$extra" ]] + +# Importing again is a no-op. +nix nario import --no-check-sigs < "$TEST_ROOT"/diff.nario + +# Import compressed narios. +for method in zstd xz; do + clearStore + nix nario import --no-check-sigs < "$TEST_ROOT/full-$method.nario" + nix store verify --no-trust -r "$new" + [[ $(nix path-info --json "$new" | jq -r .[].signatures[]) =~ my-key: ]] +done + +clearStore +nix nario import --no-check-sigs < "$TEST_ROOT"/old.nario +nix nario import --no-check-sigs < "$TEST_ROOT"/diff-zstd.nario +nix store verify --no-trust -r "$new" + +clearStore +nix nario import --no-check-sigs < "$TEST_ROOT"/old.nario +nix nario import --no-check-sigs < "$TEST_ROOT"/diff-zstd-algo.nario +nix store verify --no-trust -r "$new" +grepQuiet "^2$" "$new/data" + +# Importing fails if the base has a different NAR hash. +clearStore +nix nario import --no-check-sigs < "$TEST_ROOT"/old.nario +chmod -R u+w "$old" +echo corrupt >> "$old/data" +expectStderr 1 nix nario import --no-check-sigs < "$TEST_ROOT"/diff.nario | grepQuiet "binary diff against path '$old', which has NAR hash '.*' instead of the expected"