diff --git a/NEWS.md b/NEWS.md index cc462e0b..1d7fc50a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,12 @@ # packrat (development version) +- `restore()` downloads the sources of packages from CRAN-like repositories + concurrently before installing them, when `curl` 7.75.0 or later is the + download method. Restores that install many packages no longer wait on one + download at a time. Set `options(packrat.prefetch.sources = FALSE)` to turn + this off, and `packrat.prefetch.concurrency` to change the number of + simultaneous downloads (default 8). + - Fixed a race condition where concurrent R processes installing the same package could fail while inserting it into the package cache. diff --git a/R/downloader.R b/R/downloader.R index 64900947..52ef99d4 100644 --- a/R/downloader.R +++ b/R/downloader.R @@ -95,37 +95,7 @@ downloadFile <- function( # pass extra arguments for 'curl' downloader if (method == "curl") { - # use '-L' to follow redirects - if (!grepl("\\b-L\\b", extra)) { - extra <- paste(extra, "-L") - } - - # switch off the curl globbing parser - if (!grepl("\\b-g\\b", extra)) { - extra <- paste(extra, "-g") - } - - # use '-f' to ensure we fail on server errors - if (!grepl("\\b-f\\b", extra)) { - extra <- paste(extra, "-f") - } - - # make curl quiet -- avoid polluting console with e.g. - # curl: (22) The requested URL returned error: 404 Not Found - if (!grepl("\\b-s\\b", extra)) { - extra <- paste(extra, "-s") - } - - # lower connection timeout - connect.timeout <- getOption("packrat.connect.timeout") - if (!is.null(connect.timeout) && !grepl("\\b--connect-timeout\\b", extra)) { - extra <- paste(extra, "--connect-timeout", connect.timeout) - } - - # redirect stderr to stdout, for nicer output in RStudio - if (!grepl("\\b--stderr -\\b", extra)) { - extra <- paste(extra, "--stderr -") - } + extra <- curlExtraArgs(extra) } # catch warnings in the call @@ -152,6 +122,44 @@ downloadFile <- function( return(result) } +# Add the arguments packrat always passes to curl to the user's 'extra' +# arguments, unless the user already supplied them. +curlExtraArgs <- function(extra) { + # use '-L' to follow redirects + if (!grepl("\\b-L\\b", extra)) { + extra <- paste(extra, "-L") + } + + # switch off the curl globbing parser + if (!grepl("\\b-g\\b", extra)) { + extra <- paste(extra, "-g") + } + + # use '-f' to ensure we fail on server errors + if (!grepl("\\b-f\\b", extra)) { + extra <- paste(extra, "-f") + } + + # make curl quiet -- avoid polluting console with e.g. + # curl: (22) The requested URL returned error: 404 Not Found + if (!grepl("\\b-s\\b", extra)) { + extra <- paste(extra, "-s") + } + + # lower connection timeout + connect.timeout <- getOption("packrat.connect.timeout") + if (!is.null(connect.timeout) && !grepl("\\b--connect-timeout\\b", extra)) { + extra <- paste(extra, "--connect-timeout", connect.timeout) + } + + # redirect stderr to stdout, for nicer output in RStudio + if (!grepl("\\b--stderr -\\b", extra)) { + extra <- paste(extra, "--stderr -") + } + + extra +} + # Attempt download.packages multiple times. # # Assumes we are downloading a single package. diff --git a/R/packrat.R b/R/packrat.R index 2fe52da4..73b9b1ba 100644 --- a/R/packrat.R +++ b/R/packrat.R @@ -324,6 +324,15 @@ initImpl <- function( #' locations, it will use R's internal \code{tar} implementation, which may #' cause errors with long filenames. #' +#' When \code{curl} 7.75.0 or later is the download method, Packrat +#' downloads package files from CRAN-like repositories concurrently before +#' it installs them. It skips packages from the cache and, on macOS and +#' Windows, packages that it installs from a binary repository. On Linux and +#' macOS, Packrat uses curl when \code{download.file.method} is not set. Set +#' \code{packrat.prefetch.sources} to \code{FALSE} to turn this off, or +#' \code{packrat.prefetch.concurrency} to change the number of simultaneous +#' downloads (default 8). +#' #' @param project The project directory. When in packrat mode, if this is #' \code{NULL}, then the directory associated with the current packrat project #' is used. Otherwise, the project directory specified is used. diff --git a/R/prefetch.R b/R/prefetch.R new file mode 100644 index 00000000..f7c2a3d5 --- /dev/null +++ b/R/prefetch.R @@ -0,0 +1,289 @@ +# Download package sources for a restore concurrently, before any package is +# installed. +# +# installPkg() downloads each package just before installing it, one curl +# process at a time. When most of the restore is spent in those downloads, +# fetching every needed source up front with a single `curl --parallel` is +# much faster. installPkg() skips the download for any source that is already +# in srcDir(), so the install loop is unchanged; anything the prefetch fails +# to fetch is downloaded there as before. +# +# The prefetch only handles packages from CRAN-like repositories, and only +# when curl is the download method. It never fails the restore. + +prefetchInstallActions <- c("add", "upgrade", "downgrade", "crossgrade") + +# Minimum curl version: --parallel arrived in 7.66.0 and the %{exitcode} +# write-out variable in 7.75.0. +prefetchMinimumCurlVersion <- "7.75.0" + +prefetchEnabled <- function() { + isTRUE(getOption("packrat.prefetch.sources", TRUE)) +} + +prefetchConcurrency <- function() { + concurrency <- as.integer(getOption("packrat.prefetch.concurrency", 8L)) + if (length(concurrency) != 1L || is.na(concurrency) || concurrency < 1L) { + stop("'packrat.prefetch.concurrency' must be a positive integer") + } + concurrency +} + +# Returns the curl version as a numeric_version, or NULL if curl can't be run. +curlVersion <- function() { + output <- tryCatch( + suppressWarnings(system2( + "curl", + "--version", + stdout = TRUE, + stderr = FALSE + )), + error = function(e) character() + ) + version <- regmatches( + output[1], + regexpr("(?<=^curl )[0-9.]+", output[1], perl = TRUE) + ) + if (!length(version)) { + return(NULL) + } + numeric_version(version) +} + +# Package records that will be installed and need a source download: from a +# CRAN-like repository, not in the cache, and not already in srcDir(). Uses +# only local checks, so a restore served entirely from the cache doesn't +# contact the repository. +prefetchCandidates <- function(pkgRecords, actions, repos, project) { + installs <- names(actions)[actions %in% prefetchInstallActions] + if (!length(installs) || !length(repos)) { + return(list()) + } + + records <- Filter(Negate(is.null), searchPackages(pkgRecords, installs)) + Filter( + function(pkgRecord) { + isFromCranlikeRepo(pkgRecord, repos) && + !identical(pkgRecord$source, "unknown") && + is.null(cachedPackagePath(project, pkgRecord)) && + !file.exists(prefetchDestfile(pkgRecord, project)) + }, + records + ) +} + +prefetchDestfile <- function(pkgRecord, project) { + file.path(srcDir(project), pkgRecord$name, pkgSrcFilename(pkgRecord)) +} + +# Download URLs and destination for each candidate. Returns a data frame with +# one row per URL to try, in the order to try them: name, version, url, and +# destfile. Mirrors the URL choice in getSourceForPkgRecord(): the +# repository's current file when the version matches available.packages(), +# otherwise each archive location downloadArchiveWithRetries() tries. +prefetchTargets <- function(records, repos, project, available) { + rows <- lapply(records, function(pkgRecord) { + current <- pkgRecord$name %in% + rownames(available) && + identical(pkgRecord$version, available[pkgRecord$name, "Version"]) + url <- if (current) { + paste( + available[pkgRecord$name, "Repository"], + pkgSrcFilename(pkgRecord), + sep = "/" + ) + } else { + archiveUrls(repos, pkgRecord) + } + data.frame( + name = pkgRecord$name, + version = pkgRecord$version, + url = url, + destfile = prefetchDestfile(pkgRecord, project), + stringsAsFactors = FALSE + ) + }) + do.call(rbind, rows) +} + +# Quote a value for a curl config file. +curlConfigQuote <- function(value) { + paste0("\"", gsub("([\"\\\\])", "\\\\\\1", value), "\"") +} + +# Send R's user agent unless 'extra' already sets one. Repositories such as +# Posit Package Manager use it to decide whether to serve binaries, and a +# single-package download (through libcurl, or through curl with a +# download.file.extra that sets it) sends it too. +prefetchUserAgentArgs <- function(extra) { + userAgent <- getOption("HTTPUserAgent") + setsUserAgent <- grepl("(^|\\s)(-A|--user-agent)(\\s|=|$)", extra) || + grepl("user-agent:", extra, ignore.case = TRUE) + if (length(userAgent) != 1L || !nzchar(userAgent) || setsUserAgent) { + return(extra) + } + paste(extra, "-A", shQuote(userAgent)) +} + +# Download every target with one `curl --parallel`. Each file is written to a +# temporary name and moved into place only when its transfer succeeded, so a +# failed or partial download never looks like a usable source. Returns the +# targets that were not downloaded. +prefetchDownload <- function(targets, concurrency) { + partial <- paste0(targets$destfile, ".prefetch") + for (dir in unique(dirname(targets$destfile))) { + dir.create(dir, recursive = TRUE, showWarnings = FALSE) + } + on.exit(unlink(partial[file.exists(partial)]), add = TRUE) + + config <- tempfile("packrat-prefetch-", fileext = ".txt") + on.exit(unlink(config), add = TRUE) + writeLines( + paste0( + "url = ", + curlConfigQuote(targets$url), + "\noutput = ", + curlConfigQuote(partial) + ), + config + ) + + # The same arguments a single-package download uses, including the user's + # download.file.extra (user agent, .netrc file, and so on). The trailing + # --write-out replaces any earlier one so each transfer reports its result. + extra <- "" + if (identical(getOption("download.file.method"), "curl")) { + extra <- paste(getOption("download.file.extra", ""), collapse = " ") + } + extra <- prefetchUserAgentArgs(extra) + writeOut <- "packrat-prefetch %{exitcode} %{http_code} %{filename_effective}\\n" + command <- paste( + "curl", + "--speed-limit 1 --speed-time", + as.integer(getOption("timeout", 60)), + curlExtraArgs(extra), + "--parallel --parallel-max", + concurrency, + "-K", + shQuote(config), + "-w", + shQuote(writeOut) + ) + output <- suppressWarnings(system(command, intern = TRUE)) + + results <- regmatches( + output, + regexec("^packrat-prefetch ([0-9]+) ([0-9]+) (.*)$", output) + ) + results <- Filter(length, results) + succeeded <- vapply(results, function(r) r[2] == "0", logical(1)) + succeededFiles <- vapply(results[succeeded], `[`, character(1), 4) + + ok <- partial %in% succeededFiles & file.exists(partial) + ok[ok] <- file.rename(partial[ok], targets$destfile[ok]) + + failed <- targets[!ok, , drop = FALSE] + failed$http_code <- vapply( + partial[!ok], + function(file) { + match <- Filter(function(r) identical(r[4], file), results) + if (length(match)) match[[1]][3] else NA_character_ + }, + character(1), + USE.NAMES = FALSE + ) + failed +} + +# Download each package from the first of its URLs that works. Each round +# tries the next URL of every package not yet downloaded, with one +# prefetchDownload(). Returns the last failed attempt of each package that +# could not be downloaded. +prefetchWithRetries <- function(targets, concurrency) { + failed <- targets[0, , drop = FALSE] + failed$http_code <- character() + while (nrow(targets)) { + first <- !duplicated(targets$name) + attemptFailed <- prefetchDownload( + targets[first, , drop = FALSE], + concurrency + ) + targets <- targets[ + !first & targets$name %in% attemptFailed$name, + , + drop = FALSE + ] + failed <- rbind( + failed, + attemptFailed[!attemptFailed$name %in% targets$name, , drop = FALSE] + ) + } + failed +} + +prefetchPackageSources <- function(pkgRecords, actions, repos, project) { + if (!prefetchEnabled()) { + return(invisible()) + } + + tryCatch( + { + candidates <- prefetchCandidates(pkgRecords, actions, repos, project) + # installPkg() installs these from a binary repository and doesn't need + # their sources. + candidates <- Filter( + function(pkgRecord) !installsFromBinaryRepository(pkgRecord, repos), + candidates + ) + if (!length(candidates)) { + return(invisible()) + } + available <- availablePackagesSource(repos = repos) + targets <- prefetchTargets(candidates, repos, project, available) + if (!identical(inferAppropriateDownloadMethod(targets$url[1]), "curl")) { + return(invisible()) + } + version <- curlVersion() + if (is.null(version) || version < prefetchMinimumCurlVersion) { + return(invisible()) + } + + concurrency <- prefetchConcurrency() + packages <- length(unique(targets$name)) + message( + "Prefetching sources for ", + packages, + " packages (", + concurrency, + " at a time) ... ", + appendLF = FALSE + ) + start <- Sys.time() + failed <- prefetchWithRetries(targets, concurrency) + elapsed <- as.numeric(difftime(Sys.time(), start, units = "secs")) + message(sprintf( + "fetched %d of %d in %.1f seconds", + packages - nrow(failed), + packages, + elapsed + )) + for (i in seq_len(nrow(failed))) { + message(sprintf( + "\tCould not prefetch %s (%s) from %s (HTTP %s); it will be downloaded during installation", + failed$name[i], + failed$version[i], + failed$url[i], + failed$http_code[i] + )) + } + }, + error = function(e) { + warning( + "Prefetching package sources failed; packages will be downloaded during installation: ", + conditionMessage(e), + call. = FALSE + ) + } + ) + invisible() +} diff --git a/R/restore-routines.R b/R/restore-routines.R index 30c63043..a70c57ac 100644 --- a/R/restore-routines.R +++ b/R/restore-routines.R @@ -37,32 +37,43 @@ hashTarball <- function(path) { tools::md5sum(files = normalizePath(path, mustWork = TRUE)) } -restoreWithCopyFromCache <- function(project, pkgRecord, cacheCopyStatus) { +# Returns the path to a usable global cache entry for this package record, or +# NULL when the package must be installed from somewhere else. +cachedPackagePath <- function(project, pkgRecord) { # don't copy from cache if disabled for this project if (!isUsingCache(project)) { - return(FALSE) + return(NULL) } # don't try to use cache if we don't have a hash if (!length(pkgRecord$hash)) { - return(FALSE) + return(NULL) } # don't try to cache uncacheable packages (ie, packages that # need to be reinstalled each time for whatever reason) if (!isCacheable(pkgRecord$name)) { - return(FALSE) + return(NULL) } # ensure that the cache package path exists source <- cacheLibDir(pkgRecord$name, pkgRecord$hash, pkgRecord$name) if (!file_test("-d", source)) { - return(FALSE) + return(NULL) } # sanity check for cache corruption -- we've seen some cases where # a cache entry exists, but it's just an empty folder if (isCorruptPackageCacheEntry(source)) { + return(NULL) + } + + source +} + +restoreWithCopyFromCache <- function(project, pkgRecord, cacheCopyStatus) { + source <- cachedPackagePath(project, pkgRecord) + if (is.null(source)) { return(FALSE) } diff --git a/R/restore.R b/R/restore.R index 761f47b3..1c5ad4c9 100644 --- a/R/restore.R +++ b/R/restore.R @@ -426,8 +426,9 @@ useAlternativeArchiveLocations <- function() { getOption("packrat.alternative.archive.layouts", TRUE) } -# Returns TRUE when the package is downloaded, FALSE otherwise. -downloadArchiveWithRetries <- function(repos, pkgRecord, destFile) { +# The URLs where an archived version of a package may be, in the order to try +# them. +archiveUrls <- function(repos, pkgRecord) { pkgSrcFile <- pkgSrcFilename(pkgRecord) # Many repositories follow the CRAN structure, but not all. Try the CRAN @@ -465,25 +466,29 @@ downloadArchiveWithRetries <- function(repos, pkgRecord, destFile) { ) } - foundVersion <- FALSE - for (formatter in formatters) { - for (repo in repos) { - archiveUrl <- formatter(repo) - tryCatch( - { - downloadWithRetries( - archiveUrl, - destfile = destFile, - mode = "wb", - quiet = TRUE - ) - return(TRUE) - }, - error = function(e) { - # ignore error and try next repo/archive path. - } - ) - } + repos <- sub("/+$", "", repos) + unlist(lapply(formatters, function(formatter) { + vapply(repos, formatter, character(1), USE.NAMES = FALSE) + })) +} + +# Returns TRUE when the package is downloaded, FALSE otherwise. +downloadArchiveWithRetries <- function(repos, pkgRecord, destFile) { + for (archiveUrl in archiveUrls(repos, pkgRecord)) { + tryCatch( + { + downloadWithRetries( + archiveUrl, + destfile = destFile, + mode = "wb", + quiet = TRUE + ) + return(TRUE) + }, + error = function(e) { + # ignore error and try next repo/archive path. + } + ) } return(FALSE) } @@ -572,6 +577,16 @@ removePkgs <- function(project, pkgNames, lib.loc = libDir(project)) { pkgNames } +# Whether installPkg() tries to install a package as a binary from a binary +# repository, rather than from its sources. +installsFromBinaryRepository <- function(pkgRecord, repos) { + hasBinaryRepositories() && + binaryRepositoriesEnabled() && + isFromCranlikeRepo(pkgRecord, repos) && + pkgRecord$name %in% availablePackagesBinary(repos = repos)[, "Package"] && + versionMatchesDb(pkgRecord, availablePackagesBinary(repos = repos)) +} + # Installs a single package from its record. Returns the method used to install # the package (built source, downloaded binary, etc.) installPkg <- function(pkgRecord, project, repos, lib = libDir(project)) { @@ -628,13 +643,7 @@ installPkg <- function(pkgRecord, project, repos, lib = libDir(project)) { needsInstall <- TRUE # Try downloading a binary (when appropriate). - if ( - hasBinaryRepositories() && - binaryRepositoriesEnabled() && - isFromCranlikeRepo(pkgRecord, repos) && - pkgRecord$name %in% availablePackagesBinary(repos = repos)[, "Package"] && - versionMatchesDb(pkgRecord, availablePackagesBinary(repos = repos)) - ) { + if (installsFromBinaryRepository(pkgRecord, repos)) { tempdir <- tempdir() tryCatch( { @@ -994,6 +1003,7 @@ restoreImpl <- function( # Play the list, if there's anything to play if (!dry.run) { + prefetchPackageSources(pkgRecords, actions, repos, project) playActions(pkgRecords, actions, repos, project, targetLib) if (restartNeeded) { if (!restart || !attemptRestart()) { diff --git a/man/restore.Rd b/man/restore.Rd index 990fbb96..290c3e18 100644 --- a/man/restore.Rd +++ b/man/restore.Rd @@ -92,6 +92,15 @@ will be lost. look for the system \code{tar} on Windows. If no binary is found in those locations, it will use R's internal \code{tar} implementation, which may cause errors with long filenames. + + When \code{curl} 7.75.0 or later is the download method, Packrat + downloads the sources of every package it needs from CRAN-like + repositories concurrently, before installing any of them. Set the option + \code{packrat.prefetch.sources} to \code{FALSE} to download each package + just before it is installed instead. The option + \code{packrat.prefetch.concurrency} sets the number of simultaneous + downloads (default 8). Packages that fail to download concurrently are + downloaded again before they are installed. } \seealso{ \code{\link{snapshot}}, the command that creates the snapshots diff --git a/tests/testthat/helper.R b/tests/testthat/helper.R new file mode 100644 index 00000000..56ccf8de --- /dev/null +++ b/tests/testthat/helper.R @@ -0,0 +1,24 @@ +pkgRecord <- function(name, version, source = "CRAN", ...) { + list(name = name, version = version, source = source, hash = "hash", ...) +} + +availableMatrix <- function(versions, repository) { + matrix( + c(versions, rep(repository, length(versions))), + ncol = 2, + dimnames = list(names(versions), c("Version", "Repository")) + ) +} + +skip_if_no_parallel_curl <- function() { + version <- curlVersion() + if (is.null(version) || version < prefetchMinimumCurlVersion) { + skip("curl with --parallel and %{exitcode} is not available") + } +} + +# A file:// URL for a local directory, on any platform. +fileURL <- function(path) { + path <- normalizePath(path, winslash = "/") + paste0(if (startsWith(path, "/")) "file://" else "file:///", path) +} diff --git a/tests/testthat/test-prefetch.R b/tests/testthat/test-prefetch.R new file mode 100644 index 00000000..b2b5e4cc --- /dev/null +++ b/tests/testthat/test-prefetch.R @@ -0,0 +1,392 @@ +test_that("prefetchTargets uses the current file or the archive locations", { + srcDir <- withr::local_tempdir() + withr::local_envvar(R_PACKRAT_SRC_DIR = srcDir) + withr::local_options(packrat.alternative.archive.layouts = FALSE) + local_mocked_bindings(cachedPackagePath = function(project, pkgRecord) NULL) + + repo <- "https://example.com/cran" + available <- availableMatrix( + c(current = "1.0", older = "2.0"), + paste0(repo, "/src/contrib") + ) + records <- list( + pkgRecord("current", "1.0"), + pkgRecord("older", "1.5"), + pkgRecord("missing", "0.1") + ) + actions <- c(current = "add", older = "downgrade", missing = "add") + + repos <- c(CRAN = paste0(repo, "/"), other = "https://example.com/other") + targets <- prefetchTargets( + prefetchCandidates(records, actions, repos, NULL), + repos, + NULL, + available + ) + + expect_equal( + targets$name, + c("current", "older", "older", "missing", "missing") + ) + expect_equal( + targets$url, + c( + "https://example.com/cran/src/contrib/current_1.0.tar.gz", + "https://example.com/cran/src/contrib/Archive/older/older_1.5.tar.gz", + "https://example.com/other/src/contrib/Archive/older/older_1.5.tar.gz", + "https://example.com/cran/src/contrib/Archive/missing/missing_0.1.tar.gz", + "https://example.com/other/src/contrib/Archive/missing/missing_0.1.tar.gz" + ) + ) + expect_equal( + targets$destfile, + file.path( + srcDir, + c("current", "older", "older", "missing", "missing"), + c( + "current_1.0.tar.gz", + "older_1.5.tar.gz", + "older_1.5.tar.gz", + "missing_0.1.tar.gz", + "missing_0.1.tar.gz" + ) + ) + ) +}) + +test_that("archiveUrls tries every repository with each archive layout", { + repos <- c( + CRAN = "https://example.com/cran/", + other = "https://example.com/other" + ) + record <- pkgRecord("pkg", "1.0") + + expect_equal( + archiveUrls(repos, record), + c( + "https://example.com/cran/src/contrib/Archive/pkg/pkg_1.0.tar.gz", + "https://example.com/other/src/contrib/Archive/pkg/pkg_1.0.tar.gz", + "https://example.com/cran/src/contrib/Archive/pkg/1.0/pkg_1.0.tar.gz", + "https://example.com/other/src/contrib/Archive/pkg/1.0/pkg_1.0.tar.gz", + "https://example.com/cran/src/contrib/pkg_1.0.tar.gz", + "https://example.com/other/src/contrib/pkg_1.0.tar.gz" + ) + ) + + withr::local_options(packrat.alternative.archive.layouts = FALSE) + expect_equal( + archiveUrls(repos, record), + c( + "https://example.com/cran/src/contrib/Archive/pkg/pkg_1.0.tar.gz", + "https://example.com/other/src/contrib/Archive/pkg/pkg_1.0.tar.gz" + ) + ) +}) + +test_that("prefetchWithRetries tries each package's next URL until one works", { + attempts <- list() + local_mocked_bindings( + prefetchDownload = function(targets, concurrency) { + attempts[[length(attempts) + 1L]] <<- targets$url + failed <- targets[!grepl("good", targets$url), , drop = FALSE] + failed$http_code <- rep("404", nrow(failed)) + failed + } + ) + + targets <- data.frame( + name = c("first", "second", "second", "never", "never"), + version = "1.0", + url = c( + "good/first", + "bad/second", + "good/second", + "bad/never", + "worse/never" + ), + destfile = "unused", + stringsAsFactors = FALSE + ) + + failed <- prefetchWithRetries(targets, concurrency = 2L) + + expect_equal( + attempts, + list( + c("good/first", "bad/second", "bad/never"), + c("good/second", "worse/never") + ) + ) + expect_equal(failed$name, "never") + expect_equal(failed$url, "worse/never") + expect_equal(failed$http_code, "404") +}) + +test_that("prefetchCandidates skips packages that won't be downloaded from a repository", { + srcDir <- withr::local_tempdir() + withr::local_envvar(R_PACKRAT_SRC_DIR = srcDir) + local_mocked_bindings( + cachedPackagePath = function(project, pkgRecord) { + if (identical(pkgRecord$name, "cached")) "/cache/cached" else NULL + } + ) + dir.create(file.path(srcDir, "present")) + file.create(file.path(srcDir, "present", "present_1.0.tar.gz")) + + records <- list( + pkgRecord("wanted", "1.0"), + pkgRecord("cached", "1.0"), + pkgRecord("present", "1.0"), + pkgRecord( + "fromgit", + "1.0", + source = "github", + remote_host = "api.github.com", + remote_repo = "fromgit" + ), + pkgRecord("local", "1.0", source = "source"), + pkgRecord("removed", "1.0") + ) + actions <- c( + wanted = "add", + cached = "add", + present = "add", + fromgit = "add", + local = "add", + removed = "remove" + ) + + candidates <- prefetchCandidates( + records, + actions, + c(CRAN = "https://example.com/cran"), + NULL + ) + + expect_equal(vapply(candidates, `[[`, character(1), "name"), "wanted") +}) + +test_that("prefetchDownload moves successful downloads into place and reports failures", { + skip_if_no_parallel_curl() + skip_on_cran() + + repo <- withr::local_tempdir() + writeLines("one", file.path(repo, "one_1.0.tar.gz")) + writeLines("two", file.path(repo, "two_1.0.tar.gz")) + srcDir <- withr::local_tempdir() + withr::local_options( + download.file.method = "curl", + download.file.extra = NULL + ) + + repoURL <- fileURL(repo) + targets <- data.frame( + name = c("one", "two", "gone"), + version = "1.0", + url = paste0(repoURL, "/", c("one", "two", "gone"), "_1.0.tar.gz"), + destfile = file.path( + srcDir, + c("one", "two", "gone"), + c("one_1.0.tar.gz", "two_1.0.tar.gz", "gone_1.0.tar.gz") + ), + stringsAsFactors = FALSE + ) + + failed <- prefetchDownload(targets, concurrency = 2L) + + expect_equal(failed$name, "gone") + expect_equal(readLines(targets$destfile[1]), "one") + expect_equal(readLines(targets$destfile[2]), "two") + expect_false(file.exists(targets$destfile[3])) + expect_length( + list.files(srcDir, pattern = "\\.prefetch$", recursive = TRUE), + 0 + ) +}) + +test_that("prefetchPackageSources does nothing without a capable curl", { + local_mocked_bindings( + hasBinaryRepositories = function() FALSE, + availablePackagesSource = function(repos) { + availableMatrix(c(pkg = "1.0"), "https://example.com/cran/src/contrib") + }, + cachedPackagePath = function(project, pkgRecord) NULL, + inferAppropriateDownloadMethod = function(url) "curl", + curlVersion = function() numeric_version("7.61.1"), + prefetchDownload = function(targets, concurrency) { + stop("should not download") + } + ) + withr::local_envvar(R_PACKRAT_SRC_DIR = withr::local_tempdir()) + + expect_silent( + prefetchPackageSources( + list(pkgRecord("pkg", "1.0")), + c(pkg = "add"), + c(CRAN = "https://example.com/cran"), + NULL + ) + ) +}) + +test_that("prefetchPackageSources turns errors into a warning", { + local_mocked_bindings( + hasBinaryRepositories = function() FALSE, + availablePackagesSource = function(repos) { + availableMatrix(c(pkg = "1.0"), "https://example.com/cran/src/contrib") + }, + cachedPackagePath = function(project, pkgRecord) NULL, + inferAppropriateDownloadMethod = function(url) "curl", + curlVersion = function() numeric_version("8.5.0"), + prefetchDownload = function(targets, concurrency) stop("network is down") + ) + withr::local_envvar(R_PACKRAT_SRC_DIR = withr::local_tempdir()) + + expect_warning( + suppressMessages( + prefetchPackageSources( + list(pkgRecord("pkg", "1.0")), + c(pkg = "add"), + c(CRAN = "https://example.com/cran"), + NULL + ) + ), + "network is down" + ) +}) + +test_that("prefetchPackageSources doesn't contact the repository when everything is cached", { + local_mocked_bindings( + hasBinaryRepositories = function() FALSE, + cachedPackagePath = function(project, pkgRecord) "/cache/pkg", + availablePackagesSource = function(repos) { + stop("should not fetch the package index") + } + ) + withr::local_envvar(R_PACKRAT_SRC_DIR = withr::local_tempdir()) + + expect_silent( + prefetchPackageSources( + list(pkgRecord("pkg", "1.0")), + c(pkg = "add"), + c(CRAN = "https://example.com/cran"), + NULL + ) + ) +}) + +test_that("prefetchPackageSources can be turned off", { + local_mocked_bindings( + availablePackagesSource = function(repos) stop("should not run") + ) + withr::local_options(packrat.prefetch.sources = FALSE) + + expect_silent( + prefetchPackageSources( + list(pkgRecord("pkg", "1.0")), + c(pkg = "add"), + c(CRAN = "https://example.com/cran"), + NULL + ) + ) +}) + +test_that("prefetchUserAgentArgs sends R's user agent unless one is set", { + withr::local_options(HTTPUserAgent = "R (4.5.0 x86_64-pc-linux-gnu)") + + expect_equal( + prefetchUserAgentArgs(""), + paste("", "-A", shQuote("R (4.5.0 x86_64-pc-linux-gnu)")) + ) + expect_equal(prefetchUserAgentArgs("-A custom"), "-A custom") + expect_equal( + prefetchUserAgentArgs("--user-agent=custom"), + "--user-agent=custom" + ) + expect_equal( + prefetchUserAgentArgs("-H 'User-Agent: custom'"), + "-H 'User-Agent: custom'" + ) + + withr::local_options(HTTPUserAgent = NULL) + expect_equal(prefetchUserAgentArgs("-L"), "-L") +}) + +test_that("prefetchPackageSources skips packages installed from a binary repository", { + local_mocked_bindings( + hasBinaryRepositories = function() TRUE, + binaryRepositoriesEnabled = function() TRUE, + availablePackagesBinary = function(repos) { + cbind( + Package = "binary", + availableMatrix(c(binary = "1.0"), "https://example.com/cran/bin") + ) + }, + availablePackagesSource = function(repos) { + availableMatrix( + c(binary = "1.0", source = "1.0"), + "https://example.com/cran/src/contrib" + ) + }, + cachedPackagePath = function(project, pkgRecord) NULL, + inferAppropriateDownloadMethod = function(url) "curl", + curlVersion = function() numeric_version("8.5.0"), + prefetchDownload = function(targets, concurrency) { + fetched <<- targets$name + targets[0, , drop = FALSE] + } + ) + withr::local_envvar(R_PACKRAT_SRC_DIR = withr::local_tempdir()) + fetched <- NULL + + suppressMessages( + prefetchPackageSources( + list(pkgRecord("binary", "1.0"), pkgRecord("source", "1.0")), + c(binary = "add", source = "add"), + c(CRAN = "https://example.com/cran"), + NULL + ) + ) + + expect_equal(fetched, "source") +}) + +test_that("installPkg uses a prefetched source without downloading it again", { + skip_if_no_parallel_curl() + skip_on_cran() + + repo <- withr::local_tempdir() + writeLines("pkg", file.path(repo, "pkg_1.0.tar.gz")) + srcDir <- withr::local_tempdir() + withr::local_envvar(R_PACKRAT_SRC_DIR = srcDir) + withr::local_options(download.file.extra = NULL) + + installed <- NULL + local_mocked_bindings( + hasBinaryRepositories = function() FALSE, + availablePackagesSource = function(repos) { + availableMatrix(c(pkg = "1.0"), fileURL(repo)) + }, + cachedPackagePath = function(project, pkgRecord) NULL, + inferAppropriateDownloadMethod = function(url) "curl", + restoreWithCopyFromCache = function(...) FALSE, + restoreWithCopyFromUntrustedCache = function(...) FALSE, + getSourceForPkgRecord = function(...) stop("should not download"), + archivePackageType = function(...) "source", + install_local_path = function(path, ...) installed <<- path, + annotatePkgDesc = function(...) NULL, + isUsingCache = function(project) FALSE + ) + + record <- pkgRecord("pkg", "1.0") + repos <- c(CRAN = "https://example.com/cran") + suppressMessages( + prefetchPackageSources(list(record), c(pkg = "add"), repos, NULL) + ) + prefetched <- file.path(srcDir, "pkg", "pkg_1.0.tar.gz") + expect_equal(readLines(prefetched), "pkg") + + installPkg(record, NULL, repos, lib = withr::local_tempdir()) + + expect_equal(installed, prefetched) +})