restore: download package sources concurrently before installing

NEWS.md

@@ -1,5 +1,12 @@
 # packrat (development version)
 
+- `restore()` downloads the sources of packages from CRAN-like repositories
+  concurrently before installing them, when `curl` 7.75.0 or later is the
+  download method. Restores that install many packages no longer wait on one
+  download at a time. Set `options(packrat.prefetch.sources = FALSE)` to turn
+  this off, and `packrat.prefetch.concurrency` to change the number of
+  simultaneous downloads (default 8).
+
 - Fixed a race condition where concurrent R processes installing the same
   package could fail while inserting it into the package cache.
 

R/downloader.R

@@ -95,37 +95,7 @@ downloadFile <- function(
 
   # pass extra arguments for 'curl' downloader
   if (method == "curl") {
-    # use '-L' to follow redirects
-    if (!grepl("\\b-L\\b", extra)) {
-      extra <- paste(extra, "-L")
-    }
-
-    # switch off the curl globbing parser
-    if (!grepl("\\b-g\\b", extra)) {
-      extra <- paste(extra, "-g")
-    }
-
-    # use '-f' to ensure we fail on server errors
-    if (!grepl("\\b-f\\b", extra)) {
-      extra <- paste(extra, "-f")
-    }
-
-    # make curl quiet -- avoid polluting console with e.g.
-    # curl: (22) The requested URL returned error: 404 Not Found
-    if (!grepl("\\b-s\\b", extra)) {
-      extra <- paste(extra, "-s")
-    }
-
-    # lower connection timeout
-    connect.timeout <- getOption("packrat.connect.timeout")
-    if (!is.null(connect.timeout) && !grepl("\\b--connect-timeout\\b", extra)) {
-      extra <- paste(extra, "--connect-timeout", connect.timeout)
-    }
-
-    # redirect stderr to stdout, for nicer output in RStudio
-    if (!grepl("\\b--stderr -\\b", extra)) {
-      extra <- paste(extra, "--stderr -")
-    }
+    extra <- curlExtraArgs(extra)
   }
 
   # catch warnings in the call
@@ -152,6 +122,44 @@ downloadFile <- function(
   return(result)
 }
 
+# Add the arguments packrat always passes to curl to the user's 'extra'
+# arguments, unless the user already supplied them.
+curlExtraArgs <- function(extra) {
+  # use '-L' to follow redirects
+  if (!grepl("\\b-L\\b", extra)) {
+    extra <- paste(extra, "-L")
+  }
+
+  # switch off the curl globbing parser
+  if (!grepl("\\b-g\\b", extra)) {
+    extra <- paste(extra, "-g")
+  }
+
+  # use '-f' to ensure we fail on server errors
+  if (!grepl("\\b-f\\b", extra)) {
+    extra <- paste(extra, "-f")
+  }
+
+  # make curl quiet -- avoid polluting console with e.g.
+  # curl: (22) The requested URL returned error: 404 Not Found
+  if (!grepl("\\b-s\\b", extra)) {
+    extra <- paste(extra, "-s")
+  }
+
+  # lower connection timeout
+  connect.timeout <- getOption("packrat.connect.timeout")
+  if (!is.null(connect.timeout) && !grepl("\\b--connect-timeout\\b", extra)) {
+    extra <- paste(extra, "--connect-timeout", connect.timeout)
+  }
+
+  # redirect stderr to stdout, for nicer output in RStudio
+  if (!grepl("\\b--stderr -\\b", extra)) {
+    extra <- paste(extra, "--stderr -")
+  }
+
+  extra
+}
+
 # Attempt download.packages multiple times.
 #
 # Assumes we are downloading a single package.

R/packrat.R

@@ -324,6 +324,15 @@ initImpl <- function(
 #'   locations, it will use R's internal \code{tar} implementation, which may
 #'   cause errors with long filenames.
 #'
+#'   When \code{curl} 7.75.0 or later is the download method, Packrat
+#'   downloads package files from CRAN-like repositories concurrently before
+#'   it installs them. It skips packages from the cache and, on macOS and
+#'   Windows, packages that it installs from a binary repository. On Linux and
+#'   macOS, Packrat uses curl when \code{download.file.method} is not set. Set
+#'   \code{packrat.prefetch.sources} to \code{FALSE} to turn this off, or
+#'   \code{packrat.prefetch.concurrency} to change the number of simultaneous
+#'   downloads (default 8).
+#'
 #' @param project The project directory. When in packrat mode, if this is
 #'   \code{NULL}, then the directory associated with the current packrat project
 #'   is used. Otherwise, the project directory specified is used.