diff --git a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala index 5e923ad35a3ab..e420df3102ad7 100644 --- a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala +++ b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala @@ -34,7 +34,7 @@ import org.apache.ivy.plugins.matcher.GlobPatternMatcher import org.apache.ivy.plugins.repository.file.FileRepository import org.apache.ivy.plugins.resolver.{ChainResolver, FileSystemResolver, IBiblioResolver} -import org.apache.spark.SparkException +import org.apache.spark.{SparkBuildInfo, SparkException} import org.apache.spark.internal.{Logging, LogKeys} import org.apache.spark.util.ArrayImplicits._ @@ -42,6 +42,31 @@ import org.apache.spark.util.ArrayImplicits._ private[spark] object MavenUtils extends Logging { val JAR_IVY_SETTING_PATH_KEY: String = "spark.jars.ivySettings" + /** System property Ivy reads to determine the HTTP User-Agent. */ + private[util] val USER_AGENT_PROPERTY = "http.agent" + + /** + * User-Agent identifying Spark to remote artifact repositories, in the form + * `product RWS comment` per RFC 9110 Section 10.1.5, e.g. + * `Apache-Spark/4.0.0 (Apache-Ivy/2.5.3)`. Repository operators use this to attribute traffic + * and to identify the tool and version behind it. + */ + private[util] lazy val sparkUserAgent: String = + s"Apache-Spark/${SparkBuildInfo.spark_version} (Apache-Ivy/${Ivy.getIvyVersion})" + + /** + * Set the `http.agent` system property so Ivy identifies itself as Spark when fetching from + * remote repositories. Set once and left in place: Ivy's HttpClient-backed handler captures the + * User-Agent when its singleton is class-initialized, so restoring the previous value afterwards + * would have no effect on requests and would race with concurrent resolution. An explicitly + * configured value always wins, so an operator can still override with `-Dhttp.agent=...`. + */ + private def setUserAgentIfAbsent(): Unit = { + if (System.getProperty(USER_AGENT_PROPERTY) == null) { + System.setProperty(USER_AGENT_PROPERTY, sparkUserAgent) + } + } + // Exposed for testing // var printStream = SparkSubmit.printStream @@ -277,6 +302,7 @@ private[spark] object MavenUtils extends Logging { remoteRepos: Option[String], ivyPath: Option[String], useLocalM2AsCache: Boolean = true)(implicit printStream: PrintStream): IvySettings = { + setUserAgentIfAbsent() val ivySettings: IvySettings = new IvySettings processIvyPathArg(ivySettings, ivyPath) @@ -310,6 +336,7 @@ private[spark] object MavenUtils extends Logging { */ def loadIvySettings(settingsFile: String, remoteRepos: Option[String], ivyPath: Option[String])( implicit printStream: PrintStream): IvySettings = { + setUserAgentIfAbsent() val uri = new URI(settingsFile) val file = Option(uri.getScheme).getOrElse("file") match { case "file" => new File(uri.getPath) diff --git a/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala b/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala index d30422ca8dd54..294cbb9e28ee4 100644 --- a/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala +++ b/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala @@ -25,12 +25,14 @@ import java.nio.file.{Files, Paths} import scala.collection.mutable.ArrayBuffer import scala.jdk.CollectionConverters._ +import org.apache.ivy.Ivy import org.apache.ivy.core.module.descriptor.MDArtifact import org.apache.ivy.core.settings.IvySettings import org.apache.ivy.plugins.resolver.{AbstractResolver, ChainResolver, FileSystemResolver, IBiblioResolver} import org.scalatest.BeforeAndAfterEach import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite +import org.apache.spark.SparkBuildInfo import org.apache.spark.util.MavenUtils.MavenCoordinate class MavenUtilsSuite @@ -39,6 +41,8 @@ class MavenUtilsSuite private var tempIvyPath: String = _ + private var originalUserAgent: Option[String] = None + private val noOpOutputStream = new OutputStream { def write(b: Int) = {} } @@ -57,10 +61,18 @@ class MavenUtilsSuite override def beforeEach(): Unit = { super.beforeEach() + // `buildIvySettings` and `loadIvySettings` set this global property as a side effect; + // save and clear it so each test is isolated and the value does not leak to other suites. + originalUserAgent = Option(System.getProperty(MavenUtils.USER_AGENT_PROPERTY)) + System.clearProperty(MavenUtils.USER_AGENT_PROPERTY) tempIvyPath = SparkFileUtils.createTempDir(namePrefix = "ivy").getAbsolutePath() } override def afterEach(): Unit = { + originalUserAgent match { + case Some(userAgent) => System.setProperty(MavenUtils.USER_AGENT_PROPERTY, userAgent) + case None => System.clearProperty(MavenUtils.USER_AGENT_PROPERTY) + } SparkFileUtils.deleteRecursively(new File(tempIvyPath)) super.afterEach() } @@ -309,4 +321,41 @@ class MavenUtilsSuite s" Resolved jars are: $jarPath") } } + + test("SPARK-59229: user agent identifies Spark and the underlying Ivy version") { + val userAgent = MavenUtils.sparkUserAgent + // `product RWS comment` per RFC 9110 Section 10.1.5. + assert(userAgent.matches("""^Apache-Spark/\S+ \(Apache-Ivy/\S+\)$"""), + s"User agent [$userAgent] does not match the RFC 9110 product/comment form") + assert(userAgent === s"Apache-Spark/${SparkBuildInfo.spark_version} " + + s"(Apache-Ivy/${Ivy.getIvyVersion})") + } + + test("SPARK-59229: buildIvySettings sets the user agent when it is unset") { + assert(System.getProperty(MavenUtils.USER_AGENT_PROPERTY) === null) + MavenUtils.buildIvySettings(None, Some(tempIvyPath)) + assert(System.getProperty(MavenUtils.USER_AGENT_PROPERTY) === MavenUtils.sparkUserAgent) + } + + test("SPARK-59229: loadIvySettings sets the user agent when it is unset") { + val settingsText = + s""" + | + | + | + |""".stripMargin + val settingsFile = Paths.get(tempIvyPath, "ivysettings.xml") + Files.write(settingsFile, settingsText.getBytes(StandardCharsets.UTF_8)) + + assert(System.getProperty(MavenUtils.USER_AGENT_PROPERTY) === null) + MavenUtils.loadIvySettings(settingsFile.toString, None, Some(tempIvyPath)) + assert(System.getProperty(MavenUtils.USER_AGENT_PROPERTY) === MavenUtils.sparkUserAgent) + } + + test("SPARK-59229: an explicitly configured user agent is not overwritten") { + val configuredUserAgent = "AcmeCorp/1.0" + System.setProperty(MavenUtils.USER_AGENT_PROPERTY, configuredUserAgent) + MavenUtils.buildIvySettings(None, Some(tempIvyPath)) + assert(System.getProperty(MavenUtils.USER_AGENT_PROPERTY) === configuredUserAgent) + } } diff --git a/docs/configuration.md b/docs/configuration.md index bf03269c1cec8..3be56203058e6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -963,6 +963,12 @@ Apart from these, the following properties are also available, and may be useful will be searched for in the local maven repo, then maven central and finally any additional remote repositories given by the command-line option --repositories. For more details, see Advanced Dependency Management. +

+ When fetching these artifacts, Spark identifies itself to remote repositories with a + User-Agent of the form + Apache-Spark/<spark-version> (Apache-Ivy/<ivy-version>), by setting the + http.agent system property if it is not already set. Set + -Dhttp.agent=... to override it. 1.5.0