diff --git a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala
index 5e923ad35a3ab..e420df3102ad7 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala
@@ -34,7 +34,7 @@ import org.apache.ivy.plugins.matcher.GlobPatternMatcher
import org.apache.ivy.plugins.repository.file.FileRepository
import org.apache.ivy.plugins.resolver.{ChainResolver, FileSystemResolver, IBiblioResolver}
-import org.apache.spark.SparkException
+import org.apache.spark.{SparkBuildInfo, SparkException}
import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.util.ArrayImplicits._
@@ -42,6 +42,31 @@ import org.apache.spark.util.ArrayImplicits._
private[spark] object MavenUtils extends Logging {
val JAR_IVY_SETTING_PATH_KEY: String = "spark.jars.ivySettings"
+ /** System property Ivy reads to determine the HTTP User-Agent. */
+ private[util] val USER_AGENT_PROPERTY = "http.agent"
+
+ /**
+ * User-Agent identifying Spark to remote artifact repositories, in the form
+ * `product RWS comment` per RFC 9110 Section 10.1.5, e.g.
+ * `Apache-Spark/4.0.0 (Apache-Ivy/2.5.3)`. Repository operators use this to attribute traffic
+ * and to identify the tool and version behind it.
+ */
+ private[util] lazy val sparkUserAgent: String =
+ s"Apache-Spark/${SparkBuildInfo.spark_version} (Apache-Ivy/${Ivy.getIvyVersion})"
+
+ /**
+ * Set the `http.agent` system property so Ivy identifies itself as Spark when fetching from
+ * remote repositories. Set once and left in place: Ivy's HttpClient-backed handler captures the
+ * User-Agent when its singleton is class-initialized, so restoring the previous value afterwards
+ * would have no effect on requests and would race with concurrent resolution. An explicitly
+ * configured value always wins, so an operator can still override with `-Dhttp.agent=...`.
+ */
+ private def setUserAgentIfAbsent(): Unit = {
+ if (System.getProperty(USER_AGENT_PROPERTY) == null) {
+ System.setProperty(USER_AGENT_PROPERTY, sparkUserAgent)
+ }
+ }
+
// Exposed for testing
// var printStream = SparkSubmit.printStream
@@ -277,6 +302,7 @@ private[spark] object MavenUtils extends Logging {
remoteRepos: Option[String],
ivyPath: Option[String],
useLocalM2AsCache: Boolean = true)(implicit printStream: PrintStream): IvySettings = {
+ setUserAgentIfAbsent()
val ivySettings: IvySettings = new IvySettings
processIvyPathArg(ivySettings, ivyPath)
@@ -310,6 +336,7 @@ private[spark] object MavenUtils extends Logging {
*/
def loadIvySettings(settingsFile: String, remoteRepos: Option[String], ivyPath: Option[String])(
implicit printStream: PrintStream): IvySettings = {
+ setUserAgentIfAbsent()
val uri = new URI(settingsFile)
val file = Option(uri.getScheme).getOrElse("file") match {
case "file" => new File(uri.getPath)
diff --git a/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala b/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala
index d30422ca8dd54..294cbb9e28ee4 100644
--- a/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala
+++ b/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala
@@ -25,12 +25,14 @@ import java.nio.file.{Files, Paths}
import scala.collection.mutable.ArrayBuffer
import scala.jdk.CollectionConverters._
+import org.apache.ivy.Ivy
import org.apache.ivy.core.module.descriptor.MDArtifact
import org.apache.ivy.core.settings.IvySettings
import org.apache.ivy.plugins.resolver.{AbstractResolver, ChainResolver, FileSystemResolver, IBiblioResolver}
import org.scalatest.BeforeAndAfterEach
import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
+import org.apache.spark.SparkBuildInfo
import org.apache.spark.util.MavenUtils.MavenCoordinate
class MavenUtilsSuite
@@ -39,6 +41,8 @@ class MavenUtilsSuite
private var tempIvyPath: String = _
+ private var originalUserAgent: Option[String] = None
+
private val noOpOutputStream = new OutputStream {
def write(b: Int) = {}
}
@@ -57,10 +61,18 @@ class MavenUtilsSuite
override def beforeEach(): Unit = {
super.beforeEach()
+ // `buildIvySettings` and `loadIvySettings` set this global property as a side effect;
+ // save and clear it so each test is isolated and the value does not leak to other suites.
+ originalUserAgent = Option(System.getProperty(MavenUtils.USER_AGENT_PROPERTY))
+ System.clearProperty(MavenUtils.USER_AGENT_PROPERTY)
tempIvyPath = SparkFileUtils.createTempDir(namePrefix = "ivy").getAbsolutePath()
}
override def afterEach(): Unit = {
+ originalUserAgent match {
+ case Some(userAgent) => System.setProperty(MavenUtils.USER_AGENT_PROPERTY, userAgent)
+ case None => System.clearProperty(MavenUtils.USER_AGENT_PROPERTY)
+ }
SparkFileUtils.deleteRecursively(new File(tempIvyPath))
super.afterEach()
}
@@ -309,4 +321,41 @@ class MavenUtilsSuite
s" Resolved jars are: $jarPath")
}
}
+
+ test("SPARK-59229: user agent identifies Spark and the underlying Ivy version") {
+ val userAgent = MavenUtils.sparkUserAgent
+ // `product RWS comment` per RFC 9110 Section 10.1.5.
+ assert(userAgent.matches("""^Apache-Spark/\S+ \(Apache-Ivy/\S+\)$"""),
+ s"User agent [$userAgent] does not match the RFC 9110 product/comment form")
+ assert(userAgent === s"Apache-Spark/${SparkBuildInfo.spark_version} " +
+ s"(Apache-Ivy/${Ivy.getIvyVersion})")
+ }
+
+ test("SPARK-59229: buildIvySettings sets the user agent when it is unset") {
+ assert(System.getProperty(MavenUtils.USER_AGENT_PROPERTY) === null)
+ MavenUtils.buildIvySettings(None, Some(tempIvyPath))
+ assert(System.getProperty(MavenUtils.USER_AGENT_PROPERTY) === MavenUtils.sparkUserAgent)
+ }
+
+ test("SPARK-59229: loadIvySettings sets the user agent when it is unset") {
+ val settingsText =
+ s"""
+ |--repositories. For more details, see
Advanced Dependency Management.
+
User-Agent of the form
+ Apache-Spark/<spark-version> (Apache-Ivy/<ivy-version>), by setting the
+ http.agent system property if it is not already set. Set
+ -Dhttp.agent=... to override it.