[SPARK-59229][CORE] Identify Apache Spark in the User-Agent for Ivy dependency resolution

common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala

@@ -34,14 +34,39 @@ import org.apache.ivy.plugins.matcher.GlobPatternMatcher
 import org.apache.ivy.plugins.repository.file.FileRepository
 import org.apache.ivy.plugins.resolver.{ChainResolver, FileSystemResolver, IBiblioResolver}
 
-import org.apache.spark.SparkException
+import org.apache.spark.{SparkBuildInfo, SparkException}
 import org.apache.spark.internal.{Logging, LogKeys}
 import org.apache.spark.util.ArrayImplicits._
 
 /** Provides utility functions to be used inside SparkSubmit. */
 private[spark] object MavenUtils extends Logging {
   val JAR_IVY_SETTING_PATH_KEY: String = "spark.jars.ivySettings"
 
+  /** System property Ivy reads to determine the HTTP User-Agent. */
+  private[util] val USER_AGENT_PROPERTY = "http.agent"
+
+  /**
+   * User-Agent identifying Spark to remote artifact repositories, in the form
+   * `product RWS comment` per RFC 9110 Section 10.1.5, e.g.
+   * `Apache-Spark/4.0.0 (Apache-Ivy/2.5.3)`. Repository operators use this to attribute traffic
+   * and to identify the tool and version behind it.
+   */
+  private[util] lazy val sparkUserAgent: String =
+    s"Apache-Spark/${SparkBuildInfo.spark_version} (Apache-Ivy/${Ivy.getIvyVersion})"
+
+  /**
+   * Set the `http.agent` system property so Ivy identifies itself as Spark when fetching from
+   * remote repositories. Set once and left in place: Ivy's HttpClient-backed handler captures the
+   * User-Agent when its singleton is class-initialized, so restoring the previous value afterwards
+   * would have no effect on requests and would race with concurrent resolution. An explicitly
+   * configured value always wins, so an operator can still override with `-Dhttp.agent=...`.
+   */
+  private def setUserAgentIfAbsent(): Unit = {
+    if (System.getProperty(USER_AGENT_PROPERTY) == null) {
+      System.setProperty(USER_AGENT_PROPERTY, sparkUserAgent)
+    }
+  }
+
   // Exposed for testing
   // var printStream = SparkSubmit.printStream
 
@@ -277,6 +302,7 @@ private[spark] object MavenUtils extends Logging {
       remoteRepos: Option[String],
       ivyPath: Option[String],
       useLocalM2AsCache: Boolean = true)(implicit printStream: PrintStream): IvySettings = {
+    setUserAgentIfAbsent()
     val ivySettings: IvySettings = new IvySettings
     processIvyPathArg(ivySettings, ivyPath)
 
@@ -310,6 +336,7 @@ private[spark] object MavenUtils extends Logging {
    */
   def loadIvySettings(settingsFile: String, remoteRepos: Option[String], ivyPath: Option[String])(
       implicit printStream: PrintStream): IvySettings = {
+    setUserAgentIfAbsent()
     val uri = new URI(settingsFile)
     val file = Option(uri.getScheme).getOrElse("file") match {
       case "file" => new File(uri.getPath)

common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala

@@ -25,12 +25,14 @@ import java.nio.file.{Files, Paths}
 import scala.collection.mutable.ArrayBuffer
 import scala.jdk.CollectionConverters._
 
+import org.apache.ivy.Ivy
 import org.apache.ivy.core.module.descriptor.MDArtifact
 import org.apache.ivy.core.settings.IvySettings
 import org.apache.ivy.plugins.resolver.{AbstractResolver, ChainResolver, FileSystemResolver, IBiblioResolver}
 import org.scalatest.BeforeAndAfterEach
 import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
 
+import org.apache.spark.SparkBuildInfo
 import org.apache.spark.util.MavenUtils.MavenCoordinate
 
 class MavenUtilsSuite
@@ -39,6 +41,8 @@ class MavenUtilsSuite
 
   private var tempIvyPath: String = _
 
+  private var originalUserAgent: Option[String] = None
+
   private val noOpOutputStream = new OutputStream {
     def write(b: Int) = {}
   }
@@ -57,10 +61,18 @@ class MavenUtilsSuite
 
   override def beforeEach(): Unit = {
     super.beforeEach()
+    // `buildIvySettings` and `loadIvySettings` set this global property as a side effect;
+    // save and clear it so each test is isolated and the value does not leak to other suites.
+    originalUserAgent = Option(System.getProperty(MavenUtils.USER_AGENT_PROPERTY))
+    System.clearProperty(MavenUtils.USER_AGENT_PROPERTY)
     tempIvyPath = SparkFileUtils.createTempDir(namePrefix = "ivy").getAbsolutePath()
   }
 
   override def afterEach(): Unit = {
+    originalUserAgent match {
+      case Some(userAgent) => System.setProperty(MavenUtils.USER_AGENT_PROPERTY, userAgent)
+      case None => System.clearProperty(MavenUtils.USER_AGENT_PROPERTY)
+    }
     SparkFileUtils.deleteRecursively(new File(tempIvyPath))
     super.afterEach()
   }
@@ -309,4 +321,41 @@ class MavenUtilsSuite
         s" Resolved jars are: $jarPath")
     }
   }
+
+  test("SPARK-59229: user agent identifies Spark and the underlying Ivy version") {
+    val userAgent = MavenUtils.sparkUserAgent
+    // `product RWS comment` per RFC 9110 Section 10.1.5.
+    assert(userAgent.matches("""^Apache-Spark/\S+ \(Apache-Ivy/\S+\)$"""),
+      s"User agent [$userAgent] does not match the RFC 9110 product/comment form")
+    assert(userAgent === s"Apache-Spark/${SparkBuildInfo.spark_version} " +
+      s"(Apache-Ivy/${Ivy.getIvyVersion})")
+  }
+
+  test("SPARK-59229: buildIvySettings sets the user agent when it is unset") {
+    assert(System.getProperty(MavenUtils.USER_AGENT_PROPERTY) === null)
+    MavenUtils.buildIvySettings(None, Some(tempIvyPath))
+    assert(System.getProperty(MavenUtils.USER_AGENT_PROPERTY) === MavenUtils.sparkUserAgent)
+  }
+
+  test("SPARK-59229: loadIvySettings sets the user agent when it is unset") {
+    val settingsText =
+      s"""
+         |<ivysettings>
+         |  <caches defaultCacheDir="$tempIvyPath/cache"/>
+         |</ivysettings>
+         |""".stripMargin
+    val settingsFile = Paths.get(tempIvyPath, "ivysettings.xml")
+    Files.write(settingsFile, settingsText.getBytes(StandardCharsets.UTF_8))
+
+    assert(System.getProperty(MavenUtils.USER_AGENT_PROPERTY) === null)
+    MavenUtils.loadIvySettings(settingsFile.toString, None, Some(tempIvyPath))
+    assert(System.getProperty(MavenUtils.USER_AGENT_PROPERTY) === MavenUtils.sparkUserAgent)
+  }
+
+  test("SPARK-59229: an explicitly configured user agent is not overwritten") {
+    val configuredUserAgent = "AcmeCorp/1.0"
+    System.setProperty(MavenUtils.USER_AGENT_PROPERTY, configuredUserAgent)
+    MavenUtils.buildIvySettings(None, Some(tempIvyPath))
+    assert(System.getProperty(MavenUtils.USER_AGENT_PROPERTY) === configuredUserAgent)
+  }
 }

docs/configuration.md

@@ -963,6 +963,12 @@ Apart from these, the following properties are also available, and may be useful
     will be searched for in the local maven repo, then maven central and finally any additional remote
     repositories given by the command-line option <code>--repositories</code>. For more details, see
     <a href="submitting-applications.html#advanced-dependency-management">Advanced Dependency Management</a>.
+    <p/>
+    When fetching these artifacts, Spark identifies itself to remote repositories with a
+    <code>User-Agent</code> of the form
+    <code>Apache-Spark/&lt;spark-version&gt; (Apache-Ivy/&lt;ivy-version&gt;)</code>, by setting the
+    <code>http.agent</code> system property if it is not already set. Set
+    <code>-Dhttp.agent=...</code> to override it.
   </td>
   <td>1.5.0</td>
 </tr>