CORE-1562: ci: build the Spark test image from the official apache/spark image

integration_tests/docker-compose-spark.yml

@@ -11,24 +11,33 @@ services:
       - "4040:4040"
     depends_on:
       spark-hive-metastore:
-        condition: service_started
+        condition: service_healthy
       rustfs-setup:
         condition: service_completed_successfully
     command: >
       --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
       --name Thrift JDBC/ODBC Server
     healthcheck:
-      test: ["CMD-SHELL", "nc -z localhost 10000"]
+      test:
+        [
+          "CMD",
+          "beeline",
+          "-u",
+          "jdbc:hive2://localhost:10000",
+          "-n",
+          "dbt",
+          "--silent=true",
+          "-e",
+          "select 1",
+        ]
       interval: 10s
       timeout: 5s
       retries: 20
       start_period: 30s
     volumes:
-      - spark-warehouse:/spark-warehouse/
-      - ./docker/spark/hive-site.xml:/usr/spark/conf/hive-site.xml
-      - ./docker/spark/spark-defaults.conf:/usr/spark/conf/spark-defaults.conf
-    environment:
-      - WAIT_FOR=spark-hive-metastore:5432
+      - spark-warehouse:/opt/spark/work-dir/spark-warehouse
+      - ./docker/spark/hive-site.xml:/opt/spark/conf/hive-site.xml
+      - ./docker/spark/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf
 
   spark-hive-metastore:
     image: postgres:15-alpine
@@ -38,6 +47,11 @@ services:
       - POSTGRES_USER=dbt
       - POSTGRES_PASSWORD=dbt
       - POSTGRES_DB=metastore
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U dbt -d metastore"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
 
   # S3-compatible storage for external seed tables.
   rustfs:

integration_tests/docker/spark/Dockerfile

@@ -1,36 +1,28 @@
-ARG OPENJDK_VERSION=8
-FROM eclipse-temurin:${OPENJDK_VERSION}-jre
+ARG SPARK_IMAGE=apache/spark:3.3.3
+FROM ${SPARK_IMAGE}
 
-ARG SPARK_VERSION=3.3.2
-ARG HADOOP_VERSION=3
+# delta-core_<scala> before Delta 3.0, delta-spark_<scala> from 3.0.
+ARG DELTA_ARTIFACT=delta-core_2.12
 ARG DELTA_VERSION=2.2.0
+ARG POSTGRES_JDBC_VERSION=42.7.13
+# Must match the Hadoop version bundled with the Spark image (3.3.2 here).
 ARG HADOOP_FULL_VERSION=3.3.2
 ARG AWS_SDK_VERSION=1.11.1026
 
-ENV SPARK_HOME /usr/spark
-ENV PATH="/usr/spark/bin:/usr/spark/sbin:${PATH}"
+ENV PATH="${SPARK_HOME}/bin:${SPARK_HOME}/sbin:${PATH}"
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends wget netcat-openbsd procps libpostgresql-jdbc-java && \
-    wget -q "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
-    tar xzf "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
-    rm "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
-    mv "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" /usr/spark && \
-    ln -s /usr/share/java/postgresql-jdbc4.jar /usr/spark/jars/postgresql-jdbc4.jar && \
-    wget -q "https://repo1.maven.org/maven2/io/delta/delta-core_2.12/${DELTA_VERSION}/delta-core_2.12-${DELTA_VERSION}.jar" \
-         -P /usr/spark/jars/ && \
-    wget -q "https://repo1.maven.org/maven2/io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar" \
-         -P /usr/spark/jars/ && \
-    wget -q "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_FULL_VERSION}/hadoop-aws-${HADOOP_FULL_VERSION}.jar" \
-         -P /usr/spark/jars/ && \
-    wget -q "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_VERSION}/aws-java-sdk-bundle-${AWS_SDK_VERSION}.jar" \
-         -P /usr/spark/jars/ && \
-    apt-get remove -y wget && \
-    apt-get autoremove -y && \
-    apt-get clean
+# ADD gives files downloaded from a URL mode 600 (root only), hence a+r.
+ADD --chmod=a+r \
+    "https://repo1.maven.org/maven2/io/delta/${DELTA_ARTIFACT}/${DELTA_VERSION}/${DELTA_ARTIFACT}-${DELTA_VERSION}.jar" \
+    "https://repo1.maven.org/maven2/io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar" \
+    "https://repo1.maven.org/maven2/org/postgresql/postgresql/${POSTGRES_JDBC_VERSION}/postgresql-${POSTGRES_JDBC_VERSION}.jar" \
+    "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_FULL_VERSION}/hadoop-aws-${HADOOP_FULL_VERSION}.jar" \
+    "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_VERSION}/aws-java-sdk-bundle-${AWS_SDK_VERSION}.jar" \
+    ${SPARK_HOME}/jars/
 
-COPY entrypoint.sh /scripts/
-RUN chmod +x /scripts/entrypoint.sh
+# Spark's default warehouse dir, under the spark-owned work-dir. Creating it
+# here lets the compose volume mounted on it inherit that ownership.
+RUN mkdir spark-warehouse
 
-ENTRYPOINT ["/scripts/entrypoint.sh"]
+ENTRYPOINT ["spark-submit"]
 CMD ["--help"]