CORE-1557: ci: build the e2e Spark image from the official apache/spark image

.github/workflows/test-warehouse.yml

@@ -250,6 +250,11 @@ jobs:
         run: |
           docker compose up -d --wait vertica
 
+      - name: Show warehouse service logs
+        if: failure()
+        working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
+        run: docker compose logs --no-color
+
       - name: Setup Python
         uses: actions/setup-python@v6
         with:

tests/e2e_dbt_project/docker-compose.yml

@@ -251,6 +251,12 @@ services:
     build:
       context: ./docker/spark
       dockerfile: Dockerfile
+      # Not 3.5 until CORE-1558 lands (PyHive fails on its DDL results); not
+      # v3.3.2, whose JDK 11.0.16 crashes on GitHub's runners (JDK-8287073).
+      args:
+        SPARK_IMAGE: apache/spark:3.3.3
+        DELTA_ARTIFACT: delta-core_2.12
+        DELTA_VERSION: 2.2.0
     ports:
       - "10000:10000"
       - "4040:4040"
@@ -261,19 +267,28 @@ services:
       --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
       --name Thrift JDBC/ODBC Server
     healthcheck:
-      test: ["CMD-SHELL", "nc -z localhost 10000"]
+      test:
+        [
+          "CMD",
+          "beeline",
+          "-u",
+          "jdbc:hive2://localhost:10000",
+          "-n",
+          "dbt",
+          "--silent=true",
+          "-e",
+          "select 1",
+        ]
       interval: 10s
       timeout: 5s
       retries: 20
       start_period: 30s
     user: "0"
     volumes:
       - spark-warehouse:/spark-warehouse/
-      - ./docker/spark/hive-site.xml:/usr/spark/conf/hive-site.xml
-      - ./docker/spark/spark-defaults.conf:/usr/spark/conf/spark-defaults.conf
+      - ./docker/spark/hive-site.xml:/opt/spark/conf/hive-site.xml
+      - ./docker/spark/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf
       - ./data:/seed-data:ro
-    environment:
-      - WAIT_FOR=spark-hive-metastore:5432
 
   spark-hive-metastore:
     image: postgres:15-alpine
@@ -284,7 +299,11 @@ services:
       - POSTGRES_PASSWORD=${SPARK_METASTORE_PASSWORD:-dbt}
       - POSTGRES_DB=metastore
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U ${SPARK_METASTORE_USER:-dbt}"]
+      test:
+        [
+          "CMD-SHELL",
+          "pg_isready -h 127.0.0.1 -U ${SPARK_METASTORE_USER:-dbt} -d metastore",
+        ]
       interval: 5s
       timeout: 5s
       retries: 10

tests/e2e_dbt_project/docker/spark/Dockerfile

@@ -1,33 +1,22 @@
-ARG OPENJDK_VERSION=8
-FROM eclipse-temurin:${OPENJDK_VERSION}-jre
+ARG SPARK_IMAGE=apache/spark:3.3.3
+FROM ${SPARK_IMAGE}
 
-ARG SPARK_VERSION=3.3.2
-ARG HADOOP_VERSION=3
+# delta-core_<scala> before Delta 3.0, delta-spark_<scala> from 3.0.
+ARG DELTA_ARTIFACT=delta-core_2.12
 ARG DELTA_VERSION=2.2.0
+ARG POSTGRES_JDBC_VERSION=42.7.13
 
-ENV SPARK_HOME /usr/spark
-ENV PATH="/usr/spark/bin:/usr/spark/sbin:${PATH}"
+ENV PATH="${SPARK_HOME}/bin:${SPARK_HOME}/sbin:${PATH}"
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends wget netcat-openbsd procps libpostgresql-jdbc-java && \
-    wget -q "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
-    tar xzf "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
-    rm "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
-    mv "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" /usr/spark && \
-    ln -s /usr/share/java/postgresql-jdbc4.jar /usr/spark/jars/postgresql-jdbc4.jar && \
-    wget -q "https://repo1.maven.org/maven2/io/delta/delta-core_2.12/${DELTA_VERSION}/delta-core_2.12-${DELTA_VERSION}.jar" \
-         -P /usr/spark/jars/ && \
-    wget -q "https://repo1.maven.org/maven2/io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar" \
-         -P /usr/spark/jars/ && \
-    apt-get remove -y wget && \
-    apt-get autoremove -y && \
-    apt-get clean
+# ADD gives files downloaded from a URL mode 600 (root only), hence a+r.
+ADD --chmod=a+r \
+    "https://repo1.maven.org/maven2/io/delta/${DELTA_ARTIFACT}/${DELTA_VERSION}/${DELTA_ARTIFACT}-${DELTA_VERSION}.jar" \
+    "https://repo1.maven.org/maven2/io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar" \
+    "https://repo1.maven.org/maven2/org/postgresql/postgresql/${POSTGRES_JDBC_VERSION}/postgresql-${POSTGRES_JDBC_VERSION}.jar" \
+    ${SPARK_HOME}/jars/
 
-COPY entrypoint.sh /scripts/
-RUN chmod +x /scripts/entrypoint.sh && \
-    groupadd -r spark && useradd -r -g spark -m spark && \
-    chown -R spark:spark /usr/spark /scripts
+# Spark's warehouse dir is relative to the workdir; /spark-warehouse is the volume.
+WORKDIR /
 
-USER spark
-ENTRYPOINT ["/scripts/entrypoint.sh"]
+ENTRYPOINT ["spark-submit"]
 CMD ["--help"]