From 6ae03fde72226cd1cfc1f887f7923e382501e59b Mon Sep 17 00:00:00 2001 From: Elad Barkay Date: Wed, 7 Oct 2026 17:13:44 +0300 Subject: [PATCH 1/2] ci: build the Spark test image from the official apache/spark image (CORE-1562) The Spark Dockerfile downloaded the ~300MB Spark 3.3.2 tarball from archive.apache.org on every run. That mirror is heavily throttled, so "Start Spark" took 43-73 minutes and the job regularly hit its 90-minute timeout mid-test. Build on apache/spark:3.3.3 and ADD only the jars Spark needs: Delta, the Postgres JDBC driver, and the existing hadoop-aws / AWS SDK pair for reading seeds from RustFS, at unchanged versions (the image ships the matching Hadoop 3.3.2). apache/spark:v3.3.2 is not usable: its JDK 11.0.16 crashes on the GitHub runners (JDK-8287073). Run as the image's spark user: the image creates Spark's default warehouse dir under its work-dir, so the volume mounted there inherits that ownership. Health-check the thrift server with beeline instead of nc (not in the official image), give the metastore a pg_isready healthcheck and start Spark on service_healthy, which makes the entrypoint's wait loop redundant, so spark-submit is the ENTRYPOINT. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019akn5nGXcW5MA7BcVvxyo6 --- integration_tests/docker-compose-spark.yml | 28 +++++++++--- integration_tests/docker/spark/Dockerfile | 46 ++++++++------------ integration_tests/docker/spark/entrypoint.sh | 23 ---------- 3 files changed, 40 insertions(+), 57 deletions(-) delete mode 100644 integration_tests/docker/spark/entrypoint.sh diff --git a/integration_tests/docker-compose-spark.yml b/integration_tests/docker-compose-spark.yml index 1d12cbf10..dbea03830 100644 --- a/integration_tests/docker-compose-spark.yml +++ b/integration_tests/docker-compose-spark.yml @@ -11,24 +11,33 @@ services: - "4040:4040" depends_on: spark-hive-metastore: - condition: service_started + condition: service_healthy rustfs-setup: condition: service_completed_successfully command: > --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 --name Thrift JDBC/ODBC Server healthcheck: - test: ["CMD-SHELL", "nc -z localhost 10000"] + test: + [ + "CMD", + "beeline", + "-u", + "jdbc:hive2://localhost:10000", + "-n", + "dbt", + "--silent=true", + "-e", + "select 1", + ] interval: 10s timeout: 5s retries: 20 start_period: 30s volumes: - - spark-warehouse:/spark-warehouse/ - - ./docker/spark/hive-site.xml:/usr/spark/conf/hive-site.xml - - ./docker/spark/spark-defaults.conf:/usr/spark/conf/spark-defaults.conf - environment: - - WAIT_FOR=spark-hive-metastore:5432 + - spark-warehouse:/opt/spark/work-dir/spark-warehouse + - ./docker/spark/hive-site.xml:/opt/spark/conf/hive-site.xml + - ./docker/spark/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf spark-hive-metastore: image: postgres:15-alpine @@ -38,6 +47,11 @@ services: - POSTGRES_USER=dbt - POSTGRES_PASSWORD=dbt - POSTGRES_DB=metastore + healthcheck: + test: ["CMD-SHELL", "pg_isready -U dbt"] + interval: 5s + timeout: 5s + retries: 10 # S3-compatible storage for external seed tables. rustfs: diff --git a/integration_tests/docker/spark/Dockerfile b/integration_tests/docker/spark/Dockerfile index 492d7a657..22d4050f1 100644 --- a/integration_tests/docker/spark/Dockerfile +++ b/integration_tests/docker/spark/Dockerfile @@ -1,36 +1,28 @@ -ARG OPENJDK_VERSION=8 -FROM eclipse-temurin:${OPENJDK_VERSION}-jre +ARG SPARK_IMAGE=apache/spark:3.3.3 +FROM ${SPARK_IMAGE} -ARG SPARK_VERSION=3.3.2 -ARG HADOOP_VERSION=3 +# delta-core_ before Delta 3.0, delta-spark_ from 3.0. +ARG DELTA_ARTIFACT=delta-core_2.12 ARG DELTA_VERSION=2.2.0 +ARG POSTGRES_JDBC_VERSION=42.7.13 +# Must match the Hadoop version bundled with the Spark image (3.3.2 here). ARG HADOOP_FULL_VERSION=3.3.2 ARG AWS_SDK_VERSION=1.11.1026 -ENV SPARK_HOME /usr/spark -ENV PATH="/usr/spark/bin:/usr/spark/sbin:${PATH}" +ENV PATH="${SPARK_HOME}/bin:${SPARK_HOME}/sbin:${PATH}" -RUN apt-get update && \ - apt-get install -y --no-install-recommends wget netcat-openbsd procps libpostgresql-jdbc-java && \ - wget -q "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ - tar xzf "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ - rm "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ - mv "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" /usr/spark && \ - ln -s /usr/share/java/postgresql-jdbc4.jar /usr/spark/jars/postgresql-jdbc4.jar && \ - wget -q "https://repo1.maven.org/maven2/io/delta/delta-core_2.12/${DELTA_VERSION}/delta-core_2.12-${DELTA_VERSION}.jar" \ - -P /usr/spark/jars/ && \ - wget -q "https://repo1.maven.org/maven2/io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar" \ - -P /usr/spark/jars/ && \ - wget -q "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_FULL_VERSION}/hadoop-aws-${HADOOP_FULL_VERSION}.jar" \ - -P /usr/spark/jars/ && \ - wget -q "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_VERSION}/aws-java-sdk-bundle-${AWS_SDK_VERSION}.jar" \ - -P /usr/spark/jars/ && \ - apt-get remove -y wget && \ - apt-get autoremove -y && \ - apt-get clean +# ADD gives files downloaded from a URL mode 600 (root only), hence a+r. +ADD --chmod=a+r \ + "https://repo1.maven.org/maven2/io/delta/${DELTA_ARTIFACT}/${DELTA_VERSION}/${DELTA_ARTIFACT}-${DELTA_VERSION}.jar" \ + "https://repo1.maven.org/maven2/io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar" \ + "https://repo1.maven.org/maven2/org/postgresql/postgresql/${POSTGRES_JDBC_VERSION}/postgresql-${POSTGRES_JDBC_VERSION}.jar" \ + "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_FULL_VERSION}/hadoop-aws-${HADOOP_FULL_VERSION}.jar" \ + "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_VERSION}/aws-java-sdk-bundle-${AWS_SDK_VERSION}.jar" \ + ${SPARK_HOME}/jars/ -COPY entrypoint.sh /scripts/ -RUN chmod +x /scripts/entrypoint.sh +# Spark's default warehouse dir, under the spark-owned work-dir. Creating it +# here lets the compose volume mounted on it inherit that ownership. +RUN mkdir spark-warehouse -ENTRYPOINT ["/scripts/entrypoint.sh"] +ENTRYPOINT ["spark-submit"] CMD ["--help"] diff --git a/integration_tests/docker/spark/entrypoint.sh b/integration_tests/docker/spark/entrypoint.sh deleted file mode 100644 index 36b8174c9..000000000 --- a/integration_tests/docker/spark/entrypoint.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -if [ -n "$WAIT_FOR" ]; then - WAIT_FOR_TIMEOUT_SECONDS="${WAIT_FOR_TIMEOUT_SECONDS:-180}" - IFS=';' read -a HOSTPORT_ARRAY <<< "$WAIT_FOR" - for HOSTPORT in "${HOSTPORT_ARRAY[@]}" - do - WAIT_FOR_HOST=${HOSTPORT%:*} - WAIT_FOR_PORT=${HOSTPORT#*:} - START_TS=$(date +%s) - - echo Waiting for $WAIT_FOR_HOST to listen on $WAIT_FOR_PORT... - while ! nc -z "$WAIT_FOR_HOST" "$WAIT_FOR_PORT"; do - if [ $(( $(date +%s) - START_TS )) -ge "$WAIT_FOR_TIMEOUT_SECONDS" ]; then - echo "Timed out waiting for $WAIT_FOR_HOST:$WAIT_FOR_PORT after ${WAIT_FOR_TIMEOUT_SECONDS}s" - exit 1 - fi - sleep 2 - done - done -fi - -exec spark-submit "$@" From d2a59bf7a88aba6f2b6f34323ae71e17e5e25ee9 Mon Sep 17 00:00:00 2001 From: Elad Barkay Date: Wed, 7 Oct 2026 17:31:33 +0300 Subject: [PATCH 2/2] ci: check the Spark metastore's readiness over TCP On a fresh volume, the postgres entrypoint first runs a temporary, socket-only server while it creates the database, then restarts. pg_isready over the socket can pass during that window, so Spark could start before the real server is up. Check 127.0.0.1 and the metastore database instead. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019akn5nGXcW5MA7BcVvxyo6 --- integration_tests/docker-compose-spark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/docker-compose-spark.yml b/integration_tests/docker-compose-spark.yml index dbea03830..e14e3425e 100644 --- a/integration_tests/docker-compose-spark.yml +++ b/integration_tests/docker-compose-spark.yml @@ -48,7 +48,7 @@ services: - POSTGRES_PASSWORD=dbt - POSTGRES_DB=metastore healthcheck: - test: ["CMD-SHELL", "pg_isready -U dbt"] + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U dbt -d metastore"] interval: 5s timeout: 5s retries: 10