diff --git a/integration_tests/docker-compose-spark.yml b/integration_tests/docker-compose-spark.yml index 1d12cbf10..e14e3425e 100644 --- a/integration_tests/docker-compose-spark.yml +++ b/integration_tests/docker-compose-spark.yml @@ -11,24 +11,33 @@ services: - "4040:4040" depends_on: spark-hive-metastore: - condition: service_started + condition: service_healthy rustfs-setup: condition: service_completed_successfully command: > --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 --name Thrift JDBC/ODBC Server healthcheck: - test: ["CMD-SHELL", "nc -z localhost 10000"] + test: + [ + "CMD", + "beeline", + "-u", + "jdbc:hive2://localhost:10000", + "-n", + "dbt", + "--silent=true", + "-e", + "select 1", + ] interval: 10s timeout: 5s retries: 20 start_period: 30s volumes: - - spark-warehouse:/spark-warehouse/ - - ./docker/spark/hive-site.xml:/usr/spark/conf/hive-site.xml - - ./docker/spark/spark-defaults.conf:/usr/spark/conf/spark-defaults.conf - environment: - - WAIT_FOR=spark-hive-metastore:5432 + - spark-warehouse:/opt/spark/work-dir/spark-warehouse + - ./docker/spark/hive-site.xml:/opt/spark/conf/hive-site.xml + - ./docker/spark/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf spark-hive-metastore: image: postgres:15-alpine @@ -38,6 +47,11 @@ services: - POSTGRES_USER=dbt - POSTGRES_PASSWORD=dbt - POSTGRES_DB=metastore + healthcheck: + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U dbt -d metastore"] + interval: 5s + timeout: 5s + retries: 10 # S3-compatible storage for external seed tables. rustfs: diff --git a/integration_tests/docker/spark/Dockerfile b/integration_tests/docker/spark/Dockerfile index 492d7a657..22d4050f1 100644 --- a/integration_tests/docker/spark/Dockerfile +++ b/integration_tests/docker/spark/Dockerfile @@ -1,36 +1,28 @@ -ARG OPENJDK_VERSION=8 -FROM eclipse-temurin:${OPENJDK_VERSION}-jre +ARG SPARK_IMAGE=apache/spark:3.3.3 +FROM ${SPARK_IMAGE} -ARG SPARK_VERSION=3.3.2 -ARG HADOOP_VERSION=3 +# delta-core_ before Delta 3.0, delta-spark_ from 3.0. +ARG DELTA_ARTIFACT=delta-core_2.12 ARG DELTA_VERSION=2.2.0 +ARG POSTGRES_JDBC_VERSION=42.7.13 +# Must match the Hadoop version bundled with the Spark image (3.3.2 here). ARG HADOOP_FULL_VERSION=3.3.2 ARG AWS_SDK_VERSION=1.11.1026 -ENV SPARK_HOME /usr/spark -ENV PATH="/usr/spark/bin:/usr/spark/sbin:${PATH}" +ENV PATH="${SPARK_HOME}/bin:${SPARK_HOME}/sbin:${PATH}" -RUN apt-get update && \ - apt-get install -y --no-install-recommends wget netcat-openbsd procps libpostgresql-jdbc-java && \ - wget -q "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ - tar xzf "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ - rm "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ - mv "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" /usr/spark && \ - ln -s /usr/share/java/postgresql-jdbc4.jar /usr/spark/jars/postgresql-jdbc4.jar && \ - wget -q "https://repo1.maven.org/maven2/io/delta/delta-core_2.12/${DELTA_VERSION}/delta-core_2.12-${DELTA_VERSION}.jar" \ - -P /usr/spark/jars/ && \ - wget -q "https://repo1.maven.org/maven2/io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar" \ - -P /usr/spark/jars/ && \ - wget -q "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_FULL_VERSION}/hadoop-aws-${HADOOP_FULL_VERSION}.jar" \ - -P /usr/spark/jars/ && \ - wget -q "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_VERSION}/aws-java-sdk-bundle-${AWS_SDK_VERSION}.jar" \ - -P /usr/spark/jars/ && \ - apt-get remove -y wget && \ - apt-get autoremove -y && \ - apt-get clean +# ADD gives files downloaded from a URL mode 600 (root only), hence a+r. +ADD --chmod=a+r \ + "https://repo1.maven.org/maven2/io/delta/${DELTA_ARTIFACT}/${DELTA_VERSION}/${DELTA_ARTIFACT}-${DELTA_VERSION}.jar" \ + "https://repo1.maven.org/maven2/io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar" \ + "https://repo1.maven.org/maven2/org/postgresql/postgresql/${POSTGRES_JDBC_VERSION}/postgresql-${POSTGRES_JDBC_VERSION}.jar" \ + "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_FULL_VERSION}/hadoop-aws-${HADOOP_FULL_VERSION}.jar" \ + "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_VERSION}/aws-java-sdk-bundle-${AWS_SDK_VERSION}.jar" \ + ${SPARK_HOME}/jars/ -COPY entrypoint.sh /scripts/ -RUN chmod +x /scripts/entrypoint.sh +# Spark's default warehouse dir, under the spark-owned work-dir. Creating it +# here lets the compose volume mounted on it inherit that ownership. +RUN mkdir spark-warehouse -ENTRYPOINT ["/scripts/entrypoint.sh"] +ENTRYPOINT ["spark-submit"] CMD ["--help"] diff --git a/integration_tests/docker/spark/entrypoint.sh b/integration_tests/docker/spark/entrypoint.sh deleted file mode 100644 index 36b8174c9..000000000 --- a/integration_tests/docker/spark/entrypoint.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -if [ -n "$WAIT_FOR" ]; then - WAIT_FOR_TIMEOUT_SECONDS="${WAIT_FOR_TIMEOUT_SECONDS:-180}" - IFS=';' read -a HOSTPORT_ARRAY <<< "$WAIT_FOR" - for HOSTPORT in "${HOSTPORT_ARRAY[@]}" - do - WAIT_FOR_HOST=${HOSTPORT%:*} - WAIT_FOR_PORT=${HOSTPORT#*:} - START_TS=$(date +%s) - - echo Waiting for $WAIT_FOR_HOST to listen on $WAIT_FOR_PORT... - while ! nc -z "$WAIT_FOR_HOST" "$WAIT_FOR_PORT"; do - if [ $(( $(date +%s) - START_TS )) -ge "$WAIT_FOR_TIMEOUT_SECONDS" ]; then - echo "Timed out waiting for $WAIT_FOR_HOST:$WAIT_FOR_PORT after ${WAIT_FOR_TIMEOUT_SECONDS}s" - exit 1 - fi - sleep 2 - done - done -fi - -exec spark-submit "$@"