diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index 0775a24b0..95f086db7 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -250,6 +250,11 @@ jobs: run: | docker compose up -d --wait vertica + - name: Show warehouse service logs + if: failure() + working-directory: ${{ env.E2E_DBT_PROJECT_DIR }} + run: docker compose logs --no-color + - name: Setup Python uses: actions/setup-python@v6 with: diff --git a/tests/e2e_dbt_project/docker-compose.yml b/tests/e2e_dbt_project/docker-compose.yml index 44c616a35..c56d26790 100644 --- a/tests/e2e_dbt_project/docker-compose.yml +++ b/tests/e2e_dbt_project/docker-compose.yml @@ -251,6 +251,12 @@ services: build: context: ./docker/spark dockerfile: Dockerfile + # Not 3.5 until CORE-1558 lands (PyHive fails on its DDL results); not + # v3.3.2, whose JDK 11.0.16 crashes on GitHub's runners (JDK-8287073). + args: + SPARK_IMAGE: apache/spark:3.3.3 + DELTA_ARTIFACT: delta-core_2.12 + DELTA_VERSION: 2.2.0 ports: - "10000:10000" - "4040:4040" @@ -261,7 +267,18 @@ services: --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 --name Thrift JDBC/ODBC Server healthcheck: - test: ["CMD-SHELL", "nc -z localhost 10000"] + test: + [ + "CMD", + "beeline", + "-u", + "jdbc:hive2://localhost:10000", + "-n", + "dbt", + "--silent=true", + "-e", + "select 1", + ] interval: 10s timeout: 5s retries: 20 @@ -269,11 +286,9 @@ services: user: "0" volumes: - spark-warehouse:/spark-warehouse/ - - ./docker/spark/hive-site.xml:/usr/spark/conf/hive-site.xml - - ./docker/spark/spark-defaults.conf:/usr/spark/conf/spark-defaults.conf + - ./docker/spark/hive-site.xml:/opt/spark/conf/hive-site.xml + - ./docker/spark/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf - ./data:/seed-data:ro - environment: - - WAIT_FOR=spark-hive-metastore:5432 spark-hive-metastore: image: postgres:15-alpine @@ -284,7 +299,11 @@ services: - POSTGRES_PASSWORD=${SPARK_METASTORE_PASSWORD:-dbt} - POSTGRES_DB=metastore healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${SPARK_METASTORE_USER:-dbt}"] + test: + [ + "CMD-SHELL", + "pg_isready -h 127.0.0.1 -U ${SPARK_METASTORE_USER:-dbt} -d metastore", + ] interval: 5s timeout: 5s retries: 10 diff --git a/tests/e2e_dbt_project/docker/spark/Dockerfile b/tests/e2e_dbt_project/docker/spark/Dockerfile index aab2ab394..8c04b870b 100644 --- a/tests/e2e_dbt_project/docker/spark/Dockerfile +++ b/tests/e2e_dbt_project/docker/spark/Dockerfile @@ -1,33 +1,22 @@ -ARG OPENJDK_VERSION=8 -FROM eclipse-temurin:${OPENJDK_VERSION}-jre +ARG SPARK_IMAGE=apache/spark:3.3.3 +FROM ${SPARK_IMAGE} -ARG SPARK_VERSION=3.3.2 -ARG HADOOP_VERSION=3 +# delta-core_ before Delta 3.0, delta-spark_ from 3.0. +ARG DELTA_ARTIFACT=delta-core_2.12 ARG DELTA_VERSION=2.2.0 +ARG POSTGRES_JDBC_VERSION=42.7.13 -ENV SPARK_HOME /usr/spark -ENV PATH="/usr/spark/bin:/usr/spark/sbin:${PATH}" +ENV PATH="${SPARK_HOME}/bin:${SPARK_HOME}/sbin:${PATH}" -RUN apt-get update && \ - apt-get install -y --no-install-recommends wget netcat-openbsd procps libpostgresql-jdbc-java && \ - wget -q "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ - tar xzf "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ - rm "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ - mv "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" /usr/spark && \ - ln -s /usr/share/java/postgresql-jdbc4.jar /usr/spark/jars/postgresql-jdbc4.jar && \ - wget -q "https://repo1.maven.org/maven2/io/delta/delta-core_2.12/${DELTA_VERSION}/delta-core_2.12-${DELTA_VERSION}.jar" \ - -P /usr/spark/jars/ && \ - wget -q "https://repo1.maven.org/maven2/io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar" \ - -P /usr/spark/jars/ && \ - apt-get remove -y wget && \ - apt-get autoremove -y && \ - apt-get clean +# ADD gives files downloaded from a URL mode 600 (root only), hence a+r. +ADD --chmod=a+r \ + "https://repo1.maven.org/maven2/io/delta/${DELTA_ARTIFACT}/${DELTA_VERSION}/${DELTA_ARTIFACT}-${DELTA_VERSION}.jar" \ + "https://repo1.maven.org/maven2/io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar" \ + "https://repo1.maven.org/maven2/org/postgresql/postgresql/${POSTGRES_JDBC_VERSION}/postgresql-${POSTGRES_JDBC_VERSION}.jar" \ + ${SPARK_HOME}/jars/ -COPY entrypoint.sh /scripts/ -RUN chmod +x /scripts/entrypoint.sh && \ - groupadd -r spark && useradd -r -g spark -m spark && \ - chown -R spark:spark /usr/spark /scripts +# Spark's warehouse dir is relative to the workdir; /spark-warehouse is the volume. +WORKDIR / -USER spark -ENTRYPOINT ["/scripts/entrypoint.sh"] +ENTRYPOINT ["spark-submit"] CMD ["--help"] diff --git a/tests/e2e_dbt_project/docker/spark/entrypoint.sh b/tests/e2e_dbt_project/docker/spark/entrypoint.sh deleted file mode 100644 index 36b8174c9..000000000 --- a/tests/e2e_dbt_project/docker/spark/entrypoint.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -if [ -n "$WAIT_FOR" ]; then - WAIT_FOR_TIMEOUT_SECONDS="${WAIT_FOR_TIMEOUT_SECONDS:-180}" - IFS=';' read -a HOSTPORT_ARRAY <<< "$WAIT_FOR" - for HOSTPORT in "${HOSTPORT_ARRAY[@]}" - do - WAIT_FOR_HOST=${HOSTPORT%:*} - WAIT_FOR_PORT=${HOSTPORT#*:} - START_TS=$(date +%s) - - echo Waiting for $WAIT_FOR_HOST to listen on $WAIT_FOR_PORT... - while ! nc -z "$WAIT_FOR_HOST" "$WAIT_FOR_PORT"; do - if [ $(( $(date +%s) - START_TS )) -ge "$WAIT_FOR_TIMEOUT_SECONDS" ]; then - echo "Timed out waiting for $WAIT_FOR_HOST:$WAIT_FOR_PORT after ${WAIT_FOR_TIMEOUT_SECONDS}s" - exit 1 - fi - sleep 2 - done - done -fi - -exec spark-submit "$@"