Bump integration infra to Iceberg 1.12.0 and Spark 4.1.3

Also bump hadoop-aws, AWS SDK, Hive metastore client, aws-cli, and
pyspark. Hive 4.2 moved IMetaStoreClient into the new
hive-standalone-metastore-client artifact, so download it as well.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

Author: kevinjqliu

dev/docker-compose-integration.yml

@@ -85,7 +85,7 @@ services:
       retries: 10
       start_period: 5s
   aws-cli:
-    image: amazon/aws-cli:2.37.2
+    image: amazon/aws-cli:2.38.5
     container_name: pyiceberg-aws-cli
     networks:
       iceberg_net:

dev/docker-compose.yml

@@ -33,7 +33,7 @@ services:
       retries: 10
       start_period: 5s
   aws-cli:
-    image: amazon/aws-cli:2.37.2
+    image: amazon/aws-cli:2.38.5
     container_name: pyiceberg-aws-cli
     depends_on:
       object-store:

dev/spark/Dockerfile

@@ -13,17 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG BASE_IMAGE_SPARK_VERSION=4.0.1
+ARG BASE_IMAGE_SPARK_VERSION=4.1.3
 
 FROM apache/spark:${BASE_IMAGE_SPARK_VERSION}
 
 # Dependency versions - keep these compatible
 # Changing these will invalidate the JAR download cache layer
-ARG ICEBERG_VERSION=1.11.0
-ARG ICEBERG_SPARK_RUNTIME_VERSION=4.0_2.13
-ARG HADOOP_VERSION=3.4.1
-ARG AWS_SDK_VERSION=2.24.6
-ARG HIVE_METASTORE_VERSION=4.0.1
+ARG ICEBERG_VERSION=1.12.0
+ARG ICEBERG_SPARK_RUNTIME_VERSION=4.1_2.13
+ARG HADOOP_VERSION=3.4.2
+ARG AWS_SDK_VERSION=2.57.3
+ARG HIVE_METASTORE_VERSION=4.2.1
 ARG MAVEN_MIRROR=https://repo.maven.apache.org/maven2
 
 USER root
@@ -51,6 +51,7 @@ RUN set -e && \
         "org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar" \
         "org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar" \
         "org/apache/hive/hive-standalone-metastore-common/${HIVE_METASTORE_VERSION}/hive-standalone-metastore-common-${HIVE_METASTORE_VERSION}.jar" \
+        "org/apache/hive/hive-standalone-metastore-client/${HIVE_METASTORE_VERSION}/hive-standalone-metastore-client-${HIVE_METASTORE_VERSION}.jar" \
         "software/amazon/awssdk/bundle/${AWS_SDK_VERSION}/bundle-${AWS_SDK_VERSION}.jar"; \
     do \
         jar_name=$(basename "${jar_path}") && \

pyproject.toml

@@ -116,7 +116,7 @@ dev = [
     "moto[server]>=5.0.2",
     "typing-extensions==4.16.0",
     "pytest-mock==3.15.1",
-    "pyspark[connect]==4.0.1",
+    "pyspark[connect]==4.1.3",
     "protobuf==7.36.2", # match Spark Connect's gencode
     "cython>=3.0.0",
     "deptry>=0.14",
@@ -258,7 +258,7 @@ filterwarnings = [
   "ignore:subprocess.*is still running:ResourceWarning",
   # Ignore google-crc32c C extension missing warning (common on Python 3.14+)
   "ignore:As the c extension couldn't be imported:RuntimeWarning:google_crc32c",
-  # Ignore Spark 4.0.1 pandas conversion warning under pandas 3.0
+  # Ignore Spark pandas conversion warning under pandas 3.0
   "ignore:The copy keyword is deprecated and will be removed in a future version.*",
 ]