diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml
index 1242064..9ff3a0a 100644
--- a/.github/workflows/containers.yml
+++ b/.github/workflows/containers.yml
@@ -42,12 +42,9 @@ jobs:
           - xgb-ci.jvm
           - xgb-ci.jvm_gpu_build
           - xgb-ci.manylinux_2_28_x86_64
-          - xgb-ci.manylinux2014_x86_64
           - xgb-ci.i386
         runner: [linux-amd64-cpu]
         include:
-          - image_repo: xgb-ci.manylinux2014_aarch64
-            runner: linux-arm64-cpu
           - image_repo: xgb-ci.manylinux_2_28_aarch64
             runner: linux-arm64-cpu
           - image_repo: xgb-ci.gpu_build_rockylinux8_aarch64
diff --git a/containers/ci_container.yml b/containers/ci_container.yml
index 74663bd..050a682 100644
--- a/containers/ci_container.yml
+++ b/containers/ci_container.yml
@@ -100,12 +100,6 @@ xgb-ci.manylinux_2_28_x86_64:
 xgb-ci.manylinux_2_28_aarch64:
   container_def: manylinux_2_28_aarch64
 
-xgb-ci.manylinux2014_x86_64:
-  container_def: manylinux2014_x86_64
-
-xgb-ci.manylinux2014_aarch64:
-  container_def: manylinux2014_aarch64
-
 xgb-ci.jvm:
   container_def: jvm
 
diff --git a/containers/dockerfile/Dockerfile.manylinux2014_aarch64 b/containers/dockerfile/Dockerfile.manylinux2014_aarch64
deleted file mode 100644
index 5f93f4f..0000000
--- a/containers/dockerfile/Dockerfile.manylinux2014_aarch64
+++ /dev/null
@@ -1,19 +0,0 @@
-FROM quay.io/pypa/manylinux2014_aarch64
-
-SHELL ["/bin/bash", "-c"]
-
-ENV GOSU_VERSION=1.10
-
-COPY scripts/install_gosu.sh /scripts/
-
-RUN yum update -y && yum install -y java-1.8.0-openjdk-devel
-
-# Install lightweight sudo (not bound to TTY)
-RUN sh /scripts/install_gosu.sh
-
-# Default entry-point to use if running locally
-# It will preserve attributes of created files
-COPY entrypoint.sh /scripts/
-
-WORKDIR /workspace
-ENTRYPOINT ["/scripts/entrypoint.sh"]
diff --git a/containers/dockerfile/Dockerfile.manylinux2014_x86_64 b/containers/dockerfile/Dockerfile.manylinux2014_x86_64
deleted file mode 100644
index 3801f9e..0000000
--- a/containers/dockerfile/Dockerfile.manylinux2014_x86_64
+++ /dev/null
@@ -1,19 +0,0 @@
-FROM quay.io/pypa/manylinux2014_x86_64
-
-SHELL ["/bin/bash", "-c"]
-
-ENV GOSU_VERSION=1.10
-
-COPY scripts/install_gosu.sh /scripts/
-
-RUN yum update -y && yum install -y java-1.8.0-openjdk-devel
-
-# Install lightweight sudo (not bound to TTY)
-RUN sh /scripts/install_gosu.sh
-
-# Default entry-point to use if running locally
-# It will preserve attributes of created files
-COPY entrypoint.sh /scripts/
-
-WORKDIR /workspace
-ENTRYPOINT ["/scripts/entrypoint.sh"]
diff --git a/containers/dockerfile/Dockerfile.manylinux_2_28_aarch64 b/containers/dockerfile/Dockerfile.manylinux_2_28_aarch64
index d9fa794..2d60db8 100644
--- a/containers/dockerfile/Dockerfile.manylinux_2_28_aarch64
+++ b/containers/dockerfile/Dockerfile.manylinux_2_28_aarch64
@@ -15,7 +15,8 @@ RUN \
     dnf -y update && \
     dnf -y install dnf-plugins-core && \
     dnf config-manager --set-enabled powertools && \
-    dnf install -y tar unzip wget xz git which ninja-build gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \
+    dnf install -y tar unzip wget xz git which ninja-build java-17-openjdk-devel \
+                   gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \
     # Miniforge
     wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Miniforge3-$MINIFORGE_VERSION-Linux-aarch64.sh && \
     bash conda.sh -b -p /opt/miniforge
diff --git a/containers/dockerfile/Dockerfile.manylinux_2_28_x86_64 b/containers/dockerfile/Dockerfile.manylinux_2_28_x86_64
index e890840..f8fef5f 100644
--- a/containers/dockerfile/Dockerfile.manylinux_2_28_x86_64
+++ b/containers/dockerfile/Dockerfile.manylinux_2_28_x86_64
@@ -6,6 +6,8 @@ ENV GOSU_VERSION=1.10
 
 COPY scripts/install_gosu.sh /scripts/
 
+RUN dnf -y update && dnf -y install java-17-openjdk-devel
+
 # Install lightweight sudo (not bound to TTY)
 RUN sh /scripts/install_gosu.sh