(hudi) branch master updated: chore(docker): reduce base_java17 and spark_base image size (#18542)

[vh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 0a28d695eb15 chore(docker): reduce base_java17 and spark_base image size (#18542)
0a28d695eb15 is described below

commit 0a28d695eb1558b01bb38b11996a436a2e3e23dd
Author: kartikeyaagrawal <52...@users.noreply.github.com>
AuthorDate: Thu Apr 30 15:10:08 2026 +0530

    chore(docker): reduce base_java17 and spark_base image size (#18542)
    
    Addresses #18523.
    
    Shrinks the Java 17 integration-test image from ~3.56 GB to ~2.58 GB
    (~27%) without changing any runtime behavior. The container commands,
    environment variables, exposed ports, and entrypoint are identical.
    
    base_java17/Dockerfile:
    - switch base image from eclipse-temurin:17-jdk to 17-jre-jammy. The
      container only runs Hadoop; no Java compilation happens inside it,
      so the JDK toolchain is not needed.
    - convert to a multi-stage build. Stage 1 downloads and extracts the
      Hadoop tarball; stage 2 only COPYs the extracted tree. curl,
      ca-certificates, and the tar.gz no longer land in the final layer.
    - use --no-install-recommends and clean apt lists in the runtime stage.
    - drop the unused .asc signature download and the now-dead wget dep.
    
    spark_base/Dockerfile:
    - replace the Python-3.10.14-from-source build (which pulled in
      build-essential and a full compile toolchain, then built CPython
      with --enable-optimizations inside the image) with the distro
      python3-minimal + python3-pip packages. PySpark only needs a
      Python runtime at runtime.
---
 docker/hoodie/hadoop/base_java17/Dockerfile | 41 ++++++++++++++++++-----------
 docker/hoodie/hadoop/spark_base/Dockerfile  | 21 +++------------
 2 files changed, 30 insertions(+), 32 deletions(-)

diff --git a/docker/hoodie/hadoop/base_java17/Dockerfile b/docker/hoodie/hadoop/base_java17/Dockerfile
index 45108610b19e..a5c265d3aaac 100644
--- a/docker/hoodie/hadoop/base_java17/Dockerfile
+++ b/docker/hoodie/hadoop/base_java17/Dockerfile
@@ -15,7 +15,23 @@
 #  See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM eclipse-temurin:17-jdk
+# --- Stage 1: fetch + extract Hadoop (throwaway) ---
+FROM eclipse-temurin:17-jre-jammy AS hadoop-builder
+
+ARG HADOOP_VERSION=3.4.0
+ARG HADOOP_URL=https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz
+
+RUN set -x \
+    && DEBIAN_FRONTEND=noninteractive apt-get -yq update \
+    && apt-get -yq install --no-install-recommends curl ca-certificates \
+    && curl -fSL "${HADOOP_URL}" -o /tmp/hadoop.tar.gz \
+    && mkdir -p /opt \
+    && tar -xzf /tmp/hadoop.tar.gz -C /opt/ \
+    && rm /tmp/hadoop.tar.gz \
+    && mkdir -p /opt/hadoop-${HADOOP_VERSION}/logs
+
+# --- Stage 2: runtime ---
+FROM eclipse-temurin:17-jre-jammy
 LABEL maintainer="Hoodie"
 USER root
 
@@ -23,28 +39,23 @@ USER root
 ENV LANG C.UTF-8
 
 ARG HADOOP_VERSION=3.4.0
-ARG HADOOP_URL=https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz
-ENV HADOOP_VERSION ${HADOOP_VERSION}
-ENV HADOOP_URL ${HADOOP_URL}
+ENV HADOOP_VERSION=${HADOOP_VERSION}
 
-RUN set -x \
-    && DEBIAN_FRONTEND=noninteractive apt-get -yq update && apt-get -yq install curl wget netcat-openbsd procps \
-    && echo "Fetch URL2 is : ${HADOOP_URL}" \
-    && curl -fSL "${HADOOP_URL}" -o /tmp/hadoop.tar.gz \
-    && curl -fSL "${HADOOP_URL}.asc" -o /tmp/hadoop.tar.gz.asc \
-    && mkdir -p /opt/hadoop-$HADOOP_VERSION/logs \
-    && tar -xvf /tmp/hadoop.tar.gz -C /opt/ \
-    && rm /tmp/hadoop.tar.gz* \
-    && ln -s /opt/hadoop-$HADOOP_VERSION/etc/hadoop /etc/hadoop \
+RUN DEBIAN_FRONTEND=noninteractive apt-get -yq update \
+    && apt-get -yq install --no-install-recommends netcat-openbsd procps \
+    && rm -rf /var/lib/apt/lists/* \
     && mkdir /hadoop-data
 
-ENV HADOOP_PREFIX=/opt/hadoop-$HADOOP_VERSION
+COPY --from=hadoop-builder /opt/hadoop-${HADOOP_VERSION} /opt/hadoop-${HADOOP_VERSION}
+RUN ln -s /opt/hadoop-${HADOOP_VERSION}/etc/hadoop /etc/hadoop
+
+ENV HADOOP_PREFIX=/opt/hadoop-${HADOOP_VERSION}
 ENV HADOOP_CONF_DIR=/etc/hadoop
 ENV MULTIHOMED_NETWORK=1
 ENV HADOOP_HOME=${HADOOP_PREFIX}
 ENV HADOOP_INSTALL=${HADOOP_HOME}
 ENV USER=root
-ENV PATH /usr/bin:/bin:$HADOOP_PREFIX/bin/:$PATH
+ENV PATH=/usr/bin:/bin:${HADOOP_PREFIX}/bin/:$PATH
 
 # Exposing a union of ports across hadoop versions
 # Well known ports including ssh
diff --git a/docker/hoodie/hadoop/spark_base/Dockerfile b/docker/hoodie/hadoop/spark_base/Dockerfile
index 68bfbaae76d8..4210faf21bfe 100644
--- a/docker/hoodie/hadoop/spark_base/Dockerfile
+++ b/docker/hoodie/hadoop/spark_base/Dockerfile
@@ -41,23 +41,10 @@ RUN echo "Installing Spark-version (${SPARK_VERSION})" \
       && rm spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \
       && cd /
 
-# Install build dependencies
-RUN apt-get update && apt-get install -y \
-    wget build-essential libncursesw5-dev \
-    libssl-dev libgdbm-dev libreadline-dev libbz2-dev \
-    libsqlite3-dev libffi-dev zlib1g-dev curl \
-    && cd /usr/src \
-    && wget https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tgz \
-    && tar xzf Python-3.10.14.tgz \
-    && cd Python-3.10.14 \
-    && ./configure --enable-optimizations \
-    && make -j"$(nproc)" \
-    && make altinstall \
-    && ln -sf /usr/local/bin/python3.10 /usr/bin/python \
-    && ln -sf /usr/local/bin/python3.10 /usr/bin/python3 \
-    && curl -sS https://bootstrap.pypa.io/get-pip.py | python \
-    && pip install --upgrade pip \
-    && cd / && rm -rf /usr/src/Python-3.10.14* \
+# Install Python runtime from distro package (avoids ~400MB build toolchain)
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends python3-minimal python3-pip \
+    && ln -sf /usr/bin/python3 /usr/bin/python \
     && rm -rf /var/lib/apt/lists/*
 
 #Give permission to execute scripts