You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2020/01/14 19:41:35 UTC
[kudu] 03/03: KUDU-3005: Reduce size of kudu-python docker image
This is an automated email from the ASF dual-hosted git repository.
granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 1650e2f39449a1b89d4c3abba1106b5ad6bf824a
Author: Grant Henke <gr...@apache.org>
AuthorDate: Wed Dec 18 08:25:11 2019 -0600
KUDU-3005: Reduce size of kudu-python docker image
This patch breaks out the python environment bootstrapping from
the dev environment bootstrapping. This allows us to use the
runtime image for the kudu-python docker image instead of the
giant dev image.
The result of this change is a 440MB uncompressed image instead
of a 1.92GB uncompressed image.
I manually tested the image by running the python example against
a quickstart cluster:
https://github.com/apache/kudu/blob/master/examples/python/basic-python-example/basic_example.py
Change-Id: I159907c4a85e919d98c92df97cd44d0300d1cd29
Reviewed-on: http://gerrit.cloudera.org:8080/14984
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Reviewed-by: Andrew Wong <aw...@cloudera.com>
Tested-by: Kudu Jenkins
---
.dockerignore | 1 +
docker/Dockerfile | 21 +++++++---
docker/bootstrap-dev-env.sh | 32 ---------------
docker/bootstrap-python-env.sh | 93 ++++++++++++++++++++++++++++++++++++++++++
4 files changed, 110 insertions(+), 37 deletions(-)
diff --git a/.dockerignore b/.dockerignore
index 007dd33..1379088 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -30,6 +30,7 @@
# Docker files.
!docker/bootstrap-dev-env.sh
+!docker/bootstrap-python-env.sh
!docker/bootstrap-runtime-env.sh
!docker/kudu-entrypoint.sh
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 266c372..444dd32 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -62,7 +62,11 @@ ARG BASE_OS
FROM $BASE_OS as dev
COPY ./docker/bootstrap-dev-env.sh /
-RUN ./bootstrap-dev-env.sh && rm bootstrap-dev-env.sh
+COPY ./docker/bootstrap-python-env.sh /
+RUN ./bootstrap-dev-env.sh \
+ && ./bootstrap-python-env.sh \
+ && rm bootstrap-dev-env.sh \
+ && rm bootstrap-python-env.sh
# Common label arguments.
# VCS_REF is not specified to improve docker caching.
@@ -221,13 +225,20 @@ LABEL name="Apache Kudu Build" \
# ---- Kudu Python ----
# Builds a runtime image with the Kudu python client pre-installed.
#
-FROM dev AS kudu-python
+FROM runtime AS kudu-python
+
+COPY ./docker/bootstrap-python-env.sh /
+RUN ./bootstrap-python-env.sh \
+ && rm bootstrap-python-env.sh
+
+ARG INSTALL_DIR="/opt/kudu"
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib64
WORKDIR $INSTALL_DIR/python
# Copy the requirements file.
COPY ./python/requirements.txt requirements.txt
-COPY --from=build /usr/local /usr/local/
+COPY --from=build /usr/local/lib/libkudu_client* /usr/local/lib/
+COPY --from=build /usr/local/include/kudu /usr/local/include/kudu
COPY --from=build /kudu/python/dist/kudu-python-*.tar.gz .
RUN pip install -r requirements.txt \
&& rm -rf requirements.txt \
@@ -241,8 +252,8 @@ ARG VCS_TYPE
ARG VCS_URL
ARG VERSION
-LABEL org.label-schema.name="Apache Kudu Python Client Base" \
- org.label-schema.description="Builds a runtime image with the Kudu python client pre-installed." \
+LABEL org.label-schema.name="Apache Kudu Python Client" \
+ org.label-schema.description="An image with the Kudu Python client pre-installed." \
# Common labels.
org.label-schema.dockerfile=$DOCKERFILE \
org.label-schema.maintainer=$MAINTAINER \
diff --git a/docker/bootstrap-dev-env.sh b/docker/bootstrap-dev-env.sh
index 4cbddcf..62754ba 100755
--- a/docker/bootstrap-dev-env.sh
+++ b/docker/bootstrap-dev-env.sh
@@ -25,29 +25,6 @@
set -xe
-function install_python_packages() {
- PYTHON_VERSION=$(python --version 2>&1 | cut -d' ' -f2)
- PYTHON_MAJOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f1)
- PYTHON_MINOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f2)
-
- # We use get-pip.py to bootstrap pip outside of system packages.
- # This prevents issues with the platform package manager knowing
- # about only some of the python packages.
- if [[ "$PYTHON_MAJOR" == "2" && "$PYTHON_MINOR" == "6" ]]; then
- # Beginning with pip 10, Python 2.6 is no longer supported.
- curl https://bootstrap.pypa.io/2.6/get-pip.py | python
- else
- # Use a stable version of pip that works with the remaining
- # versions of Python 2 and 3. pip 19.1 doesn't support Python 3.4,
- # which is the version of Python 3 shipped with Ubuntu 14.04.
- curl https://bootstrap.pypa.io/get-pip.py | python - "pip < 19.0"
- fi
- pip install --upgrade \
- cython \
- setuptools \
- setuptools_scm
-}
-
# Install the prerequisite libraries, if they are not installed.
# CentOS/RHEL
if [[ -f "/usr/bin/yum" ]]; then
@@ -92,11 +69,6 @@ if [[ -f "/usr/bin/yum" ]]; then
ruby-devel \
zlib-devel
- # Install python development packages.
- yum install -y epel-release
- yum install -y python-devel
- install_python_packages
-
# To build on a version older than 7.0, the Red Hat Developer Toolset
# must be installed (in order to have access to a C++11 capable compiler).
OS_MAJOR_VERSION=$(lsb_release -rs | cut -f1 -d.)
@@ -180,10 +152,6 @@ elif [[ -f "/usr/bin/apt-get" ]]; then
xsltproc \
zlib1g-dev
- # Install python development packages.
- apt-get install -y --no-install-recommends python-dev
- install_python_packages
-
# Reduce the image size by cleaning up after the install.
apt-get clean
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
diff --git a/docker/bootstrap-python-env.sh b/docker/bootstrap-python-env.sh
new file mode 100755
index 0000000..ac7f14b
--- /dev/null
+++ b/docker/bootstrap-python-env.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+##########################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# This script handles bootstrapping a base OS for
+# the Apache Kudu base docker images.
+#
+##########################################################
+
+set -xe
+
+function install_python_packages() {
+ PYTHON_VERSION=$(python --version 2>&1 | cut -d' ' -f2)
+ PYTHON_MAJOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f1)
+ PYTHON_MINOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f2)
+
+ # We use get-pip.py to bootstrap pip outside of system packages.
+ # This prevents issues with the platform package manager knowing
+ # about only some of the python packages.
+ if [[ "$PYTHON_MAJOR" == "2" && "$PYTHON_MINOR" == "6" ]]; then
+ # Beginning with pip 10, Python 2.6 is no longer supported.
+ curl https://bootstrap.pypa.io/2.6/get-pip.py | python
+ else
+ # Use a stable version of pip that works with the remaining
+ # versions of Python 2 and 3. pip 19.1 doesn't support Python 3.4,
+ # which is the version of Python 3 shipped with Ubuntu 14.04.
+ curl https://bootstrap.pypa.io/get-pip.py | python - "pip < 19.0"
+ fi
+ pip install --upgrade \
+ cython \
+ setuptools \
+ setuptools_scm
+}
+
+# Install the prerequisite libraries, if they are not installed.
+# CentOS/RHEL
+if [[ -f "/usr/bin/yum" ]]; then
+ # Update the repo.
+ yum update -y
+
+ # Install curl, used when installing pip.
+ yum install -y ca-certificates curl
+
+ # Install python development packages.
+ yum install -y epel-release
+ # g++ is required to check for int128 support in setup.py.
+ yum install -y gcc gcc-c++ python-devel
+ install_python_packages
+
+ # Reduce the image size by cleaning up after the install.
+ yum clean all
+ rm -rf /var/cache/yum /tmp/* /var/tmp/*
+# Ubuntu/Debian
+elif [[ -f "/usr/bin/apt-get" ]]; then
+ # Ensure the Debian frontend is noninteractive.
+ export DEBIAN_FRONTEND=noninteractive
+
+ # Update the repo.
+ apt-get update -y
+
+ # Install curl, used when installing pip.
+ apt-get install -y --no-install-recommends ca-certificates curl
+
+ # Install python development packages.
+ # g++ is required to check for int128 support in setup.py.
+ apt-get install -y --no-install-recommends g++ python-dev
+ install_python_packages
+
+ # Reduce the image size by cleaning up after the install.
+ apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+ unset DEBIAN_FRONTEND
+else
+ echo "Unsupported OS"
+ exit 1
+fi