You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/06/05 23:42:32 UTC
[impala] 02/02: IMPALA-8425: part 1: reduce size of binaries in
container
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 45c5652b6694864afef2af3fdf93d3bd8dfa7190
Author: Tim Armstrong <ta...@cloudera.com>
AuthorDate: Fri May 31 09:18:54 2019 -0700
IMPALA-8425: part 1: reduce size of binaries in container
* Symlink impalad/catalog/statestored inside container.
This doesn't seem to really save any space - there's
some kind of deduplication going on.
* Don't include libfesupport.so, which shouldn't be needed.
* strip debug symbols from the binary.
* Only include the libkuduclient.so libraries for Kudu
This shaves ~1.1GB from the image size- 250MB as a result
of the impalad binary changes and the remainder from the
Kudu changs.
Change-Id: I95ff479bedd3b93e6569e72f03f42acd9dba8b14
Reviewed-on: http://gerrit.cloudera.org:8080/13487
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
docker/daemon_entrypoint.sh | 4 ++--
docker/impala_base/Dockerfile | 26 ++++++++++++++------------
docker/setup_build_context.py | 25 +++++++++++++++++--------
3 files changed, 33 insertions(+), 22 deletions(-)
diff --git a/docker/daemon_entrypoint.sh b/docker/daemon_entrypoint.sh
index 1bc7752..16fa7f0 100755
--- a/docker/daemon_entrypoint.sh
+++ b/docker/daemon_entrypoint.sh
@@ -27,9 +27,9 @@ export IMPALA_HOME=/opt/impala
# Add directories containing dynamic libraries required by the daemons that
# are not on the system library paths.
-export LD_LIBRARY_PATH=/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/
+export LD_LIBRARY_PATH=/opt/impala/lib
+LD_LIBRARY_PATH+=:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/
LD_LIBRARY_PATH+=:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/
-LD_LIBRARY_PATH+=:/opt/kudu/release/lib
# Add directory with optional plugins that can be mounted for the container.
LD_LIBRARY_PATH+=:/opt/impala/lib/plugins
diff --git a/docker/impala_base/Dockerfile b/docker/impala_base/Dockerfile
index 3f9bee0..225adda 100644
--- a/docker/impala_base/Dockerfile
+++ b/docker/impala_base/Dockerfile
@@ -24,24 +24,26 @@ RUN apt-get update && \
apt-get install -y openjdk-8-jre-headless \
libsasl2-2 libsasl2-modules libsasl2-modules-gssapi-mit \
tzdata liblzo2-2 && \
- apt-get clean
-
-# Use a non-privileged impala user to run the daemons in the container.
-# That user should own everything in the /opt/impala subdirectory.
-RUN groupadd -r impala && useradd --no-log-init -r -g impala impala && \
- mkdir -p /opt/impala && chown impala /opt/impala && \
- chmod ugo+w /etc/passwd
-USER impala
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/*
# Copy build artifacts required for the daemon processes.
# Need to have multiple copy commands to preserve directory structure.
-COPY bin /opt/impala/bin
COPY lib /opt/impala/lib
COPY www /opt/impala/www
-COPY kudu /opt/kudu
+COPY bin /opt/impala/bin
+# Symlink here instead of in setup_build_context to avoid duplicate binaries.
+RUN cd /opt/impala/bin && ln -s impalad statestored && ln -s impalad catalogd && \
# Create conf directory for later config injection.
-RUN mkdir /opt/impala/conf
+ mkdir /opt/impala/conf && \
# Create logs directory to collect container logs.
-RUN mkdir /opt/impala/logs
+ mkdir /opt/impala/logs
+
+# Use a non-privileged impala user to run the daemons in the container.
+# That user should own everything in the /opt/impala subdirectory.
+RUN groupadd -r impala && useradd --no-log-init -r -g impala impala && \
+ mkdir -p /opt/impala && chown impala -R /opt/impala && \
+ chmod ugo+w /etc/passwd
+USER impala
WORKDIR /opt/impala/
diff --git a/docker/setup_build_context.py b/docker/setup_build_context.py
index 817263e..b953522 100755
--- a/docker/setup_build_context.py
+++ b/docker/setup_build_context.py
@@ -23,6 +23,7 @@
import glob
import os
import shutil
+from subprocess import check_call
IMPALA_HOME = os.environ["IMPALA_HOME"]
OUTPUT_DIR = os.path.join(IMPALA_HOME, "docker/build_context")
@@ -30,7 +31,13 @@ DOCKERFILE = os.path.join(IMPALA_HOME, "docker/impala_base/Dockerfile")
IMPALA_TOOLCHAIN = os.environ["IMPALA_TOOLCHAIN"]
IMPALA_GCC_VERSION = os.environ["IMPALA_GCC_VERSION"]
+IMPALA_BINUTILS_VERSION = os.environ["IMPALA_BINUTILS_VERSION"]
GCC_HOME = os.path.join(IMPALA_TOOLCHAIN, "gcc-{0}".format(IMPALA_GCC_VERSION))
+BINUTILS_HOME = os.path.join(
+ IMPALA_TOOLCHAIN, "binutils-{0}".format(IMPALA_BINUTILS_VERSION))
+STRIP = os.path.join(BINUTILS_HOME, "bin/strip")
+KUDU_HOME = os.environ["IMPALA_KUDU_HOME"]
+KUDU_LIB_DIR = os.path.join(KUDU_HOME, "release/lib")
# Ensure the output directory exists and is empty.
if os.path.exists(OUTPUT_DIR):
@@ -47,18 +54,20 @@ os.mkdir(LIB_DIR)
def symlink_file_into_dir(src_file, dst_dir):
- """Helper to symlink 'src_file' into 'dst_dir'."""
- os.symlink(src_file, os.path.join(dst_dir, os.path.basename(src_file)))
+ """Helper to symlink 'src_file' into 'dst_dir'."""
+ os.symlink(src_file, os.path.join(dst_dir, os.path.basename(src_file)))
# Impala binaries and native dependencies.
-for bin in ["impalad", "statestored", "catalogd", "libfesupport.so"]:
- symlink_file_into_dir(os.path.join(IMPALA_HOME, "be/build/latest/service", bin),
- BIN_DIR)
+check_call([STRIP, "--strip-debug",
+ os.path.join(IMPALA_HOME, "be/build/latest/service/impalad"),
+ "-o", os.path.join(BIN_DIR, "impalad")])
for lib in ["libstdc++", "libgcc"]:
- for so in glob.glob(os.path.join(GCC_HOME, "lib64/{0}*.so*".format(lib))):
- symlink_file_into_dir(so, LIB_DIR)
-os.symlink(os.environ["IMPALA_KUDU_HOME"], os.path.join(OUTPUT_DIR, "kudu"))
+ for so in glob.glob(os.path.join(GCC_HOME, "lib64/{0}*.so*".format(lib))):
+ symlink_file_into_dir(so, LIB_DIR)
+
+for so in glob.glob(os.path.join(KUDU_LIB_DIR, "libkudu_client.so*")):
+ symlink_file_into_dir(so, LIB_DIR)
# Impala dependencies.
dep_classpath = file(os.path.join(IMPALA_HOME, "fe/target/build-classpath.txt")).read()