You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2022/10/11 20:34:18 UTC
[impala] branch master updated (3821b8c82 -> 11e66523d)
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 3821b8c82 IMPALA-11648: validate-java-pom-versions.sh should skip pom.xml in toolchain
new 3962ae197 IMPALA-8770: Support building Docker images on Redhat-based distributions
new 72812c595 IMPALA-11610: Pass environment variables into dockerized-impala-run-tests.sh
new 3d269e465 IMPALA-11634: Provide an option to use Java 11 for docker images
new 11e66523d IMPALA-11526: Install en_US.UTF-8 locale into docker images
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
bin/bootstrap_system.sh | 2 +-
bin/impala-config.sh | 16 ++
.../dockerized-impala-bootstrap-and-test.sh | 32 ++--
bin/jenkins/dockerized-impala-preserve-vars.py | 56 +++++++
bin/jenkins/dockerized-impala-run-tests.sh | 11 +-
bin/jenkins/install_docker.sh | 90 +++++++++++
docker/CMakeLists.txt | 129 ++++++++++------
docker/admissiond/Dockerfile | 6 +
docker/catalogd/Dockerfile | 6 +
docker/daemon_entrypoint.sh | 135 ++++++++++++++++-
docker/impala_base/Dockerfile | 16 +-
docker/impala_profile_tool/Dockerfile | 16 +-
docker/impalad_coord_exec/Dockerfile | 6 +
docker/impalad_coordinator/Dockerfile | 6 +
docker/impalad_executor/Dockerfile | 6 +
docker/install_os_packages.sh | 164 +++++++++++++++++++++
docker/setup_build_context.py | 65 ++++++--
docker/statestored/Dockerfile | 5 +
.../queries/QueryTest/utf8-string-functions.test | 80 ++++++++++
19 files changed, 752 insertions(+), 95 deletions(-)
create mode 100755 bin/jenkins/dockerized-impala-preserve-vars.py
create mode 100755 bin/jenkins/install_docker.sh
create mode 100755 docker/install_os_packages.sh
[impala] 03/04: IMPALA-11634: Provide an option to use Java 11 for docker images
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 3d269e465e59b7f3af9f124598e66781ba8cd829
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Wed Sep 21 21:18:27 2022 -0700
IMPALA-11634: Provide an option to use Java 11 for docker images
Currently, Docker images install Java 8 for Impala's use. This
adds the IMPALA_DOCKER_USE_JAVA11 environment variable. When
set to true, this installs Java 11 rather than Java 8. It
defaults to false. The daemon_entrypoint.sh script is modified
to detect Java 11 correctly. As a workaround for IMPALA-11260,
this appends a list of "--add-opens" statements to JAVA_TOOL_OPTIONS
when running with Java 11.
Testing:
- Ran a set of dockerized tests on Rocky 8.5 with Java 11
Change-Id: Icc1dbd3f6a2279840218dc1da2b60077e211a328
Reviewed-on: http://gerrit.cloudera.org:8080/19031
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Reviewed-by: Laszlo Gaal <la...@cloudera.com>
Tested-by: Joe McDonnell <jo...@cloudera.com>
---
bin/impala-config.sh | 6 ++
.../dockerized-impala-bootstrap-and-test.sh | 3 +-
docker/CMakeLists.txt | 3 +
docker/daemon_entrypoint.sh | 65 +++++++++++++++++++++-
docker/install_os_packages.sh | 27 ++++++++-
5 files changed, 99 insertions(+), 5 deletions(-)
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index fd705d84d..151f4b4b8 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -236,6 +236,12 @@ export IMPALA_COS_VERSION=3.1.0-5.9.3
export IMPALA_REDHAT7_DOCKER_BASE=${IMPALA_REDHAT7_DOCKER_BASE:-"centos:centos7.9.2009"}
export IMPALA_REDHAT8_DOCKER_BASE=${IMPALA_REDHAT8_DOCKER_BASE:-"rockylinux:8.5"}
+# When set to true, this modifies the Docker image build logic to install Java 11
+# rather than the default of Java 8. This is strictly about what is shipped in the
+# Docker image, and it has no impact on what version of Java is used to compile
+# Impala's Java code.
+export IMPALA_DOCKER_USE_JAVA11=${IMPALA_DOCKER_USE_JAVA11:-"false"}
+
# When IMPALA_(CDP_COMPONENT)_URL are overridden, they may contain '$(platform_label)'
# which will be substituted for the CDP platform label in bootstrap_toolchain.py
unset IMPALA_HADOOP_URL
diff --git a/bin/jenkins/dockerized-impala-bootstrap-and-test.sh b/bin/jenkins/dockerized-impala-bootstrap-and-test.sh
index 7317ba93c..19b8c0220 100755
--- a/bin/jenkins/dockerized-impala-bootstrap-and-test.sh
+++ b/bin/jenkins/dockerized-impala-bootstrap-and-test.sh
@@ -35,7 +35,8 @@ source ./bin/bootstrap_system.sh
# NOTE: A Jenkins job can also call dockerized-impala-preserve-vars.py directly
# to preserve additional variables.
./bin/jenkins/dockerized-impala-preserve-vars.py \
- EE_TEST EE_TEST_FILES JDBC_TEST EXPLORATION_STRATEGY CMAKE_BUILD_TYPE
+ EE_TEST EE_TEST_FILES JDBC_TEST EXPLORATION_STRATEGY CMAKE_BUILD_TYPE \
+ IMPALA_DOCKER_USE_JAVA11
# Execute the tests using su to re-login so that group change made above
# setup_docker takes effect. This does a full re-login and does not stay
diff --git a/docker/CMakeLists.txt b/docker/CMakeLists.txt
index 4692a4830..c94e97898 100644
--- a/docker/CMakeLists.txt
+++ b/docker/CMakeLists.txt
@@ -88,6 +88,9 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
if (${build_type} STREQUAL "debug")
set(INSTALL_OS_PACKAGES_ARGS "--install-debug-tools")
endif()
+ if ($ENV{IMPALA_DOCKER_USE_JAVA11} STREQUAL "true")
+ set(INSTALL_OS_PACKAGES_ARGS "${INSTALL_OS_PACKAGES_ARGS} --use-java11")
+ endif()
# Target for the base Impala image.
add_custom_target(impala_base_image_${build_type}
# Run docker build inside the build context directory so that all dependencies are
diff --git a/docker/daemon_entrypoint.sh b/docker/daemon_entrypoint.sh
index 6cc7bd89e..a62cc81f5 100755
--- a/docker/daemon_entrypoint.sh
+++ b/docker/daemon_entrypoint.sh
@@ -52,11 +52,35 @@ fi
# Need libjvm.so and libjsig.so on LD_LIBRARY_PATH
# Ubuntu and Redhat use different locations for JAVA_HOME.
+# This detection logic handles both Java 8 and Java 11.
+# If both are present (which shouldn't happen), Java 11 is preferred.
JAVA_HOME=Unknown
+USING_JAVA11=false
if [[ $DISTRIBUTION == Ubuntu ]]; then
- JAVA_HOME=$(compgen -G /usr/lib/jvm/java-8-openjdk-*)
+ # Since the Java location includes the CPU architecture, use a glob
+ # to find Java home
+ if compgen -G "/usr/lib/jvm/java-11-openjdk*" ; then
+ JAVA_HOME=$(compgen -G "/usr/lib/jvm/java-11-openjdk*")
+ USING_JAVA11=true
+ if compgen -G "/usr/lib/jvm/java-8-openjdk*" ; then
+ echo "WARNING: Java 8 is also present"
+ fi
+ elif compgen -G "/usr/lib/jvm/java-8-openjdk*" ; then
+ echo "Detected Java 8"
+ JAVA_HOME=$(compgen -G "/usr/lib/jvm/java-8-openjdk*")
+ fi
elif [[ $DISTRIBUTION == Redhat ]]; then
- JAVA_HOME=/usr/lib/jvm/jre-1.8.0
+ if [[ -d /usr/lib/jvm/jre-11 ]]; then
+ echo "Detected Java 11"
+ JAVA_HOME=/usr/lib/jvm/jre-11
+ USING_JAVA11=true
+ if [[ -d /usr/lib/jvm/jre-1.8.0 ]]; then
+ echo "WARNING: Java 8 is also present"
+ fi
+ elif [[ -d /usr/lib/jvm/jre-1.8.0 ]]; then
+ echo "Detected Java 8"
+ JAVA_HOME=/usr/lib/jvm/jre-1.8.0
+ fi
fi
if [[ $JAVA_HOME == Unknown ]]; then
@@ -93,9 +117,46 @@ done
echo "CLASSPATH: $CLASSPATH"
echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
+# IMPALA-11260: ehcache underestimates sizes on Java 9+ due to restrictions
+# on reflection. As a workaround, this turns off the restrictions to allow
+# ehcache to function. The real fix needs to happen on the ehcache side.
+if $USING_JAVA11 ; then
+ JAVA11_OPTIONS=" --add-opens=java.base/java.io=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/java.lang.module=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/java.lang.ref=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/java.lang.reflect=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/java.lang=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/java.net=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/java.nio.charset=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/java.nio.file.attribute=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/java.nio=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/java.security=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/java.util.concurrent=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/java.util.jar=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/java.util.zip=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/java.util=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/jdk.internal.math=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/jdk.internal.module=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/jdk.internal.util.jar=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=java.base/sun.nio.fs=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=jdk.dynalink/jdk.dynalink.beans=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=jdk.dynalink/jdk.dynalink.linker.support=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=jdk.dynalink/jdk.dynalink.linker=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=jdk.dynalink/jdk.dynalink.support=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=jdk.dynalink/jdk.dynalink=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED"
+ JAVA11_OPTIONS+=" --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED"
+ JAVA_TOOL_OPTIONS="$JAVA11_OPTIONS $JAVA_TOOL_OPTIONS"
+fi
+
# Default to 2GB heap. Allow overriding by externally-set JAVA_TOOL_OPTIONS.
export JAVA_TOOL_OPTIONS="-Xmx2g $JAVA_TOOL_OPTIONS"
+echo "JAVA_TOOL_OPTIONS: $JAVA_TOOL_OPTIONS"
+
# Various Hadoop libraries depend on having a username. If we're running under
# an unknown username, create an entry in the password file for this user.
if ! whoami ; then
diff --git a/docker/install_os_packages.sh b/docker/install_os_packages.sh
index db4348d1a..e05fdaa68 100755
--- a/docker/install_os_packages.sh
+++ b/docker/install_os_packages.sh
@@ -24,10 +24,12 @@
set -euo pipefail
INSTALL_DEBUG_TOOLS=false
+USE_JAVA11=false
function print_usage {
echo "install_os_packages.sh - Helper script to install OS dependencies"
echo "[--install-debug-tools] : Also install debug tools like curl, iproute, etc"
+ echo "[--use-java11] : Use Java 11 rather than the default Java 8."
}
while [ -n "$*" ]
@@ -36,6 +38,9 @@ do
--install-debug-tools)
INSTALL_DEBUG_TOOLS=true
;;
+ --use-java11)
+ USE_JAVA11=true
+ ;;
--help|*)
print_usage
exit 1
@@ -45,6 +50,7 @@ do
done
echo "INSTALL_DEBUG_TOOLS=${INSTALL_DEBUG_TOOLS}"
+echo "USE_JAVA11=${USE_JAVA11}"
# This can get more detailed if there are specific steps
# for specific versions, but at the moment the distribution
@@ -58,6 +64,13 @@ else
if [[ $DISTRIB_ID == Ubuntu ]]; then
echo "Identified Ubuntu system."
DISTRIBUTION=Ubuntu
+
+ # Ubuntu 16.04 does not have Java 11 in its package repository,
+ # so exit with an error.
+ if [[ $DISTRIB_RELEASE == 16.04 ]] && $USE_JAVA11 ; then
+ echo "ERROR: Java 11 is not supported on Ubuntu 16.04"
+ exit 1
+ fi
fi
fi
@@ -72,12 +85,16 @@ fi
if [[ $DISTRIBUTION == Ubuntu ]]; then
export DEBIAN_FRONTEND=noninteractive
apt-get update
+ if $USE_JAVA11 ; then
+ apt-get install -y openjdk-11-jre-headless
+ else
+ apt-get install -y openjdk-8-jre-headless
+ fi
apt-get install -y \
krb5-user \
libsasl2-2 \
libsasl2-modules \
libsasl2-modules-gssapi-mit \
- openjdk-8-jre-headless \
tzdata
if $INSTALL_DEBUG_TOOLS ; then
echo "Installing extra debug tools"
@@ -92,10 +109,16 @@ if [[ $DISTRIBUTION == Ubuntu ]]; then
vim
fi
elif [[ $DISTRIBUTION == Redhat ]]; then
+ if $USE_JAVA11 ; then
+ yum install -y --disableplugin=subscription-manager \
+ java-11-openjdk-headless
+ else
+ yum install -y --disableplugin=subscription-manager \
+ java-1.8.0-openjdk-headless
+ fi
yum install -y --disableplugin=subscription-manager \
cyrus-sasl-gssapi \
cyrus-sasl-plain \
- java-1.8.0-openjdk-headless \
krb5-workstation \
openldap-devel \
tzdata
[impala] 04/04: IMPALA-11526: Install en_US.UTF-8 locale into docker images
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 11e66523d6070957f84c1fdbba3e26ecf3888d74
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Thu Sep 29 14:36:34 2022 -0700
IMPALA-11526: Install en_US.UTF-8 locale into docker images
In IMPALA-11492, ExprTest.Utf8MaskTest was failing on some
configurations because the en_US.UTF-8 was missing. Since the
Docker images don't contain en_US.UTF-8, they are subject
to the same bug. This was confirmed by adding tests cases
to the test_utf8_strings.py end-to-end test and running it
in the dockerized tests.
This add the appropriate language pack to the list of packages
installed for the Docker build.
Testing:
- This adds end-to-end tests to test_utf8_strings.py covering the
same cases that were failing in ExprTest.Utf8MaskTest. They
failed without the added languages packs, and now succeed.
Change-Id: I353f257b3cb6d45f7d0a28f7d5319fdb457e6e3d
Reviewed-on: http://gerrit.cloudera.org:8080/19080
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Laszlo Gaal <la...@cloudera.com>
---
bin/bootstrap_system.sh | 2 +-
docker/daemon_entrypoint.sh | 21 +++++-
docker/install_os_packages.sh | 17 +++++
.../queries/QueryTest/utf8-string-functions.test | 80 ++++++++++++++++++++++
4 files changed, 118 insertions(+), 2 deletions(-)
diff --git a/bin/bootstrap_system.sh b/bin/bootstrap_system.sh
index d637106cb..45c61c34e 100755
--- a/bin/bootstrap_system.sh
+++ b/bin/bootstrap_system.sh
@@ -268,7 +268,7 @@ redhat sudo yum install -y curl gawk gcc gcc-c++ git krb5-devel krb5-server \
wget vim-common nscd cmake fuse-devel zlib-devel \
psmisc lsof openssh-server redhat-lsb java-1.8.0-openjdk-devel \
java-1.8.0-openjdk-src python3-devel python3-setuptools net-tools \
- langpacks-en
+ langpacks-en glibc-langpack-en
# Enable the Powertools repo for snappy-devel on RedHat 8
redhat8 sudo yum install -y dnf-plugins-core
diff --git a/docker/daemon_entrypoint.sh b/docker/daemon_entrypoint.sh
index a62cc81f5..08deadcab 100755
--- a/docker/daemon_entrypoint.sh
+++ b/docker/daemon_entrypoint.sh
@@ -176,7 +176,26 @@ fi
# Set ulimit core file size 0.
ulimit -c 0
+# The UTF-8 masking functions rely on the presence of en_US.utf8. Make sure
+# it is present.
+if locale -a | grep en_US.utf8 ; then
+ echo "en_US.utf8 is present"
+else
+ echo "ERROR: en_US.utf8 locale is not present."
+ exit 1
+fi
+
# Set a UTF-8 locale to enable upper/lower/initcap functions with UTF-8 mode.
-export LC_ALL=C.UTF-8
+# Use C.UTF-8 (aka C.utf8) if it is available, and fall back to en_US.utf8 if not
+#
+# Distributions can show either C.UTF-8 or C.utf8 in "locale -a", match either one
+if locale -a | grep -e "^C.UTF-8" -e "^C.utf8" ; then
+ # C.UTF-8 and C.utf8 are interchangeable as a setting for LC_ALL.
+ export LC_ALL=C.UTF-8
+else
+ # Presence of en_US.utf8 was verified above
+ export LC_ALL=en_US.utf8
+fi
+echo "LC_ALL: ${LC_ALL}"
exec "$@"
diff --git a/docker/install_os_packages.sh b/docker/install_os_packages.sh
index e05fdaa68..f11fe94b4 100755
--- a/docker/install_os_packages.sh
+++ b/docker/install_os_packages.sh
@@ -92,6 +92,7 @@ if [[ $DISTRIBUTION == Ubuntu ]]; then
fi
apt-get install -y \
krb5-user \
+ language-pack-en \
libsasl2-2 \
libsasl2-modules \
libsasl2-modules-gssapi-mit \
@@ -122,6 +123,16 @@ elif [[ $DISTRIBUTION == Redhat ]]; then
krb5-workstation \
openldap-devel \
tzdata
+
+ # UTF-8 masking functions require the presence of en_US.utf8.
+ # Install the appropriate language packs. Redhat/Centos 7 come
+ # with en_US.utf8, so there is no need to install anything.
+ if ! grep 'release 7\.' /etc/redhat-release; then
+ yum install -y --disableplugin=subscription-manager \
+ glibc-langpack-en \
+ langpacks-en
+ fi
+
if $INSTALL_DEBUG_TOOLS ; then
echo "Installing extra debug tools"
yum install -y --disableplugin=subscription-manager \
@@ -137,6 +148,12 @@ elif [[ $DISTRIBUTION == Redhat ]]; then
fi
fi
+# Verify en_US.utf8 is present
+if ! locale -a | grep en_US.utf8 ; then
+ echo "ERROR: en_US.utf8 locale is not present."
+ exit 1
+fi
+
# To minimize the size for the Docker image, clean up any unnecessary files.
if [[ $DISTRIBUTION == Ubuntu ]]; then
apt-get clean
diff --git a/testdata/workloads/functional-query/queries/QueryTest/utf8-string-functions.test b/testdata/workloads/functional-query/queries/QueryTest/utf8-string-functions.test
index 8d607c95e..9417e5ad4 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/utf8-string-functions.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/utf8-string-functions.test
@@ -181,6 +181,86 @@ select mask('SQL引擎', 'x', 'x', 'x', 'x'),
STRING,STRING,STRING,STRING,STRING
====
---- QUERY
+set utf8_mode=true;
+select mask('abcd áäèü ABCD ÁÄÈÜ');
+---- RESULTS: RAW_STRING
+'xxxx xxxx XXXX XXXX'
+---- TYPES
+STRING
+====
+---- QUERY
+set utf8_mode=true;
+select mask('Ich möchte ein Bier. Tschüss');
+---- RESULTS: RAW_STRING
+'Xxx xxxxxx xxx Xxxx. Xxxxxxx'
+---- TYPES
+STRING
+====
+---- QUERY
+set utf8_mode=true;
+select mask('Hungarian áéíöóőüúű ÁÉÍÖÓŐÜÚŰ');
+---- RESULTS: RAW_STRING
+'Xxxxxxxxx xxxxxxxxx XXXXXXXXX'
+---- TYPES
+STRING
+====
+---- QUERY
+set utf8_mode=true;
+select mask('German äöüß ÄÖÜẞ');
+---- RESULTS: RAW_STRING
+'Xxxxxx xxxx XXXX'
+---- TYPES
+STRING
+====
+---- QUERY
+set utf8_mode=true;
+select mask('French àâæçéèêëïîôœùûüÿ ÀÂÆÇÉÈÊËÏÎÔŒÙÛÜŸ');
+---- RESULTS: RAW_STRING
+'Xxxxxx xxxxxxxxxxxxxxxx XXXXXXXXXXXXXXXX'
+---- TYPES
+STRING
+====
+---- QUERY
+set utf8_mode=true;
+select mask('Greek αβξδ άέήώ ΑΒΞΔ ΆΈΉΏ 1234');
+---- RESULTS: RAW_STRING
+'Xxxxx xxxx xxxx XXXX XXXX nnnn'
+---- TYPES
+STRING
+====
+---- QUERY
+set utf8_mode=true;
+select mask_first_n('áéíöóőüúű');
+---- RESULTS: RAW_STRING
+'xxxxóőüúű'
+---- TYPES
+STRING
+====
+---- QUERY
+set utf8_mode=true;
+select mask_show_first_n('áéíöóőüúű');
+---- RESULTS: RAW_STRING
+'áéíöxxxxx'
+---- TYPES
+STRING
+====
+---- QUERY
+set utf8_mode=true;
+select mask_last_n('áéíöóőüúű');
+---- RESULTS: RAW_STRING
+'áéíöóxxxx'
+---- TYPES
+STRING
+====
+---- QUERY
+set utf8_mode=true;
+select mask_show_last_n('áéíöóőüúű')
+---- RESULTS: RAW_STRING
+'xxxxxőüúű'
+---- TYPES
+STRING
+====
+---- QUERY
set utf8_mode=false;
select upper('abcd áäèü'), lower('ABCD ÁÄÈÜ'), initcap('abcd áäèü ABCD ÁÄÈÜ');
---- RESULTS: RAW_STRING
[impala] 02/04: IMPALA-11610: Pass environment variables into dockerized-impala-run-tests.sh
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 72812c5955a23c9270008e8371d8585d5165a27d
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Fri Sep 30 20:39:10 2022 -0700
IMPALA-11610: Pass environment variables into dockerized-impala-run-tests.sh
Because dockerized-impala-bootstrap-test.sh does a relogin while
calling dockerized-impala-run-tests.sh, the environment is not
preserved.
This adds a script dockerized-impala-preserve-vars.py that takes
a list of environment variables to preserve and appends
export statements to bin/impala-config-local.sh. Since
dockerized-impala-run-tests.sh sources bin/impala-config.sh, these
variables will be carried into the test execution.
This starts by adding environment variables used by upstream
Jenkin's ubuntu-16.04-dockerized-tests. Jenkins jobs can also
call dockerized-impala-preserve-vars.py directly.
Testing:
- Hand tested the preservation script
- Verified ubuntu-16.04-dockerized-tests now respected EE_TEST
argument.
Change-Id: I325217c731883c087c724194b45d50b790c7c280
Reviewed-on: http://gerrit.cloudera.org:8080/19088
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Joe McDonnell <jo...@cloudera.com>
---
.../dockerized-impala-bootstrap-and-test.sh | 7 +++
bin/jenkins/dockerized-impala-preserve-vars.py | 56 ++++++++++++++++++++++
2 files changed, 63 insertions(+)
diff --git a/bin/jenkins/dockerized-impala-bootstrap-and-test.sh b/bin/jenkins/dockerized-impala-bootstrap-and-test.sh
index e2171da3a..7317ba93c 100755
--- a/bin/jenkins/dockerized-impala-bootstrap-and-test.sh
+++ b/bin/jenkins/dockerized-impala-bootstrap-and-test.sh
@@ -30,6 +30,13 @@ source ./bin/bootstrap_system.sh
# Install docker
./bin/jenkins/install_docker.sh
+# Preserve some important environment variables so that they are available to
+# dockerized-impala-run-tests.sh.
+# NOTE: A Jenkins job can also call dockerized-impala-preserve-vars.py directly
+# to preserve additional variables.
+./bin/jenkins/dockerized-impala-preserve-vars.py \
+ EE_TEST EE_TEST_FILES JDBC_TEST EXPLORATION_STRATEGY CMAKE_BUILD_TYPE
+
# Execute the tests using su to re-login so that group change made above
# setup_docker takes effect. This does a full re-login and does not stay
# in the current directory, so change back to $IMPALA_HOME (resolved in
diff --git a/bin/jenkins/dockerized-impala-preserve-vars.py b/bin/jenkins/dockerized-impala-preserve-vars.py
new file mode 100755
index 000000000..f36a45dab
--- /dev/null
+++ b/bin/jenkins/dockerized-impala-preserve-vars.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Since dockerized-impala-bootstrap-and-test.sh does a full re-login
+# as part of calling dockerized-impala-run-tests.sh, it loses
+# environment variables. Preserving the environment variables are important
+# for parameterized Jenkins jobs. This script takes a list of environment
+# variables and preserves them by adding export statements to
+# bin/impala-config-local.sh.
+#
+# Usage: dockerized-impala-preserve-vars.py [env var1] [env var2] ...
+# If an environment variable is not defined in the current environment,
+# it is omitted with a warning.
+
+import sys
+import os
+
+
+def main():
+ if len(sys.argv) <= 1:
+ print("Usage: {0} [env vars]".format(sys.argv[0]))
+ sys.exit(1)
+
+ if "IMPALA_HOME" not in os.environ:
+ print("ERROR: IMPALA_HOME must be defined")
+ sys.exit(1)
+
+ impala_home = os.environ["IMPALA_HOME"]
+ # Append to the end of bin/impala-config-local.sh
+ with open("{0}/bin/impala-config-local.sh".format(impala_home), "a") as f:
+ for env_var in sys.argv[1:]:
+ if env_var not in os.environ:
+ print("{0} is not defined in the environment, skipping...".format(env_var))
+ continue
+ new_export = "export {0}=\"{1}\"".format(env_var, os.environ[env_var])
+ print("Adding '{0}' to bin/impala-config-local.sh".format(new_export))
+ f.write("{0}\n".format(new_export))
+
+
+if __name__ == "__main__": main()
[impala] 01/04: IMPALA-8770: Support building Docker images on Redhat-based distributions
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 3962ae1972e9e2d6592cdd5e305c021cf38c41bd
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Sat May 21 18:53:27 2022 -0700
IMPALA-8770: Support building Docker images on Redhat-based distributions
Currently, Impala supports building and testing Docker
images on Ubuntu. This extends that same support to
Redhat-based distributions:
1. This splits out the Docker build's OS package
installation into a separate install_os_packages.sh
script. This script detects the OS and calls apt
or yum as appropriate. The script takes the argument
--install-debug-tools, which installs extra tools
like iproute2 and ping. This defaults to true for debug
images and false for release images.
2. This modifies daemon_entrypoint.sh to detect the
OS and set LD_LIBRARY_PATH appropriate to account
for different locations of Java.
3. This modifies docker/setup_build_context.py to
handle different locations of libkudu_client.so
and add extra sanity checks on various libraries
found via globs.
4. This modifies bin/jenkins/dockerized-*.sh test
infrastructure to be able to install docker on
either Ubuntu or Redhat. It also changes the exit
logic to collect the container logs.
Developers can override the base image for Redhat 7
and Redhat 8 builds via the IMPALA_REDHAT7_DOCKER_BASE
and IMPALA_REDHAT8_DOCKER_BASE environment variables.
These default to open source Redhat equivalents
(Centos 7.9 and Rocky 8.5 respectively), but they are
also known to work with Redhat UBI images.
Testing:
- Ran dockerised testing on Rocky 8.5 via the
rocky-8.5-dockerised-tests job.
- Ran GVO
- Ran a Docker build on Centos7 with UBI7 as the base image
Change-Id: Ibaff2560ef971ac2c2231a8e43921164ea1d2f4d
Reviewed-on: http://gerrit.cloudera.org:8080/19006
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Joe McDonnell <jo...@cloudera.com>
---
bin/impala-config.sh | 10 ++
.../dockerized-impala-bootstrap-and-test.sh | 24 +---
bin/jenkins/dockerized-impala-run-tests.sh | 11 +-
bin/jenkins/install_docker.sh | 90 +++++++++++++++
docker/CMakeLists.txt | 126 +++++++++++++--------
docker/admissiond/Dockerfile | 6 +
docker/catalogd/Dockerfile | 6 +
docker/daemon_entrypoint.sh | 53 ++++++++-
docker/impala_base/Dockerfile | 16 +--
docker/impala_profile_tool/Dockerfile | 16 +--
docker/impalad_coord_exec/Dockerfile | 6 +
docker/impalad_coordinator/Dockerfile | 6 +
docker/impalad_executor/Dockerfile | 6 +
docker/install_os_packages.sh | 124 ++++++++++++++++++++
docker/setup_build_context.py | 65 +++++++++--
docker/statestored/Dockerfile | 5 +
16 files changed, 477 insertions(+), 93 deletions(-)
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 92b915c6c..fd705d84d 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -226,6 +226,16 @@ export IMPALA_KITE_VERSION=1.1.0
export IMPALA_ORC_JAVA_VERSION=1.7.6
export IMPALA_COS_VERSION=3.1.0-5.9.3
+# When Impala is building docker images on Redhat-based distributions,
+# it is useful to be able to customize the base image. Some users will
+# want to use open source / free distributions like Centos/Rocky/Alma/etc.
+# Some users will want to produce images on top of official Redhat UBI
+# images (which have certain guarantees about maintenance, CVEs, etc).
+# These environment variables control the base images. They default to
+# free distributions, but Redhat UBI images are known to work.
+export IMPALA_REDHAT7_DOCKER_BASE=${IMPALA_REDHAT7_DOCKER_BASE:-"centos:centos7.9.2009"}
+export IMPALA_REDHAT8_DOCKER_BASE=${IMPALA_REDHAT8_DOCKER_BASE:-"rockylinux:8.5"}
+
# When IMPALA_(CDP_COMPONENT)_URL are overridden, they may contain '$(platform_label)'
# which will be substituted for the CDP platform label in bootstrap_toolchain.py
unset IMPALA_HADOOP_URL
diff --git a/bin/jenkins/dockerized-impala-bootstrap-and-test.sh b/bin/jenkins/dockerized-impala-bootstrap-and-test.sh
index 245489452..e2171da3a 100755
--- a/bin/jenkins/dockerized-impala-bootstrap-and-test.sh
+++ b/bin/jenkins/dockerized-impala-bootstrap-and-test.sh
@@ -27,23 +27,11 @@ cd "$ROOT_DIR"
source ./bin/bootstrap_system.sh
-# Following install instructions from
-# https://docs.docker.com/install/linux/docker-ce/ubuntu/#install-docker-ce-1
-sudo apt-get install -y apt-transport-https ca-certificates curl \
- gnupg-agent software-properties-common
-curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-# Bail if the fingerprint isn't what we expected.
-sudo apt-key fingerprint 0EBFCD88 | \
- grep '9DC8 5822 9FC7 DD38 854A E2D8 8D81 803C 0EBF CD88'
-sudo add-apt-repository \
- "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
- $(lsb_release -cs) stable"
-sudo apt-get update
-sudo apt-get install -y docker-ce docker-ce-cli containerd.io
-sudo service docker restart
-sudo groupadd -f docker
-sudo usermod -aG docker $USER
+# Install docker
+./bin/jenkins/install_docker.sh
# Execute the tests using su to re-login so that group change made above
-# setup_docker takes effect.
-sudo su $USER -c "./bin/jenkins/dockerized-impala-run-tests.sh"
+# setup_docker takes effect. This does a full re-login and does not stay
+# in the current directory, so change back to $IMPALA_HOME (resolved in
+# the current environment) before executing the script.
+sudo su - $USER -c "cd ${IMPALA_HOME} && ./bin/jenkins/dockerized-impala-run-tests.sh"
diff --git a/bin/jenkins/dockerized-impala-run-tests.sh b/bin/jenkins/dockerized-impala-run-tests.sh
index ce02c2ef4..9b6ea001a 100755
--- a/bin/jenkins/dockerized-impala-run-tests.sh
+++ b/bin/jenkins/dockerized-impala-run-tests.sh
@@ -32,7 +32,16 @@ source_impala_config() {
set -u
}
+source_impala_config
+
onexit() {
+ # Get the logs from all docker containers
+ DOCKER_LOGS_DIR="${IMPALA_HOME}/logs/docker_logs"
+ mkdir -p "${DOCKER_LOGS_DIR}"
+ for container in $(docker ps -a -q); do
+ docker logs ${container} > "${DOCKER_LOGS_DIR}/${container}.log" 2>&1 || true
+ done
+
# Clean up docker containers and networks that may have been created by
# these tests.
docker rm -f $(docker ps -a -q) || true
@@ -40,8 +49,6 @@ onexit() {
}
trap onexit EXIT
-source_impala_config
-
# Check that docker is running and that our user can interact with it.
docker run hello-world
diff --git a/bin/jenkins/install_docker.sh b/bin/jenkins/install_docker.sh
new file mode 100755
index 000000000..261663822
--- /dev/null
+++ b/bin/jenkins/install_docker.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# This script installs and starts Docker. This currently supports
+# Ubuntu and Redhat distributions. It does the following:
+# 1. Adds the appropriate repository that has Docker
+# 2. Installs Docker from that repository
+# 3. Starts the Docker service
+# 4. Adds the current user to the docker group
+#
+# After this runs, the user will still need to relogin to detect
+# membership in the docker group.
+
+set -euo pipefail
+
+# This can get more detailed if there are specific steps
+# for specific versions, but at the moment the distribution
+# is all we need.
+DISTRIBUTION=Unknown
+if [[ -f /etc/redhat-release ]]; then
+ echo "Identified Redhat system."
+ DISTRIBUTION=Redhat
+else
+ source /etc/lsb-release
+ if [[ $DISTRIB_ID == Ubuntu ]]; then
+ echo "Identified Ubuntu system."
+ DISTRIBUTION=Ubuntu
+ fi
+fi
+
+if [[ $DISTRIBUTION == Unknown ]]; then
+ echo "ERROR: Did not detect supported distribution."
+ echo "Only Ubuntu and Redhat-based distributions are supported."
+ exit 1
+fi
+
+if [[ $DISTRIBUTION == Ubuntu ]]; then
+ # Following install instructions from
+ # https://docs.docker.com/engine/install/ubuntu/
+ # TODO: These instructions are from an old version of that page, and we should
+ # look into updating them.
+ export DEBIAN_FRONTEND=noninteractive
+ sudo apt-get install -y apt-transport-https ca-certificates curl \
+ gnupg-agent software-properties-common
+ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+ # Bail if the fingerprint isn't what we expected.
+ sudo apt-key fingerprint 0EBFCD88 | \
+ grep '9DC8 5822 9FC7 DD38 854A E2D8 8D81 803C 0EBF CD88'
+ sudo add-apt-repository \
+ "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+ $(lsb_release -cs) stable"
+ sudo apt-get update
+ sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+ sudo service docker restart
+elif [[ $DISTRIBUTION == Redhat ]]; then
+ # Following install instructions from
+ # https://docs.docker.com/engine/install/centos/
+ sudo yum install -y yum-utils
+ sudo yum-config-manager \
+ --add-repo \
+ https://download.docker.com/linux/centos/docker-ce.repo
+ # Go get the key and verify it is the expected value.
+ curl -fsSL https://download.docker.com/linux/centos/gpg > docker_key.gpg
+ gpg --import docker_key.gpg
+ # Bail if the fingerprint isn't what we expected
+ gpg --fingerprint "Docker Release (CE rpm) <do...@docker.com>" | \
+ grep '060A 61C5 1B55 8A7F 742B 77AA C52F EB6B 621E 9F35'
+ sudo rpmkeys --import docker_key.gpg
+ sudo yum -y install docker-ce docker-ce-cli containerd.io docker-compose-plugin
+ sudo systemctl start docker
+fi
+
+# Add the current user to the docker group
+sudo groupadd -f docker
+sudo usermod -aG docker $USER
diff --git a/docker/CMakeLists.txt b/docker/CMakeLists.txt
index ad443e3f2..4692a4830 100644
--- a/docker/CMakeLists.txt
+++ b/docker/CMakeLists.txt
@@ -35,17 +35,34 @@ execute_process(COMMAND ${LSB_RELEASE_EXEC} -rs
OUTPUT_STRIP_TRAILING_WHITESPACE
)
-if(${LSB_RELEASE_ID} STREQUAL "Ubuntu" AND ${LSB_RELEASE_VERSION} STREQUAL "16.04")
- set(DISTRO_BASE_IMAGE "ubuntu:16.04")
- set(PIP "python-pip")
-elseif(${LSB_RELEASE_ID} STREQUAL "Ubuntu" AND ${LSB_RELEASE_VERSION} STREQUAL "18.04")
- set(DISTRO_BASE_IMAGE "ubuntu:18.04")
- set(PIP "python-pip")
-elseif(${LSB_RELEASE_ID} STREQUAL "Ubuntu" AND ${LSB_RELEASE_VERSION} STREQUAL "20.04")
- set(DISTRO_BASE_IMAGE "ubuntu:20.04")
- set(PIP "python3-pip")
-else()
- set(DISTRO_BASE_IMAGE "UNSUPPORTED")
+set(QUICKSTART_BASE_IMAGE "UNSUPPORTED")
+set(DISTRO_BASE_IMAGE "UNSUPPORTED")
+
+if(${LSB_RELEASE_ID} STREQUAL "Ubuntu")
+ if(${LSB_RELEASE_VERSION} STREQUAL "16.04" OR
+ ${LSB_RELEASE_VERSION} STREQUAL "18.04" OR
+ ${LSB_RELEASE_VERSION} STREQUAL "20.04")
+ set(DISTRO_BASE_IMAGE "ubuntu:${LSB_RELEASE_VERSION}")
+ set(QUICKSTART_BASE_IMAGE "ubuntu:${LSB_RELEASE_VERSION}")
+ endif()
+ if (${LSB_RELEASE_VERSION} STREQUAL "16.04" OR
+ ${LSB_RELEASE_VERSION} STREQUAL "18.04")
+ set(PIP "python-pip")
+ elseif (${LSB_RELEASE_VERSION} STREQUAL "20.04")
+ set(PIP "python3-pip")
+ endif()
+elseif(${LSB_RELEASE_ID} STREQUAL "RedHatEnterpriseServer" OR
+ ${LSB_RELEASE_ID} STREQUAL "RedHatEnterprise" OR
+ ${LSB_RELEASE_ID} STREQUAL "Rocky" OR
+ ${LSB_RELEASE_ID} STREQUAL "AlmaLinux" OR
+ ${LSB_RELEASE_ID} STREQUAL "CentOS")
+ # The Quickstart images currently don't support using a Redhat
+ # base image, so this doesn't set QUICKSTART_BASE_IMAGE.
+ if(${LSB_RELEASE_VERSION} MATCHES "7.*")
+ set(DISTRO_BASE_IMAGE "$ENV{IMPALA_REDHAT7_DOCKER_BASE}")
+ elseif(${LSB_RELEASE_VERSION} MATCHES "8.*")
+ set(DISTRO_BASE_IMAGE "$ENV{IMPALA_REDHAT8_DOCKER_BASE}")
+ endif()
endif()
MESSAGE(STATUS "Picked docker base image based on host OS: ${DISTRO_BASE_IMAGE}")
@@ -64,6 +81,13 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
COMMENT "Creating impala base build context build_type=${build_type}."
VERBATIM
)
+ # The base image Dockerfile takes an argument INSTALL_OS_PACKAGES_ARGS to specify
+ # arguments for install_os_packages.sh. This can can be set to "--install-debug-tools"
+ # to add extra utilities. Enable this for debug builds.
+ set(INSTALL_OS_PACKAGES_ARGS "")
+ if (${build_type} STREQUAL "debug")
+ set(INSTALL_OS_PACKAGES_ARGS "--install-debug-tools")
+ endif()
# Target for the base Impala image.
add_custom_target(impala_base_image_${build_type}
# Run docker build inside the build context directory so that all dependencies are
@@ -71,11 +95,13 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
# dependencies.
COMMAND tar cvh . -C ${CMAKE_SOURCE_DIR}/docker/impala_base/ . |
${DOCKER_BUILD} -t impala_base_${build_type}
- --build-arg BASE_IMAGE=${DISTRO_BASE_IMAGE} -
+ --build-arg BASE_IMAGE=${DISTRO_BASE_IMAGE}
+ --build-arg INSTALL_OS_PACKAGES_ARGS=${INSTALL_OS_PACKAGES_ARGS} -
WORKING_DIRECTORY ${IMPALA_BASE_BUILD_CONTEXT_DIR}/${build_type}
DEPENDS impala_base_build_context_${build_type} ${CMAKE_SOURCE_DIR}/docker/impala_base/Dockerfile
DEPENDS ${CMAKE_SOURCE_DIR}/docker/daemon_entrypoint.sh
DEPENDS ${CMAKE_SOURCE_DIR}/bin/graceful_shutdown_backends.sh
+ DEPENDS ${CMAKE_SOURCE_DIR}/docker/install_os_packages.sh
COMMENT "Building Impala base docker image build_type=${build_type}."
VERBATIM
)
@@ -134,40 +160,45 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
add_daemon_docker_images(statestored)
add_daemon_docker_images(admissiond)
- # HMS quickstart image, which requires Hive and Hadoop builds.
- set(QUICKSTART_HMS_IMAGE impala_quickstart_hms)
- set(quickstart_hms_build_dir ${CMAKE_SOURCE_DIR}/docker/quickstart_hms)
- add_custom_target(quickstart_hms_build_setup
- COMMAND rm -f ${quickstart_hms_build_dir}/hive ${quickstart_hms_build_dir}/hadoop
- COMMAND ${CMAKE_COMMAND} -E create_symlink $ENV{HIVE_HOME} ${quickstart_hms_build_dir}/hive
- COMMAND ${CMAKE_COMMAND} -E create_symlink $ENV{HADOOP_HOME} ${quickstart_hms_build_dir}/hadoop
- )
- add_custom_target(quickstart_hms_image
- # Supply the appropriate base image as an argument for the Dockerfile.
- # Use tar with -h flag to assemble a tarball including all the symlinked files and
- # directories in the build context.
- COMMAND tar cvh . -C ${quickstart_hms_build_dir} . | ${DOCKER_BUILD} --build-arg BASE_IMAGE=${DISTRO_BASE_IMAGE} -t ${QUICKSTART_HMS_IMAGE} -
- DEPENDS ${quickstart_hms_build_dir}/Dockerfile quickstart_hms_build_setup
- COMMENT "Building quickstart HMS docker image."
- VERBATIM
- )
- ADD_DEPENDENCIES(quickstart_docker_images quickstart_hms_image)
- set(exported_image_names "${exported_image_names} ${QUICKSTART_HMS_IMAGE}")
-
- # Client quickstart image, which only requires some scripts.
- set(QUICKSTART_CLIENT_IMAGE impala_quickstart_client)
- set(quickstart_client_build_dir ${CMAKE_SOURCE_DIR}/docker/quickstart_client)
- add_custom_target(quickstart_client_image
- # Supply the appropriate base image as an argument for the Dockerfile.
- # Use tar with -h flag to assemble a tarball including all the symlinked files and
- # directories in the build context.
- COMMAND tar cvh . -C ${quickstart_client_build_dir} . | ${DOCKER_BUILD} ${COMMON_DOCKER_BUILD_ARGS} --build-arg BASE_IMAGE=${DISTRO_BASE_IMAGE} --build-arg PIP=${PIP} -t ${QUICKSTART_CLIENT_IMAGE} -
- DEPENDS ${quickstart_client_build_dir}/Dockerfile ${quickstart_client_build_dir}/data-load-entrypoint.sh
- COMMENT "Building quickstart client docker image."
- VERBATIM
- )
- ADD_DEPENDENCIES(quickstart_docker_images quickstart_client_image)
- set(exported_image_names "${exported_image_names} ${QUICKSTART_CLIENT_IMAGE}")
+ # Quickstart is only supported on Ubuntu, so skip generating the Quickstart targets
+ # on other distributions.
+ # TODO: Support Quickstart images on Redhat
+ if (NOT ${QUICKSTART_BASE_IMAGE} STREQUAL "UNSUPPORTED")
+ # HMS quickstart image, which requires Hive and Hadoop builds.
+ set(QUICKSTART_HMS_IMAGE impala_quickstart_hms)
+ set(quickstart_hms_build_dir ${CMAKE_SOURCE_DIR}/docker/quickstart_hms)
+ add_custom_target(quickstart_hms_build_setup
+ COMMAND rm -f ${quickstart_hms_build_dir}/hive ${quickstart_hms_build_dir}/hadoop
+ COMMAND ${CMAKE_COMMAND} -E create_symlink $ENV{HIVE_HOME} ${quickstart_hms_build_dir}/hive
+ COMMAND ${CMAKE_COMMAND} -E create_symlink $ENV{HADOOP_HOME} ${quickstart_hms_build_dir}/hadoop
+ )
+ add_custom_target(quickstart_hms_image
+ # Supply the appropriate base image as an argument for the Dockerfile.
+ # Use tar with -h flag to assemble a tarball including all the symlinked files and
+ # directories in the build context.
+ COMMAND tar cvh . -C ${quickstart_hms_build_dir} . | ${DOCKER_BUILD} --build-arg BASE_IMAGE=${QUICKSTART_BASE_IMAGE} -t ${QUICKSTART_HMS_IMAGE} -
+ DEPENDS ${quickstart_hms_build_dir}/Dockerfile quickstart_hms_build_setup
+ COMMENT "Building quickstart HMS docker image."
+ VERBATIM
+ )
+ ADD_DEPENDENCIES(quickstart_docker_images quickstart_hms_image)
+ set(exported_image_names "${exported_image_names} ${QUICKSTART_HMS_IMAGE}")
+
+ # Client quickstart image, which only requires some scripts.
+ set(QUICKSTART_CLIENT_IMAGE impala_quickstart_client)
+ set(quickstart_client_build_dir ${CMAKE_SOURCE_DIR}/docker/quickstart_client)
+ add_custom_target(quickstart_client_image
+ # Supply the appropriate base image as an argument for the Dockerfile.
+ # Use tar with -h flag to assemble a tarball including all the symlinked files and
+ # directories in the build context.
+ COMMAND tar cvh . -C ${quickstart_client_build_dir} . | ${DOCKER_BUILD} ${COMMON_DOCKER_BUILD_ARGS} --build-arg BASE_IMAGE=${QUICKSTART_BASE_IMAGE} --build-arg PIP=${PIP} -t ${QUICKSTART_CLIENT_IMAGE} -
+ DEPENDS ${quickstart_client_build_dir}/Dockerfile ${quickstart_client_build_dir}/data-load-entrypoint.sh
+ COMMENT "Building quickstart client docker image."
+ VERBATIM
+ )
+ ADD_DEPENDENCIES(quickstart_docker_images quickstart_client_image)
+ set(exported_image_names "${exported_image_names} ${QUICKSTART_CLIENT_IMAGE}")
+ endif()
# Add a target to build utility docker images for 'build_type'. 'build_context_args' are
# passed to the setup_build_context.py script.
@@ -187,12 +218,15 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
# Run docker build inside the build context directory so that all dependencies are
# sent to the docker daemon. This allows the Dockerfile build to copy all necessary
# dependencies.
+ # Note: This currently does not specify INSTALL_OS_PACKAGES_ARGS, so it uses the
+ # default value, which installs extra debugging tools.
COMMAND tar cvh . -C ${CMAKE_SOURCE_DIR}/docker/impala_profile_tool/ . |
${DOCKER_BUILD} -t ${profile_tool_image}
--build-arg BASE_IMAGE=${DISTRO_BASE_IMAGE} -
WORKING_DIRECTORY ${IMPALA_UTILITY_BUILD_CONTEXT_DIR}/${build_type}
DEPENDS impala_utility_build_context_${build_type} ${CMAKE_SOURCE_DIR}/docker/impala_profile_tool/Dockerfile
DEPENDS ${CMAKE_SOURCE_DIR}/docker/utility_entrypoint.sh
+ DEPENDS ${CMAKE_SOURCE_DIR}/docker/install_os_packages.sh
COMMENT "Building Impala profile tool docker image build_type=${build_type}."
VERBATIM
)
diff --git a/docker/admissiond/Dockerfile b/docker/admissiond/Dockerfile
index c2135a0f0..bafd8e71b 100644
--- a/docker/admissiond/Dockerfile
+++ b/docker/admissiond/Dockerfile
@@ -23,6 +23,12 @@ FROM ${BASE_IMAGE}
# size would be to only copy in jars required by admissiond.
COPY --chown=impala lib /opt/impala/lib
+# Run the daemon_entrypoint.sh script as a sanity check. This verifies that
+# daemon_entrypoint.sh is detecting Java appropriately and constructing a
+# LD_LIBRARY_PATH that allows admissiond to start. The --version flag exits
+# before starting the JVM, so that is the limit of this check.
+RUN /opt/impala/bin/daemon_entrypoint.sh /opt/impala/bin/admissiond --version
+
# Externally-facing ports
# Debug webserver
EXPOSE 25030
diff --git a/docker/catalogd/Dockerfile b/docker/catalogd/Dockerfile
index 3babbece9..d6bfefbbf 100644
--- a/docker/catalogd/Dockerfile
+++ b/docker/catalogd/Dockerfile
@@ -23,6 +23,12 @@ FROM ${BASE_IMAGE}
# size would be to only copy in jars required by catalogd.
COPY --chown=impala lib /opt/impala/lib
+# Run the daemon_entrypoint.sh script as a sanity check. This verifies that
+# daemon_entrypoint.sh is detecting Java appropriately and constructing a
+# LD_LIBRARY_PATH that allows catalogd to start. The --version flag exits
+# before starting the JVM, so that is the limit of this check.
+RUN /opt/impala/bin/daemon_entrypoint.sh /opt/impala/bin/catalogd --version
+
# Externally-facing ports
# Debug webserver
EXPOSE 25020
diff --git a/docker/daemon_entrypoint.sh b/docker/daemon_entrypoint.sh
index e670bd5a9..6cc7bd89e 100755
--- a/docker/daemon_entrypoint.sh
+++ b/docker/daemon_entrypoint.sh
@@ -28,8 +28,57 @@ export IMPALA_HOME=/opt/impala
# Add directories containing dynamic libraries required by the daemons that
# are not on the system library paths.
export LD_LIBRARY_PATH=/opt/impala/lib
-LD_LIBRARY_PATH+=:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/
-LD_LIBRARY_PATH+=:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/
+
+# This can get more detailed if there are specific steps
+# for specific versions, but at the moment the distribution
+# is all we need.
+DISTRIBUTION=Unknown
+if [[ -f /etc/redhat-release ]]; then
+ echo "Identified Redhat image."
+ DISTRIBUTION=Redhat
+else
+ source /etc/lsb-release
+ if [[ $DISTRIB_ID == Ubuntu ]]; then
+ echo "Identified Ubuntu image."
+ DISTRIBUTION=Ubuntu
+ fi
+fi
+
+if [[ $DISTRIBUTION == Unknown ]]; then
+ echo "ERROR: Did not detect supported distribution."
+ echo "Only Ubuntu and Redhat-based distributions are supported."
+ exit 1
+fi
+
+# Need libjvm.so and libjsig.so on LD_LIBRARY_PATH
+# Ubuntu and Redhat use different locations for JAVA_HOME.
+JAVA_HOME=Unknown
+if [[ $DISTRIBUTION == Ubuntu ]]; then
+ JAVA_HOME=$(compgen -G /usr/lib/jvm/java-8-openjdk-*)
+elif [[ $DISTRIBUTION == Redhat ]]; then
+ JAVA_HOME=/usr/lib/jvm/jre-1.8.0
+fi
+
+if [[ $JAVA_HOME == Unknown ]]; then
+ echo "ERROR: Did not find Java in any expected location."
+ echo "Only Java 8 and Java 11 are supported."
+ exit 1
+fi
+
+echo "JAVA_HOME: ${JAVA_HOME}"
+# Given JAVA_HOME, find libjsig.so and libjvm.so and add them to LD_LIBRARY_PATH.
+# JAVA_HOME could be a symlink, so follow symlinks when looking for the libraries
+LIB_JSIG_DIR=$(find -L "${JAVA_HOME}" -name libjsig.so | head -1 | xargs dirname)
+if [[ -z $LIB_JSIG_DIR ]]; then
+ echo "ERROR: Could not find libjsig.so in ${JAVA_HOME}"
+ exit 1
+fi
+LIB_JVM_DIR=$(find -L "${JAVA_HOME}" -name libjvm.so |head -1 | xargs dirname)
+if [[ -z $LIB_JVM_DIR ]]; then
+ echo "ERROR: Could not find libjvm.so in ${JAVA_HOME}"
+ exit 1
+fi
+LD_LIBRARY_PATH+=:${LIB_JSIG_DIR}:${LIB_JVM_DIR}
# Add directory with optional plugins that can be mounted for the container.
LD_LIBRARY_PATH+=:/opt/impala/lib/plugins
diff --git a/docker/impala_base/Dockerfile b/docker/impala_base/Dockerfile
index d15af90d9..7b1b5963c 100644
--- a/docker/impala_base/Dockerfile
+++ b/docker/impala_base/Dockerfile
@@ -14,17 +14,17 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-ARG BASE_IMAGE=ubuntu:16.04
+
+# This uses an invalid BASE_IMAGE, because this needs to be overridden.
+ARG BASE_IMAGE=REPLACED_WITH_BASE_IMAGE
FROM ${BASE_IMAGE}
+# If set to "--install-debug-tools", then extra utilities will be installed.
+ARG INSTALL_OS_PACKAGES_ARGS=""
+
# Install minimal dependencies required for Impala services to run.
-RUN apt-get update && \
- apt-get install -y openjdk-8-jre-headless \
- libsasl2-2 libsasl2-modules libsasl2-modules-gssapi-mit \
- sudo netcat-openbsd less curl iproute2 vim iputils-ping \
- tzdata krb5-user && \
- apt-get clean && \
- rm -rf /var/lib/apt/lists/*
+ADD helper/install_os_packages.sh /root
+RUN /root/install_os_packages.sh ${INSTALL_OS_PACKAGES_ARGS}
# Use a non-privileged impala user to run the daemons in the container.
# That user should own everything in the /opt/impala subdirectory.
diff --git a/docker/impala_profile_tool/Dockerfile b/docker/impala_profile_tool/Dockerfile
index 3a55b8503..d6f21c194 100644
--- a/docker/impala_profile_tool/Dockerfile
+++ b/docker/impala_profile_tool/Dockerfile
@@ -14,20 +14,20 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-ARG BASE_IMAGE=ubuntu:16.04
+
+# This uses an invalid BASE_IMAGE, because this needs to be overridden.
+ARG BASE_IMAGE=REPLACED_WITH_BASE_IMAGE
FROM ${BASE_IMAGE}
+# If set to "--install-debug-tools", then extra utilities will be installed
+ARG INSTALL_OS_PACKAGES_ARGS="--install-debug-tools"
+
# Install dependencies required for Impala utility binaries to run, plus
# some useful utilities.
# TODO: ideally we wouldn't depend on the JVM libraries, but currently the JNI code
# in be/ is not cleanly separated from the code that doesn't use JNI.
-RUN apt-get update && \
- apt-get install -y openjdk-8-jre-headless \
- libsasl2-2 libsasl2-modules libsasl2-modules-gssapi-mit \
- sudo netcat-openbsd less curl iproute2 vim iputils-ping \
- krb5-user && \
- apt-get clean && \
- rm -rf /var/lib/apt/lists/*
+ADD helper/install_os_packages.sh /root
+RUN /root/install_os_packages.sh ${INSTALL_OS_PACKAGES_ARGS}
# Use a non-privileged impala user to run the processes in the container.
# That user should own everything in the /opt/impala subdirectory.
diff --git a/docker/impalad_coord_exec/Dockerfile b/docker/impalad_coord_exec/Dockerfile
index 1ba6c74b5..89de67b35 100644
--- a/docker/impalad_coord_exec/Dockerfile
+++ b/docker/impalad_coord_exec/Dockerfile
@@ -20,6 +20,12 @@ FROM ${BASE_IMAGE}
COPY --chown=impala lib /opt/impala/lib
+# Run the daemon_entrypoint.sh script as a sanity check. This verifies that
+# daemon_entrypoint.sh is detecting Java appropriately and constructing a
+# LD_LIBRARY_PATH that allows impalad to start. The --version flag exits
+# before starting the JVM, so that is the limit of this check.
+RUN /opt/impala/bin/daemon_entrypoint.sh /opt/impala/bin/impalad --version
+
# Externally-facing ports
# Beeswax
EXPOSE 21000
diff --git a/docker/impalad_coordinator/Dockerfile b/docker/impalad_coordinator/Dockerfile
index bf6de31ff..f00f0fe50 100644
--- a/docker/impalad_coordinator/Dockerfile
+++ b/docker/impalad_coordinator/Dockerfile
@@ -20,6 +20,12 @@ FROM ${BASE_IMAGE}
COPY --chown=impala lib /opt/impala/lib
+# Run the daemon_entrypoint.sh script as a sanity check. This verifies that
+# daemon_entrypoint.sh is detecting Java appropriately and constructing a
+# LD_LIBRARY_PATH that allows impalad to start. The --version flag exits
+# before starting the JVM, so that is the limit of this check.
+RUN /opt/impala/bin/daemon_entrypoint.sh /opt/impala/bin/impalad --version
+
# Externally-facing ports
# Beeswax
EXPOSE 21000
diff --git a/docker/impalad_executor/Dockerfile b/docker/impalad_executor/Dockerfile
index 74d82042c..6ac11e12a 100644
--- a/docker/impalad_executor/Dockerfile
+++ b/docker/impalad_executor/Dockerfile
@@ -23,6 +23,12 @@ FROM ${BASE_IMAGE}
# able to use it here.
COPY --chown=impala lib /opt/impala/lib
+# Run the daemon_entrypoint.sh script as a sanity check. This verifies that
+# daemon_entrypoint.sh is detecting Java appropriately and constructing a
+# LD_LIBRARY_PATH that allows impalad to start. The --version flag exits
+# before starting the JVM, so that is the limit of this check.
+RUN /opt/impala/bin/daemon_entrypoint.sh /opt/impala/bin/impalad --version
+
# Externally-facing ports
# Debug webserver
EXPOSE 25000
diff --git a/docker/install_os_packages.sh b/docker/install_os_packages.sh
new file mode 100755
index 000000000..db4348d1a
--- /dev/null
+++ b/docker/install_os_packages.sh
@@ -0,0 +1,124 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# This installs the minimal dependencies needed for Impala
+# services to run. It currently handles Ubuntu and Redhat
+# distributions. This takes an optional argument "--install-debug-tools"
+# which installs extra debugging tools like curl, ping, etc.
+
+set -euo pipefail
+
+INSTALL_DEBUG_TOOLS=false
+
+function print_usage {
+ echo "install_os_packages.sh - Helper script to install OS dependencies"
+ echo "[--install-debug-tools] : Also install debug tools like curl, iproute, etc"
+}
+
+while [ -n "$*" ]
+do
+ case "$1" in
+ --install-debug-tools)
+ INSTALL_DEBUG_TOOLS=true
+ ;;
+ --help|*)
+ print_usage
+ exit 1
+ ;;
+ esac
+ shift
+done
+
+echo "INSTALL_DEBUG_TOOLS=${INSTALL_DEBUG_TOOLS}"
+
+# This can get more detailed if there are specific steps
+# for specific versions, but at the moment the distribution
+# is all we need.
+DISTRIBUTION=Unknown
+if [[ -f /etc/redhat-release ]]; then
+ echo "Identified Redhat system."
+ DISTRIBUTION=Redhat
+else
+ source /etc/lsb-release
+ if [[ $DISTRIB_ID == Ubuntu ]]; then
+ echo "Identified Ubuntu system."
+ DISTRIBUTION=Ubuntu
+ fi
+fi
+
+if [[ $DISTRIBUTION == Unknown ]]; then
+ echo "ERROR: Did not detect supported distribution."
+ echo "Only Ubuntu and Redhat-based distributions are supported."
+ exit 1
+fi
+
+# Install minimal set of files.
+# Optionally install extra debug tools.
+if [[ $DISTRIBUTION == Ubuntu ]]; then
+ export DEBIAN_FRONTEND=noninteractive
+ apt-get update
+ apt-get install -y \
+ krb5-user \
+ libsasl2-2 \
+ libsasl2-modules \
+ libsasl2-modules-gssapi-mit \
+ openjdk-8-jre-headless \
+ tzdata
+ if $INSTALL_DEBUG_TOOLS ; then
+ echo "Installing extra debug tools"
+ apt-get install -y \
+ curl \
+ dnsutils \
+ iproute2 \
+ iputils-ping \
+ less \
+ netcat-openbsd \
+ sudo \
+ vim
+ fi
+elif [[ $DISTRIBUTION == Redhat ]]; then
+ yum install -y --disableplugin=subscription-manager \
+ cyrus-sasl-gssapi \
+ cyrus-sasl-plain \
+ java-1.8.0-openjdk-headless \
+ krb5-workstation \
+ openldap-devel \
+ tzdata
+ if $INSTALL_DEBUG_TOOLS ; then
+ echo "Installing extra debug tools"
+ yum install -y --disableplugin=subscription-manager \
+ bind-utils \
+ curl \
+ iproute \
+ iputils \
+ less \
+ nmap-ncat \
+ sudo \
+ vim \
+ which
+ fi
+fi
+
+# To minimize the size for the Docker image, clean up any unnecessary files.
+if [[ $DISTRIBUTION == Ubuntu ]]; then
+ apt-get clean
+ rm -rf /var/lib/apt/lists/*
+elif [[ $DISTRIBUTION == Redhat ]]; then
+ yum clean all
+ rm -rf /var/cache/yum/*
+fi
diff --git a/docker/setup_build_context.py b/docker/setup_build_context.py
index 037ed6518..cd2955d97 100755
--- a/docker/setup_build_context.py
+++ b/docker/setup_build_context.py
@@ -53,7 +53,22 @@ BINUTILS_HOME = os.path.join(
IMPALA_TOOLCHAIN_PACKAGES_HOME, "binutils-{0}".format(IMPALA_BINUTILS_VERSION))
STRIP = os.path.join(BINUTILS_HOME, "bin/strip")
KUDU_HOME = os.environ["IMPALA_KUDU_HOME"]
-KUDU_LIB_DIR = os.path.join(KUDU_HOME, "release/lib")
+KUDU_CLIENT_DIR = os.environ.get("KUDU_CLIENT_DIR")
+# Different distributions put Kudu libraries in different places.
+# Allow for libkudu_client.so to be in any of these locations:
+# - release/lib (used on Ubuntu)
+# - release/lib64 (used on Redhat)
+# - release/lib/exported
+#
+# Also, respect KUDU_CLIENT_DIR if it is set. With KUDU_CLIENT_DIR, the output
+# is under KUDU_CLIENT_DIR/usr/local, but also varies on "lib" vs "lib64"
+if KUDU_CLIENT_DIR:
+ kudu_lib_dirs = [os.path.join(KUDU_CLIENT_DIR, "usr/local/lib"),
+ os.path.join(KUDU_CLIENT_DIR, "usr/local/lib64")]
+else:
+ kudu_lib_dirs = [os.path.join(KUDU_HOME, "release/lib"),
+ os.path.join(KUDU_HOME, "release/lib64"),
+ os.path.join(KUDU_HOME, "release/lib/exported")]
# Ensure the output directory exists and is empty.
if os.path.exists(OUTPUT_DIR):
@@ -62,6 +77,9 @@ os.makedirs(OUTPUT_DIR)
BIN_DIR = os.path.join(OUTPUT_DIR, "bin")
+# Contains helper install scripts used during Docker build
+HELPER_DIR = os.path.join(OUTPUT_DIR, "helper")
+
# Contains all library dependencies for Impala Executors.
EXEC_LIB_DIR = os.path.join(OUTPUT_DIR, "exec-lib")
@@ -80,6 +98,7 @@ else:
TARGET_LIB_DIRS = [LIB_DIR, EXEC_LIB_DIR, STATESTORE_LIB_DIR]
os.mkdir(BIN_DIR)
+os.mkdir(HELPER_DIR)
for lib_dir in TARGET_LIB_DIRS:
os.mkdir(lib_dir)
@@ -122,40 +141,68 @@ else:
strip_debug_symbols(IMPALAD_BINARY, [BIN_DIR])
# Add libstc++ binaries to LIB_DIR. Strip debug symbols for release builds.
+found_libstdcpp_so = False
for libstdcpp_so in glob.glob(os.path.join(
GCC_HOME, "lib64/{0}*.so*".format("libstdc++"))):
# Ignore 'libstdc++.so.*-gdb.py'.
if not os.path.basename(libstdcpp_so).endswith(".py"):
+ found_libstdcpp_so = True
dst_dirs = TARGET_LIB_DIRS
if args.debug_build:
symlink_file_into_dirs(libstdcpp_so, dst_dirs)
else:
strip_debug_symbols(libstdcpp_so, dst_dirs)
+if not found_libstdcpp_so:
+ raise Exception("No libstdc++.so found in search path: {0}".format(
+ os.path.join(GCC_HOME, "lib64")))
+
# Add libgcc binaries to LIB_DIR.
+found_libgcc_so = False
for libgcc_so in glob.glob(os.path.join(GCC_HOME, "lib64/{0}*.so*".format("libgcc_s"))):
+ found_libgcc_so = True
symlink_file_into_dirs(libgcc_so, TARGET_LIB_DIRS)
+if not found_libgcc_so:
+ raise Exception("No libgcc.so found in search path: {0}".format(
+ os.path.join(GCC_HOME, "lib64")))
+
# Add libkudu_client binaries to LIB_DIR. Strip debug symbols for release builds.
-for kudu_client_so in glob.glob(os.path.join(KUDU_LIB_DIR, "libkudu_client.so*")):
- # All backend binaries currently link against libkudu_client.so even if they don't need
- # them.
- dst_dirs = TARGET_LIB_DIRS
- if args.debug_build:
- symlink_file_into_dirs(kudu_client_so, dst_dirs)
- else:
- strip_debug_symbols(kudu_client_so, dst_dirs)
+found_kudu_so = False
+for kudu_lib_dir in kudu_lib_dirs:
+ for kudu_client_so in glob.glob(os.path.join(kudu_lib_dir, "libkudu_client.so*")):
+ # All backend binaries currently link against libkudu_client.so even if they don't
+ # need them.
+ found_kudu_so = True
+ dst_dirs = TARGET_LIB_DIRS
+ if args.debug_build:
+ symlink_file_into_dirs(kudu_client_so, dst_dirs)
+ else:
+ strip_debug_symbols(kudu_client_so, dst_dirs)
+
+if not found_kudu_so:
+ raise Exception("No Kudu shared object found in search path: {0}".format(kudu_lib_dirs))
+
+# Add script for installing OS packages
+symlink_file_into_dir(
+ os.path.join(IMPALA_HOME, "docker/install_os_packages.sh"), HELPER_DIR)
if args.utility_context:
symlink_file_into_dir(
os.path.join(IMPALA_HOME, "docker/utility_entrypoint.sh"), BIN_DIR)
else:
# Impala Coordinator dependencies.
+ num_jars_on_classpath = 0
dep_classpath = file(os.path.join(IMPALA_HOME, "fe/target/build-classpath.txt")).read()
for jar in dep_classpath.split(":"):
+ num_jars_on_classpath += 1
assert os.path.exists(jar), "missing jar from classpath: {0}".format(jar)
symlink_file_into_dir(jar, LIB_DIR)
+ if num_jars_on_classpath == 0:
+ raise Exception("No jars listed in {0}".format(os.path.join(IMPALA_HOME,
+ "fe/target/build-classpath.txt")))
+
# Impala Coordinator jars.
num_frontend_jars = 0
for jar in glob.glob(os.path.join(IMPALA_HOME, "fe/target/impala-frontend-*.jar")):
diff --git a/docker/statestored/Dockerfile b/docker/statestored/Dockerfile
index 02e3c69e0..79ecd3bbc 100644
--- a/docker/statestored/Dockerfile
+++ b/docker/statestored/Dockerfile
@@ -21,6 +21,11 @@ FROM ${BASE_IMAGE}
# Only copy in dependencies required for the statestored.
COPY --chown=impala statestore-lib /opt/impala/lib
+# Run the daemon_entrypoint.sh script as a sanity check. This verifies that
+# daemon_entrypoint.sh is detecting Java appropriately and constructing a
+# LD_LIBRARY_PATH that allows statestored to start.
+RUN /opt/impala/bin/daemon_entrypoint.sh /opt/impala/bin/statestored --version
+
# Externally-facing ports
# Debug webserver
EXPOSE 25010