You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2023/05/19 22:35:07 UTC
[impala] 03/05: IMPALA-12081: Produce multiple Java docker images
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit c8a21c51efee2b0dd2f416c0c91853ecc6bbf96d
Author: Michael Smith <mi...@cloudera.com>
AuthorDate: Tue Apr 18 16:53:29 2023 -0700
IMPALA-12081: Produce multiple Java docker images
This changes the docker image build code so that both Java 8 and Java 11
images can be built in the same build. Specifically, it introduces new
Make targets for Java 11 docker images in addition to the regular Java 8
targets. The "docker_images" and "docker_debug_images" targets continue
to behave the same way and produce Java 8 images of the same name. The
"docker_java11_images" and "docker_debug_java11_images" produce the
daemon docker images for Java 11.
Preserves IMPALA_DOCKER_USE_JAVA11 for selecting Java 11 images when
starting a cluster with container images.
Change-Id: Ic2b124267c607242bc2fd6c8cd6486293a938f50
Reviewed-on: http://gerrit.cloudera.org:8080/19722
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
bin/impala-config.sh | 7 +++---
bin/jenkins/dockerized-impala-run-tests.sh | 6 ++++-
bin/start-impala-cluster.py | 2 ++
docker/CMakeLists.txt | 36 +++++++++++++++++-------------
docker/install_os_packages.sh | 28 +++++++++--------------
docker/setup_build_context.py | 11 +++------
6 files changed, 44 insertions(+), 46 deletions(-)
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 3f545f284..de57cc95d 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -266,10 +266,9 @@ export IMPALA_OBS_VERSION=3.1.1-hw-42
export IMPALA_REDHAT7_DOCKER_BASE=${IMPALA_REDHAT7_DOCKER_BASE:-"centos:centos7.9.2009"}
export IMPALA_REDHAT8_DOCKER_BASE=${IMPALA_REDHAT8_DOCKER_BASE:-"rockylinux:8.5"}
-# When set to true, this modifies the Docker image build logic to install Java 11
-# rather than the default of Java 8. This is strictly about what is shipped in the
-# Docker image, and it has no impact on what version of Java is used to compile
-# Impala's Java code.
+# When set to true, this instructs start-impala-cluster.py to start with Java 11 images
+# created via 'make docker_java11_images' (or 'docker_debug_java11_images'). The Java
+# version used in these images is independent of the Java version used to compile Impala.
export IMPALA_DOCKER_USE_JAVA11=${IMPALA_DOCKER_USE_JAVA11:-"false"}
# There are multiple compatible implementations of zlib. Cloudflare Zlib is an
diff --git a/bin/jenkins/dockerized-impala-run-tests.sh b/bin/jenkins/dockerized-impala-run-tests.sh
index 9b6ea001a..cb665b18f 100755
--- a/bin/jenkins/dockerized-impala-run-tests.sh
+++ b/bin/jenkins/dockerized-impala-run-tests.sh
@@ -80,7 +80,11 @@ start-impala-cluster.py --kill
# Build the docker images required to start the cluster.
# parquet-reader and impala-profile-tool are needed for e2e tests but not built for
# non-test build.
-make -j ${IMPALA_BUILD_THREADS} docker_debug_images parquet-reader impala-profile-tool
+IMAGE_TYPE=docker_debug
+if ${IMPALA_DOCKER_USE_JAVA11-false}; then
+ IMAGE_TYPE=${IMAGE_TYPE}_java11
+fi
+make -j ${IMPALA_BUILD_THREADS} ${IMAGE_TYPE}_images parquet-reader impala-profile-tool
source_impala_config
diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py
index 5a916dc46..88b886ab7 100755
--- a/bin/start-impala-cluster.py
+++ b/bin/start-impala-cluster.py
@@ -687,6 +687,8 @@ class DockerMiniClusterOperations(object):
image_tag = daemon + "_debug"
else:
image_tag = daemon
+ if os.getenv('IMPALA_DOCKER_USE_JAVA11', '') == "true":
+ image_tag += "_java11"
host_name = self.__gen_host_name__(daemon, instance)
container_name = self.__gen_container_name__(daemon, instance)
# Mount configuration into container so that we don't need to rebuild container
diff --git a/docker/CMakeLists.txt b/docker/CMakeLists.txt
index c94e97898..7f0455760 100644
--- a/docker/CMakeLists.txt
+++ b/docker/CMakeLists.txt
@@ -70,27 +70,18 @@ MESSAGE(STATUS "Picked docker base image based on host OS: ${DISTRO_BASE_IMAGE}"
if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
# Add a target to build a base docker image for 'build_type'. 'build_context_args' are
# passed to the setup_build_context.py script.
- function(add_base_image build_type build_context_args)
+ function(add_base_image build_type build_context_args install_os_packages_args)
# Build context depends on daemons and frontend jars.
# Sending the whole impala workspace including test binaries, testdata, etc
# to the docker daemon can be very expensive, so we create a build context
# with symlinks
add_custom_target(impala_base_build_context_${build_type}
COMMAND ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py ${build_context_args}
+ --output-dir ${IMPALA_BASE_BUILD_CONTEXT_DIR}/${build_type}
DEPENDS daemons java ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py
COMMENT "Creating impala base build context build_type=${build_type}."
VERBATIM
)
- # The base image Dockerfile takes an argument INSTALL_OS_PACKAGES_ARGS to specify
- # arguments for install_os_packages.sh. This can can be set to "--install-debug-tools"
- # to add extra utilities. Enable this for debug builds.
- set(INSTALL_OS_PACKAGES_ARGS "")
- if (${build_type} STREQUAL "debug")
- set(INSTALL_OS_PACKAGES_ARGS "--install-debug-tools")
- endif()
- if ($ENV{IMPALA_DOCKER_USE_JAVA11} STREQUAL "true")
- set(INSTALL_OS_PACKAGES_ARGS "${INSTALL_OS_PACKAGES_ARGS} --use-java11")
- endif()
# Target for the base Impala image.
add_custom_target(impala_base_image_${build_type}
# Run docker build inside the build context directory so that all dependencies are
@@ -99,7 +90,7 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
COMMAND tar cvh . -C ${CMAKE_SOURCE_DIR}/docker/impala_base/ . |
${DOCKER_BUILD} -t impala_base_${build_type}
--build-arg BASE_IMAGE=${DISTRO_BASE_IMAGE}
- --build-arg INSTALL_OS_PACKAGES_ARGS=${INSTALL_OS_PACKAGES_ARGS} -
+ --build-arg INSTALL_OS_PACKAGES_ARGS=${install_os_packages_args} -
WORKING_DIRECTORY ${IMPALA_BASE_BUILD_CONTEXT_DIR}/${build_type}
DEPENDS impala_base_build_context_${build_type} ${CMAKE_SOURCE_DIR}/docker/impala_base/Dockerfile
DEPENDS ${CMAKE_SOURCE_DIR}/docker/daemon_entrypoint.sh
@@ -109,13 +100,18 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
VERBATIM
)
endfunction()
- add_base_image(release "")
- add_base_image(debug "--debug-build")
+ add_base_image(release "" "")
+ add_base_image(release_java11 "" "--java 11")
+ # Debug images include debug tools
+ add_base_image(debug "--debug-build" "--install-debug-tools")
+ add_base_image(debug_java11 "--debug-build" "--install-debug-tools --java 11")
# Target to build all docker images. Dependencies are added for each docker image
# instantiated below.
add_custom_target(docker_images)
+ add_custom_target(docker_java11_images)
add_custom_target(docker_debug_images)
+ add_custom_target(docker_debug_java11_images)
add_custom_target(quickstart_docker_images)
set(exported_image_names "")
@@ -146,13 +142,20 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
function(add_daemon_docker_images daemon_name)
set(release_image ${daemon_name})
set(release_target ${daemon_name}_image)
+ set(release_java11_image ${daemon_name}_java11)
+ set(release_java11_target ${daemon_name}_java11_image)
set(debug_image ${daemon_name}_debug)
set(debug_target ${daemon_name}_debug_image)
+ set(debug_java11_image ${daemon_name}_debug_java11)
+ set(debug_java11_target ${daemon_name}_debug_java11_image)
add_daemon_docker_image(${release_target} ${daemon_name} ${release_image} release)
+ add_daemon_docker_image(${release_java11_target} ${daemon_name} ${release_java11_image} release_java11)
add_daemon_docker_image(${debug_target} ${daemon_name} ${debug_image} debug)
+ add_daemon_docker_image(${debug_java11_target} ${daemon_name} ${debug_java11_image} debug_java11)
ADD_DEPENDENCIES(docker_images ${release_target})
+ ADD_DEPENDENCIES(docker_java11_images ${release_java11_target})
ADD_DEPENDENCIES(docker_debug_images ${debug_target})
- set(exported_image_names "${exported_image_names} ${release_image}" PARENT_SCOPE)
+ ADD_DEPENDENCIES(docker_debug_java11_images ${debug_java11_target})
endfunction()
# Stamp out image targets for all of the Impala daemons.
@@ -211,7 +214,8 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
# to the docker daemon can be very expensive, so we create a build context
# with symlinks
add_custom_target(impala_utility_build_context_${build_type}
- COMMAND ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py ${build_context_args} --utility-context
+ COMMAND ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py ${build_context_args}
+ --utility-context --output-dir ${IMPALA_UTILITY_BUILD_CONTEXT_DIR}/${build_type}
DEPENDS impala-profile-tool ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py
COMMENT "Creating impala utility build context build_type=${build_type}."
VERBATIM
diff --git a/docker/install_os_packages.sh b/docker/install_os_packages.sh
index 0c97f93a1..0c863a045 100755
--- a/docker/install_os_packages.sh
+++ b/docker/install_os_packages.sh
@@ -24,12 +24,12 @@
set -euo pipefail
INSTALL_DEBUG_TOOLS=false
-USE_JAVA11=false
+JAVA_VERSION=8
function print_usage {
echo "install_os_packages.sh - Helper script to install OS dependencies"
echo "[--install-debug-tools] : Also install debug tools like curl, iproute, etc"
- echo "[--use-java11] : Use Java 11 rather than the default Java 8."
+ echo "[--java <version>] : Use specified Java version rather than the default Java 8."
}
while [ -n "$*" ]
@@ -38,8 +38,9 @@ do
--install-debug-tools)
INSTALL_DEBUG_TOOLS=true
;;
- --use-java11)
- USE_JAVA11=true
+ --java)
+ JAVA_VERSION="${2-}"
+ shift;
;;
--help|*)
print_usage
@@ -50,7 +51,7 @@ do
done
echo "INSTALL_DEBUG_TOOLS=${INSTALL_DEBUG_TOOLS}"
-echo "USE_JAVA11=${USE_JAVA11}"
+echo "JAVA_VERSION=${JAVA_VERSION}"
# This can get more detailed if there are specific steps
# for specific versions, but at the moment the distribution
@@ -67,7 +68,7 @@ else
# Ubuntu 16.04 does not have Java 11 in its package repository,
# so exit with an error.
- if [[ $DISTRIB_RELEASE == 16.04 ]] && $USE_JAVA11 ; then
+ if [[ $DISTRIB_RELEASE == 16.04 ]] && [[ $JAVA_VERSION == 11 ]] ; then
echo "ERROR: Java 11 is not supported on Ubuntu 16.04"
exit 1
fi
@@ -85,11 +86,6 @@ fi
if [[ $DISTRIBUTION == Ubuntu ]]; then
export DEBIAN_FRONTEND=noninteractive
apt-get update
- if $USE_JAVA11 ; then
- apt-get install -y openjdk-11-jre-headless
- else
- apt-get install -y openjdk-8-jre-headless
- fi
apt-get install -y \
hostname \
krb5-user \
@@ -97,6 +93,7 @@ if [[ $DISTRIBUTION == Ubuntu ]]; then
libsasl2-2 \
libsasl2-modules \
libsasl2-modules-gssapi-mit \
+ openjdk-${JAVA_VERSION}-jre-headless \
tzdata
if $INSTALL_DEBUG_TOOLS ; then
echo "Installing extra debug tools"
@@ -111,17 +108,14 @@ if [[ $DISTRIBUTION == Ubuntu ]]; then
vim
fi
elif [[ $DISTRIBUTION == Redhat ]]; then
- if $USE_JAVA11 ; then
- yum install -y --disableplugin=subscription-manager \
- java-11-openjdk-headless
- else
- yum install -y --disableplugin=subscription-manager \
- java-1.8.0-openjdk-headless
+ if [[ $JAVA_VERSION == 8 ]]; then
+ JAVA_VERSION=1.8.0
fi
yum install -y --disableplugin=subscription-manager \
cyrus-sasl-gssapi \
cyrus-sasl-plain \
hostname \
+ java-${JAVA_VERSION}-openjdk-headless \
krb5-workstation \
openldap-devel \
procps-ng \
diff --git a/docker/setup_build_context.py b/docker/setup_build_context.py
index 82f13e0bf..b46d88043 100755
--- a/docker/setup_build_context.py
+++ b/docker/setup_build_context.py
@@ -33,17 +33,12 @@ parser.add_argument("--debug-build", help="Setup build context for debug build",
parser.add_argument("--utility-context",
help="Setup utility build context instead of daemon",
action="store_true")
+parser.add_argument("--output-dir", help="Directory to use for output")
args = parser.parse_args()
IMPALA_HOME = os.environ["IMPALA_HOME"]
-if args.debug_build:
- BUILD_TYPE = "debug"
-else:
- BUILD_TYPE = "release"
-if args.utility_context:
- OUTPUT_DIR = os.path.join(IMPALA_HOME, "docker/build_context_utility", BUILD_TYPE)
-else:
- OUTPUT_DIR = os.path.join(IMPALA_HOME, "docker/build_context", BUILD_TYPE)
+BUILD_TYPE = "debug" if args.debug_build else "release"
+OUTPUT_DIR = args.output_dir
IMPALA_TOOLCHAIN_PACKAGES_HOME = os.environ["IMPALA_TOOLCHAIN_PACKAGES_HOME"]
IMPALA_GCC_VERSION = os.environ["IMPALA_GCC_VERSION"]