You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by fr...@apache.org on 2022/09/16 01:25:59 UTC
[druid] branch master updated: Some improvements about Docker (#13059)
This is an automated email from the ASF dual-hosted git repository.
frankchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new b8dd822f32 Some improvements about Docker (#13059)
b8dd822f32 is described below
commit b8dd822f32d35ddd37547d9441bf61c4cba85332
Author: Frank Chen <fr...@outlook.com>
AuthorDate: Fri Sep 16 09:25:52 2022 +0800
Some improvements about Docker (#13059)
---
distribution/docker/Dockerfile | 37 +++++++++++++++++++++-----
distribution/docker/README.md | 33 ++++++++++++++++++-----
distribution/docker/druid.sh | 2 +-
integration-tests/script/setup_druid_on_k8s.sh | 4 +--
4 files changed, 60 insertions(+), 16 deletions(-)
diff --git a/distribution/docker/Dockerfile b/distribution/docker/Dockerfile
index 3b22e9d03b..5b55ec1be6 100644
--- a/distribution/docker/Dockerfile
+++ b/distribution/docker/Dockerfile
@@ -17,8 +17,14 @@
# under the License.
#
-ARG JDK_VERSION=8
-FROM maven:3.8.4-jdk-11-slim as builder
+ARG JDK_VERSION=11
+
+# The platform is explicitly specified as x64 to build the Druid distribution.
+# This is because it's not able to build the distribution on arm64 due to dependency problem of web-console. See: https://github.com/apache/druid/issues/13012
+# Since only java jars are shipped in the final image, it's OK to build the distribution on x64.
+# Once the web-console dependency problem is resolved, we can remove the --platform directive.
+FROM --platform=linux/amd64 maven:3.8.6-jdk-11-slim as builder
+
# Rebuild from source in this stage
# This can be unset if the tarball was already built outside of Docker
ARG BUILD_FROM_SOURCE="true"
@@ -29,7 +35,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \
COPY . /src
WORKDIR /src
-RUN if [ "$BUILD_FROM_SOURCE" = "true" ]; then \
+RUN --mount=type=cache,target=/root/.m2 if [ "$BUILD_FROM_SOURCE" = "true" ]; then \
mvn -B -ff -q dependency:go-offline \
install \
-Pdist,bundle-contrib-exts \
@@ -37,20 +43,39 @@ RUN if [ "$BUILD_FROM_SOURCE" = "true" ]; then \
-Dmaven.javadoc.skip=true \
; fi
-RUN VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate \
+RUN --mount=type=cache,target=/root/.m2 VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate \
-Dexpression=project.version -DforceStdout=true \
) \
&& tar -zxf ./distribution/target/apache-druid-${VERSION}-bin.tar.gz -C /opt \
&& mv /opt/apache-druid-${VERSION} /opt/druid
-FROM amd64/busybox:1.30.0-glibc as busybox
+FROM busybox:1.35.0-glibc as busybox
-FROM gcr.io/distroless/java:$JDK_VERSION
+FROM gcr.io/distroless/java$JDK_VERSION-debian11
LABEL maintainer="Apache Druid Developers <de...@druid.apache.org>"
COPY --from=busybox /bin/busybox /busybox/busybox
RUN ["/busybox/busybox", "--install", "/bin"]
+# Predefined builtin arg, see: https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope
+ARG TARGETARCH
+
+#
+# Download bash-static binary to execute scripts that require bash.
+# Although bash-static supports multiple platforms, but there's no need for us to support all those platform, amd64 and arm64 are enough.
+#
+ARG BASH_URL_BASE="https://github.com/robxu9/bash-static/releases/download/5.1.016-1.2.3"
+RUN if [ "$TARGETARCH" = "arm64" ]; then \
+ BASH_URL="${BASH_URL_BASE}/bash-linux-aarch64" ; \
+ elif [ "$TARGETARCH" = "amd64" ]; then \
+ BASH_URL="${BASH_URL_BASE}/bash-linux-x86_64" ; \
+ else \
+ echo "Unsupported architecture ($TARGETARCH)" && exit 1; \
+ fi; \
+ echo "Downloading bash-static from ${BASH_URL}" \
+ && wget ${BASH_URL} -O /bin/bash \
+ && chmod 755 /bin/bash
+
RUN addgroup -S -g 1000 druid \
&& adduser -S -u 1000 -D -H -h /opt/druid -s /bin/sh -g '' -G druid druid
diff --git a/distribution/docker/README.md b/distribution/docker/README.md
index 8a1df57fa5..4c03e230b6 100644
--- a/distribution/docker/README.md
+++ b/distribution/docker/README.md
@@ -19,15 +19,32 @@
## Build
-From the root of the repo, run `docker build -t apache/druid:tag -f distribution/docker/Dockerfile .`
+From the root of the repo, run following command:
-## Run
+```bash
+DOCKER_BUILDKIT=1 docker build -t apache/druid:tag -f distribution/docker/Dockerfile .
+```
+
+### Building images on Apple M1/M2
+To build images on Apple M1/M2, you need to follow the instructions in this section.
-Edit `environment` to suite. Run `docker-compose -f distribution/docker/docker-compose.yml up`
+1. build Druid distribution from the root of the repo
+ ```bash
+ mvn clean package -DskipTests -Pdist
+ ```
+2. build target image
+ ```
+ DOCKER_BUILDKIT=1 docker build -t apache/druid:tag -f distribution/docker/Dockerfile --build-arg BUILD_FROM_SOURCE=false .
+ ```
-## Java 11
+## Run
-From the root of the repo, run `docker build -t apache/druid:tag -f distribution/docker/Dockerfile.java11 .` which will build Druid to run in a Java 11 environment.
+1. Edit `distribution/docker/docker-compose.yml` file to change the tag of Druid's images to the tag that's used in the 'Build' phase above.
+2. Edit `environment` file to suite if necessary.
+3. Run:
+ ```bash
+ docker-compose -f distribution/docker/docker-compose.yml up
+ ```
## MySQL Database Connector
@@ -35,6 +52,8 @@ This image contains solely the postgres metadata storage connector. If you
need the mysql metadata storage connector, you can use Dockerfile.mysql to add
it to the base image above.
-`docker build -t apache/druid:tag-mysql --build-arg DRUID_RELEASE=apache/druid:tag -f distribution/docker/Dockerfile.mysql .`
+```bash
+docker build -t apache/druid:tag-mysql --build-arg DRUID_RELEASE=apache/druid:tag -f distribution/docker/Dockerfile.mysql .
+```
-where `druid:tag` is the version to use as the base.
+where `druid:tag` is the version of Druid image to use as the base.
diff --git a/distribution/docker/druid.sh b/distribution/docker/druid.sh
index f3b506d64c..b51419991f 100755
--- a/distribution/docker/druid.sh
+++ b/distribution/docker/druid.sh
@@ -196,4 +196,4 @@ then
mkdir -p ${DRUID_DIRS_TO_CREATE}
fi
-exec java ${JAVA_OPTS} -cp $COMMON_CONF_DIR:$SERVICE_CONF_DIR:lib/*: org.apache.druid.cli.Main server $@
+exec bin/run-java ${JAVA_OPTS} -cp $COMMON_CONF_DIR:$SERVICE_CONF_DIR:lib/*: org.apache.druid.cli.Main server $@
diff --git a/integration-tests/script/setup_druid_on_k8s.sh b/integration-tests/script/setup_druid_on_k8s.sh
index f21f5613b4..34010f2314 100755
--- a/integration-tests/script/setup_druid_on_k8s.sh
+++ b/integration-tests/script/setup_druid_on_k8s.sh
@@ -32,8 +32,8 @@ mvn -B -ff -q dependency:go-offline \
-Pskip-static-checks,skip-tests \
-Dmaven.javadoc.skip=true
-docker build --build-arg BUILD_FROM_SOURCE=0 -t druid/base:v1 -f distribution/docker/Dockerfile .
-docker build --build-arg BASE_IMAGE=druid/base:v1 -t druid/cluster:v1 -f distribution/docker/DockerfileBuildTarAdvanced .
+DOCKER_BUILDKIT=1 docker build --build-arg BUILD_FROM_SOURCE=0 -t druid/base:v1 -f distribution/docker/Dockerfile .
+DOCKER_BUILDKIT=1 docker build --build-arg BASE_IMAGE=druid/base:v1 -t druid/cluster:v1 -f distribution/docker/DockerfileBuildTarAdvanced .
# This tmp dir is used for MiddleManager pod and Historical Pod to cache segments.
sudo rm -rf tmp
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org