You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by dy...@apache.org on 2019/02/08 12:12:36 UTC
[incubator-druid] branch master updated: Add docker container for
druid (#6896)
This is an automated email from the ASF dual-hosted git repository.
dylanwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new b3dcbe7 Add docker container for druid (#6896)
b3dcbe7 is described below
commit b3dcbe70ad206b703713398fa39bd18e7e2f01c2
Author: Don Bowman <db...@donbowman.ca>
AuthorDate: Fri Feb 8 07:12:28 2019 -0500
Add docker container for druid (#6896)
* Add docker container for druid
This container is an 'omnibus' (since there is such a high
overlap with the various services). It includes all contrib
extension as well as the msql connector.
It is intended to be run as `docker run NAME SERVICE`
(e.g. docker run druid:latest broker)
* Add Apache license header
* Resolve issues from Pull Request review
* Add comments at top of script per PR comments
* Revert BUILDKIT. Not available everywhere.
* Don't set hostname, allow default (IP)
Some environments (e.g. Kubernetes Deployments) don't resolve
hostname to IP.
* Switch to amd64 glibc-based busybox from 32-bit uclibc
* Override service-specific configuration
* Replace MAINTAINER w/ LABEL
* Add mysql connector to application classpath
This works around issue #3770
https://github.com/apache/incubator-druid/issues/3770
* Add docker-compose and sample environment
Signed-off-by: Don Bowman <do...@agilicus.com>
---
.dockerignore | 37 +++++++
distribution/docker/Dockerfile | 56 +++++++++++
distribution/docker/README.md | 26 +++++
distribution/docker/docker-compose.yml | 124 +++++++++++++++++++++++
distribution/docker/druid.sh | 137 ++++++++++++++++++++++++++
distribution/docker/environment | 51 ++++++++++
distribution/docker/sha256sums.txt | 19 ++++
distribution/src/assembly/source-assembly.xml | 1 +
8 files changed, 451 insertions(+)
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..d7ae5a9
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+.git
+**/*.jar
+**/*.class
+dist
+target
+*.iml
+*.ipr
+*.iws
+*.tar.gz
+*.swp
+*.swo
+.classpath
+.idea
+.project
+.settings/
+*.log
+*.DS_Store
+_site
+dependency-reduced-pom.xml
diff --git a/distribution/docker/Dockerfile b/distribution/docker/Dockerfile
new file mode 100644
index 0000000..13c53e7
--- /dev/null
+++ b/distribution/docker/Dockerfile
@@ -0,0 +1,56 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+FROM maven:3-jdk-8 as builder
+
+COPY . /src
+WORKDIR /src
+RUN mvn install -ff -DskipTests -Dforbiddenapis.skip=true -Pdist -Pbundle-contrib-exts
+
+RUN \
+ VER=$(mvn -B org.apache.maven.plugins:maven-help-plugin:3.1.1:evaluate -Dexpression=project.version -q -DforceStdout=true -f pom.xml 2>/dev/null) \
+ && tar -zxf ./distribution/target/apache-druid-${VER}-bin.tar.gz -C /opt \
+ && ln -s /opt/apache-druid-${VER} /opt/druid
+
+RUN wget -O /opt/druid/extensions/mysql-metadata-storage/mysql-connector-java-5.1.38.jar http://central.maven.org/maven2/mysql/mysql-connector-java/5.1.38/mysql-connector-java-5.1.38.jar \
+ && sha256sum --ignore-missing -c /src/distribution/docker/sha256sums.txt \
+ && ln -s /opt/druid/extensions/mysql-metadata-storage/mysql-connector-java-5.1.38.jar /opt/druid/lib
+
+RUN addgroup --gid 1000 druid \
+ && adduser --home /opt/druid --shell /bin/sh --no-create-home --uid 1000 --gecos '' --gid 1000 --disabled-password druid \
+ && mkdir -p /opt/druid/var \
+ && chown -R druid:druid /opt/druid \
+ && chmod 775 /opt/druid/var
+
+FROM amd64/busybox:1.30.0-glibc as busybox
+FROM gcr.io/distroless/java
+LABEL maintainer="Don Bowman <do...@agilicus.com>"
+
+COPY --from=busybox /bin/busybox /busybox/busybox
+RUN ["/busybox/busybox", "--install", "/bin"]
+COPY --from=builder /etc/passwd /etc/passwd
+COPY --from=builder /etc/group /etc/group
+COPY --from=builder --chown=druid /opt /opt
+COPY distribution/docker/druid.sh /druid.sh
+RUN chown -R druid:druid /opt/druid
+USER druid
+VOLUME /opt/druid/var
+WORKDIR /opt/druid
+
+ENTRYPOINT ["/druid.sh"]
diff --git a/distribution/docker/README.md b/distribution/docker/README.md
new file mode 100644
index 0000000..fdc580c
--- /dev/null
+++ b/distribution/docker/README.md
@@ -0,0 +1,26 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+## Build
+
+From the root of the repo, run `docker build -t druid:tag -f distribution/docker/Dockerfile .`
+
+## Run
+
+Edit `environment` to suite. Run 'docker-compose -f distribution/docker/docker-compose.yml up`
diff --git a/distribution/docker/docker-compose.yml b/distribution/docker/docker-compose.yml
new file mode 100644
index 0000000..0b96241
--- /dev/null
+++ b/distribution/docker/docker-compose.yml
@@ -0,0 +1,124 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+version: "2.2"
+
+volumes:
+ metadata_data: {}
+ middle_var: {}
+ historical_var: {}
+ broker_var: {}
+ coordinator_var: {}
+ overlord_var: {}
+
+services:
+ postgres:
+ container_name: postgres
+ image: postgres:latest
+ volumes:
+ - metadata_data:/var/lib/postgresql/data
+ environment:
+ - POSTGRES_PASSWORD=FoolishPassword
+ - POSTGRES_USER=druid
+ - POSTGRES_DB=druid
+
+ # Need 3.5 or later for container nodes
+ zookeeper:
+ container_name: zookeeper
+ image: zookeeper:3.5
+ environment:
+ - ZOO_MY_ID=1
+
+ coordinator:
+ image: druid
+ container_name: coordinator
+ volumes:
+ - coordinator_var:/opt/druid/var
+ depends_on:
+ - zookeeper
+ - postgres
+ ports:
+ - "3001:8081"
+ command:
+ - coordinator
+ env_file:
+ - environment
+
+ broker:
+ image: druid
+ container_name: broker
+ volumes:
+ - broker_var:/opt/druid/var
+ depends_on:
+ - zookeeper
+ - postgres
+ - coordinator
+ ports:
+ - "3002:8082"
+ command:
+ - broker
+ env_file:
+ - environment
+
+ historical:
+ image: druid
+ container_name: historical
+ volumes:
+ - historical_var:/opt/druid/var
+ depends_on:
+ - zookeeper
+ - postgres
+ - coordinator
+ ports:
+ - "3003:8083"
+ command:
+ - historical
+ env_file:
+ - environment
+
+ overlord:
+ image: druid
+ container_name: overlord
+ volumes:
+ - overlord_var:/opt/druid/var
+ depends_on:
+ - zookeeper
+ - postgres
+ ports:
+ - "4000:8090"
+ command:
+ - overlord
+ env_file:
+ - environment
+
+ middlemanager:
+ image: druid
+ container_name: middlemanager
+ volumes:
+ - middle_var:/opt/druid/var
+ depends_on:
+ - zookeeper
+ - postgres
+ - coordinator
+ ports:
+ - "4001:8091"
+ command:
+ - middleManager
+ env_file:
+ - environment
+
diff --git a/distribution/docker/druid.sh b/distribution/docker/druid.sh
new file mode 100755
index 0000000..b142ec6
--- /dev/null
+++ b/distribution/docker/druid.sh
@@ -0,0 +1,137 @@
+#!/bin/sh
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# NOTE: this is a 'run' script for the stock tarball
+# It takes 1 required argument (the name of the service,
+# e.g. 'broker', 'historical' etc). Any additional arguments
+# are passed to that service.
+#
+# It accepts 'JAVA_OPTS' as an environment variable
+#
+# Additional env vars:
+# - DRUID_LOG4J -- set the entire log4j.xml verbatim
+# - DRUID_LOG_LEVEL -- override the default log level in default log4j
+# - DRUID_XMX -- set Java Xmx
+# - DRUID_XMS -- set Java Xms
+# - DRUID_MAXNEWSIZE -- set Java max new size
+# - DRUID_NEWSIZE -- set Java new size
+# - DRUID_MAXDIRECTMEMORYSIZE -- set Java max direct memory size
+#
+# - DRUID_CONFIG -- full path to a file for druid 'common' properties
+# - DRUID_CONFIG_${service} -- full path to a file for druid 'service' properties
+
+set -e
+SERVICE="$1"
+
+echo "$(date -Is) startup service $SERVICE"
+
+# We put all the config in /tmp/conf to allow for a
+# read-only root filesystem
+cp -r /opt/druid/conf /tmp/conf
+
+# Delete the old key (if existing) and append new key=value
+setKey() {
+ service="$1"
+ key="$2"
+ value="$3"
+ case "$service" in
+ _common)
+ fname=common.runtime.properties ;;
+ *)
+ fname=runtime.properties ;;
+ esac
+ # Delete from all
+ sed -ri "/$key=/d" /tmp/conf/druid/_common/common.runtime.properties
+ [ -f /tmp/conf/druid/$service/$fname ] && sed -ri "/$key=/d" /tmp/conf/druid/$service/$fname
+ [ -f /tmp/conf/druid/$service/$fname ] && echo "$key=$value" >> /tmp/conf/druid/$service/$fname
+ [ -f /tmp/conf/druid/$service/$fname ] || echo "$key=$value" >> /tmp/conf/druid/_common/$fname
+}
+
+setJavaKey() {
+ service="$1"
+ key=$2
+ value=$3
+ file=/tmp/conf/druid/$service/jvm.config
+ sed -ri "/$key/d" $file
+ echo $value >> $file
+}
+
+## Setup host names
+if [ -n "${ZOOKEEPER}" ]
+then
+ setKey _common druid.zk.service.host "${ZOOKEEPER}"
+fi
+
+setKey $SERVICE druid.host $(ip r get 1 | awk '{print $7;exit}')
+
+
+env |grep ^druid_ | while read evar
+do
+ # Can't use IFS='=' to parse since var might have = in it (e.g. password)
+ val=$(echo "$evar" | sed -e 's?[^=]*=??')
+ var=$(echo "$evar" | sed -e 's?^\([^=]*\)=.*?\1?g' -e 's?_?.?g')
+ setKey $SERVICE "$var" "$val"
+done
+
+env |grep ^s3service | while read evar
+do
+ val=$(echo "$evar" | sed -e 's?[^=]*=??')
+ var=$(echo "$evar" | sed -e 's?^\([^=]*\)=.*?\1?g' -e 's?_?.?' -e 's?_?-?g')
+ echo "$var=$val" >> /tmp/conf/druid/_common/jets3t.properties
+done
+
+# This is to allow configuration via a Kubernetes configMap without
+# e.g. using subPath (you can also mount the configMap on /tmp/conf/druid)
+if [ -n "$DRUID_CONFIG_COMMON" ]
+then
+ cp -f "$DRUID_CONFIG_COMMON" /tmp/conf/druid/_common/common.runtime.properties
+fi
+
+SCONFIG=$(printf "%s_%s" DRUID_CONFIG ${SERVICE})
+SCONFIG=$(eval echo \$$(echo $SCONFIG))
+
+if [ -n "${SCONFIG}" ]
+then
+ cp -f "${SCONFIG}" /tmp/conf/druid/${SERVICE}/runtime.properties
+fi
+
+# Now do the java options
+
+if [ -n "$DRUID_XMX" ]; then setJavaKey ${SERVICE} -Xmx -Xmx${DRUID_XMX}; fi
+if [ -n "$DRUID_XMS" ]; then setJavaKey ${SERVICE} -Xms -Xms${DRUID_XMS}; fi
+if [ -n "$DRUID_MAXNEWSIZE" ]; then setJavaKey ${SERVICE} -XX:MaxNewSize -XX:MaxNewSize=${DRUID_MAXNEWSIZE}; fi
+if [ -n "$DRUID_NEWSIZE" ]; then setJavaKey ${SERVICE} -XX:NewSize -XX:MaxNewSize=${DRUID_NEWSIZE}; fi
+if [ -n "$DRUID_MAXDIRECTMEMORYSIZE" ]; then setJavaKey ${SERVICE} -XX:MaxDirectMemorySize -XX:MaxDirectMemorySize=${DRUID_MAXDIRECTMEMORYSIZE}; fi
+
+JAVA_OPTS="$JAVA_OPTS $(cat /tmp/conf/druid/${SERVICE}/jvm.config | xargs)"
+
+if [ -n "$DRUID_LOG_LEVEL" ]
+then
+ sed -ri 's/"info"/"'$DRUID_LOG_LEVEL'"/g' /tmp/conf/druid/_common/log4j2.xml
+fi
+
+if [ -n "$DRUID_LOG4J" ]
+then
+ echo "$DRUID_LOG4J" > /tmp/conf/druid/_common/log4j2.xml
+fi
+
+mkdir -p var/tmp var/druid/segments var/druid/indexing-logs var/druid/task var/druid/hadoop-tmp var/druid/segment-cache
+exec java ${JAVA_OPTS} -cp /tmp/conf/druid/_common:/tmp/conf/druid/${SERVICE}:lib/*: org.apache.druid.cli.Main server $@
diff --git a/distribution/docker/environment b/distribution/docker/environment
new file mode 100644
index 0000000..e61a9f3
--- /dev/null
+++ b/distribution/docker/environment
@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Java tuning
+DRUID_XMX=1g
+DRUID_XMS=1g
+DRUID_MAXNEWSIZE=250m
+DRUID_NEWSIZE=250m
+DRUID_MAXDIRECTMEMORYSIZE=6172m
+
+druid_emitter_logging_logLevel=debug
+
+druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "druid-azure-extensions", "postgresql-metadata-storage"]
+
+druid_zk_service_host=zookeeper
+
+druid_metadata_storage_host=
+druid_metadata_storage_type=postgresql
+druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
+druid_metadata_storage_connector_user=druid
+druid_metadata_storage_connector_password=FoolishPassword
+
+druid_coordinator_balancer_strategy=cachingCost
+
+druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
+druid_indexer_fork_property_druid_processing_buffer_sizeBytes=268435456
+
+druid_storage_type=azure
+druid_azure_account=YOURACCOUNT
+druid_azure_key=YOURKEY
+druid_azure_container=druid
+druid_azure_protocol=https
+druid_azure_maxTries=3
+
+DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>
diff --git a/distribution/docker/sha256sums.txt b/distribution/docker/sha256sums.txt
new file mode 100644
index 0000000..6f858cd
--- /dev/null
+++ b/distribution/docker/sha256sums.txt
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+b95bf9fe25cb5428f378a62fc842e177ca004b4ae1f9054968b2a396dcc1ec22 /opt/druid/extensions/mysql-metadata-storage/mysql-connector-java-5.1.38.jar
diff --git a/distribution/src/assembly/source-assembly.xml b/distribution/src/assembly/source-assembly.xml
index 59903d4..0d7e019 100644
--- a/distribution/src/assembly/source-assembly.xml
+++ b/distribution/src/assembly/source-assembly.xml
@@ -45,6 +45,7 @@
<exclude>%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?release\.properties]</exclude>
<exclude>.gitignore</exclude>
+ <exclude>.dockerignore</exclude>
<exclude>.travis.yml</exclude>
<exclude>publications/**</exclude>
<exclude>upload.sh</exclude>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org