You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by ji...@apache.org on 2020/02/14 05:24:55 UTC
[druid] branch master updated: add docker tutorial,
friendlier docker-compose.yml, experimental java 11 dockerfile (#9262)
This is an automated email from the ASF dual-hosted git repository.
jihoonson pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 2e54755 add docker tutorial, friendlier docker-compose.yml, experimental java 11 dockerfile (#9262)
2e54755 is described below
commit 2e54755a03635d24cb78e893946988110a8fe2b5
Author: Clint Wylie <cw...@apache.org>
AuthorDate: Thu Feb 13 21:24:45 2020 -0800
add docker tutorial, friendlier docker-compose.yml, experimental java 11 dockerfile (#9262)
* add docker tutorial, experimental java 11 dockerfile
* fix typo
* spelling
* doc adjustments
---
distribution/docker/Dockerfile.java11 | 67 ++++++++++++++++++++++++++++
distribution/docker/README.md | 4 ++
distribution/docker/docker-compose.yml | 40 ++++++-----------
distribution/docker/environment | 15 ++++---
docs/tutorials/docker.md | 80 ++++++++++++++++++++++++++++++++++
website/i18n/en.json | 3 ++
website/sidebars.json | 1 +
7 files changed, 177 insertions(+), 33 deletions(-)
diff --git a/distribution/docker/Dockerfile.java11 b/distribution/docker/Dockerfile.java11
new file mode 100644
index 0000000..bbc5340
--- /dev/null
+++ b/distribution/docker/Dockerfile.java11
@@ -0,0 +1,67 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+FROM maven:3-jdk-11-slim as builder
+
+RUN export DEBIAN_FRONTEND=noninteractive \
+ && apt-get -qq update \
+ && apt-get -qq -y install --no-install-recommends python3 python3-yaml
+
+COPY . /src
+WORKDIR /src
+RUN mvn -B -ff -q dependency:go-offline \
+ install \
+ -Pdist,bundle-contrib-exts \
+ -DskipTests \
+ -Danimal.sniffer.skip=true \
+ -Dcheckstyle.skip=true \
+ -Denforcer.skip=true \
+ -Dforbiddenapis.skip=true \
+ -Dmaven.javadoc.skip=true \
+ -Dpmd.skip=true \
+ -Dspotbugs.skip=true
+
+RUN VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.1.1:evaluate \
+ -Dexpression=project.version -DforceStdout=true \
+ ) \
+ && tar -zxf ./distribution/target/apache-druid-${VERSION}-bin.tar.gz -C /opt \
+ && ln -s /opt/apache-druid-${VERSION} /opt/druid
+
+FROM amd64/busybox:1.30.0-glibc as busybox
+
+FROM gcr.io/distroless/java:11
+LABEL maintainer="Apache Druid Developers <de...@druid.apache.org>"
+
+COPY --from=busybox /bin/busybox /busybox/busybox
+RUN ["/busybox/busybox", "--install", "/bin"]
+
+COPY --from=builder /opt /opt
+COPY distribution/docker/druid.sh /druid.sh
+
+RUN addgroup -S -g 1000 druid \
+ && adduser -S -u 1000 -D -H -h /opt/druid -s /bin/sh -g '' -G druid druid \
+ && mkdir -p /opt/druid/var \
+ && chown -R druid:druid /opt \
+ && chmod 775 /opt/druid/var
+
+USER druid
+VOLUME /opt/druid/var
+WORKDIR /opt/druid
+
+ENTRYPOINT ["/druid.sh"]
diff --git a/distribution/docker/README.md b/distribution/docker/README.md
index f91b4b3..5968dbd 100644
--- a/distribution/docker/README.md
+++ b/distribution/docker/README.md
@@ -25,6 +25,10 @@ From the root of the repo, run `docker build -t apache/druid:tag -f distribution
Edit `environment` to suite. Run `docker-compose -f distribution/docker/docker-compose.yml up`
+## Java 11 (experimental)
+
+From the root of the repo, run `docker build -t apache/druid:tag -f distribution/docker/Dockerfile.java11 .` which will build Druid to run in a Java 11 environment.
+
## MySQL Database Connector
This image contains solely the postgres metadata storage connector. If you
diff --git a/distribution/docker/docker-compose.yml b/distribution/docker/docker-compose.yml
index 7a89cc6..6a1ee3d 100644
--- a/distribution/docker/docker-compose.yml
+++ b/distribution/docker/docker-compose.yml
@@ -24,9 +24,9 @@ volumes:
historical_var: {}
broker_var: {}
coordinator_var: {}
- overlord_var: {}
router_var: {}
+
services:
postgres:
container_name: postgres
@@ -46,22 +46,23 @@ services:
- ZOO_MY_ID=1
coordinator:
- image: apache/druid
+ image: apache/druid:0.17.0
container_name: coordinator
volumes:
+ - ./storage:/opt/data
- coordinator_var:/opt/druid/var
depends_on:
- zookeeper
- postgres
ports:
- - "3001:8081"
+ - "8081:8081"
command:
- coordinator
env_file:
- environment
broker:
- image: apache/druid
+ image: apache/druid:0.17.0
container_name: broker
volumes:
- broker_var:/opt/druid/var
@@ -70,61 +71,48 @@ services:
- postgres
- coordinator
ports:
- - "3002:8082"
+ - "8082:8082"
command:
- broker
env_file:
- environment
historical:
- image: apache/druid
+ image: apache/druid:0.17.0
container_name: historical
volumes:
+ - ./storage:/opt/data
- historical_var:/opt/druid/var
depends_on:
- zookeeper
- postgres
- coordinator
ports:
- - "3003:8083"
+ - "8083:8083"
command:
- historical
env_file:
- environment
- overlord:
- image: apache/druid
- container_name: overlord
- volumes:
- - overlord_var:/opt/druid/var
- depends_on:
- - zookeeper
- - postgres
- ports:
- - "4000:8090"
- command:
- - overlord
- env_file:
- - environment
-
middlemanager:
- image: apache/druid
+ image: apache/druid:0.17.0
container_name: middlemanager
volumes:
+ - ./storage:/opt/data
- middle_var:/opt/druid/var
depends_on:
- zookeeper
- postgres
- coordinator
ports:
- - "4001:8091"
+ - "8091:8091"
command:
- middleManager
env_file:
- environment
router:
- image: apache/druid
+ image: apache/druid:0.17.0
container_name: router
volumes:
- router_var:/opt/druid/var
@@ -133,7 +121,7 @@ services:
- postgres
- coordinator
ports:
- - "4008:8888"
+ - "8888:8888"
command:
- router
env_file:
diff --git a/distribution/docker/environment b/distribution/docker/environment
index e61a9f3..2023138 100644
--- a/distribution/docker/environment
+++ b/distribution/docker/environment
@@ -26,7 +26,7 @@ DRUID_MAXDIRECTMEMORYSIZE=6172m
druid_emitter_logging_logLevel=debug
-druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "druid-azure-extensions", "postgresql-metadata-storage"]
+druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage"]
druid_zk_service_host=zookeeper
@@ -41,11 +41,12 @@ druid_coordinator_balancer_strategy=cachingCost
druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
druid_indexer_fork_property_druid_processing_buffer_sizeBytes=268435456
-druid_storage_type=azure
-druid_azure_account=YOURACCOUNT
-druid_azure_key=YOURKEY
-druid_azure_container=druid
-druid_azure_protocol=https
-druid_azure_maxTries=3
+druid_storage_type=local
+druid_storage_storageDirectory=/opt/data/segments
+druid_indexer_logs_type=file
+druid_indexer_logs_directory=/opt/data/indexing-logs
+
+druid_processing_numThreads=2
+druid_processing_numMergeBuffers=2
DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>
diff --git a/docs/tutorials/docker.md b/docs/tutorials/docker.md
new file mode 100644
index 0000000..dfb46bb
--- /dev/null
+++ b/docs/tutorials/docker.md
@@ -0,0 +1,80 @@
+---
+id: docker
+title: "Docker"
+---
+
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+In this quickstart, we will download the Apache Druid image from [Docker Hub](https://hub.docker.com/r/apache/druid) and set it up on a single machine using [Docker](https://www.docker.com/get-started) and [Docker Compose](https://docs.docker.com/compose/). The cluster will be ready to load data after completing this initial setup.
+
+Before beginning the quickstart, it is helpful to read the [general Druid overview](../design/index.md) and the [ingestion overview](../ingestion/index.md), as the tutorials will refer to concepts discussed on those pages. Additionally, familiarity with Docker is recommended.
+
+## Prerequisites
+
+* Docker
+
+## Getting started
+
+The Druid source code contains [an example `docker-compose.yml`](https://github.com/apache/druid/blob/master/distribution/docker/docker-compose.yml) which can pull an image from Docker Hub and is suited to be used as an example environment and to experiment with Docker based Druid configuration and deployments.
+
+### Compose file
+
+The example `docker-compose.yml` will create a container for each Druid service, as well as Zookeeper and a PostgreSQL container as the metadata store. Deep storage will be a local directory, by default configured as `./storage` relative to your `docker-compose.yml` file, and will be mounted as `/opt/data` and shared between Druid containers which require access to deep storage. The Druid containers are configured via an [environment file](https://github.com/apache/druid/blob/master/dist [...]
+
+### Configuration
+
+Configuration of the Druid Docker container is done via environment variables, which may additionally specify paths to [the standard Druid configuration files](../configuration/index.md)
+
+Special environment variables:
+
+* `JAVA_OPTS` -- set java options
+* `DRUID_LOG4J` -- set the entire `log4j.xml` verbatim
+* `DRUID_LOG_LEVEL` -- override the default log level in default log4j
+* `DRUID_XMX` -- set Java `Xmx`
+* `DRUID_XMS` -- set Java `Xms`
+* `DRUID_MAXNEWSIZE` -- set Java max new size
+* `DRUID_NEWSIZE` -- set Java new size
+* `DRUID_MAXDIRECTMEMORYSIZE` -- set Java max direct memory size
+* `DRUID_CONFIG_COMMON` -- full path to a file for druid 'common' properties
+* `DRUID_CONFIG_${service}` -- full path to a file for druid 'service' properties
+
+In addition to the special environment variables, the script which launches Druid in the container will also attempt to use any environment variable starting with the `druid_` prefix as a command-line configuration. For example, an environment variable
+
+```druid_metadata_storage_type=postgresql```
+
+would be translated into
+
+```-Ddruid.metadata.storage.type=postgresql```
+
+for the Druid process in the container.
+
+The Druid `docker-compose.yml` example utilizes a single environment file to specify the complete Druid configuration; however, in production use cases we suggest using either `DRUID_COMMON_CONFIG` and `DRUID_CONFIG_${service}` or specially tailored, service-specific environment files.
+## Launching the cluster
+
+Run `docker-compose up` to launch the cluster with a shell attached, or `docker-compose up -d` to run the cluster in the background. If using the example files directly, this command should be run from `distribution/docker/` in your Druid installation directory.
+
+Once the cluster has started, you can navigate to [http://localhost:8888](http://localhost:8888).
+The [Druid router process](../design/router.md), which serves the [Druid console](../operations/druid-console.md), resides at this address.
+
+
+
+It takes a few seconds for all the Druid processes to fully start up. If you open the console immediately after starting the services, you may see some errors that you can safely ignore.
+
+From here you can follow along with the [standard tutorials](./index.md#loading-data), or elaborate on your `docker-compose.yml` to add any additional external service dependencies as necessary.
diff --git a/website/i18n/en.json b/website/i18n/en.json
index 9d4075e..bf8283d 100644
--- a/website/i18n/en.json
+++ b/website/i18n/en.json
@@ -467,6 +467,9 @@
"tutorials/cluster": {
"title": "Clustered deployment"
},
+ "tutorials/docker": {
+ "title": "Docker"
+ },
"tutorials/index": {
"title": "Quickstart"
},
diff --git a/website/sidebars.json b/website/sidebars.json
index ab61879..e393196 100644
--- a/website/sidebars.json
+++ b/website/sidebars.json
@@ -3,6 +3,7 @@
"Getting started": [
"design/index",
"tutorials/index",
+ "tutorials/docker",
"operations/single-server",
"tutorials/cluster"
],
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org