You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by dz...@apache.org on 2022/12/19 08:43:39 UTC
[drill] 02/10: DRILL-8293: Add a docker-compose file to run Drill in cluster mode (#2640)
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch 1.20
in repository https://gitbox.apache.org/repos/asf/drill.git
commit bb8524116a788be01301956b3eac28e2d0b15f60
Author: James Turton <91...@users.noreply.github.com>
AuthorDate: Fri Sep 9 07:12:52 2022 +0200
DRILL-8293: Add a docker-compose file to run Drill in cluster mode (#2640)
---
distribution/docker-cluster-mode/Dockerfile | 26 ++++++++++++
distribution/docker-cluster-mode/README | 25 +++++++++++
.../docker-cluster-mode/docker-compose.yml | 48 ++++++++++++++++++++++
.../docker-cluster-mode/drill-override.conf | 28 +++++++++++++
distribution/docker-cluster-mode/logback.xml | 39 ++++++++++++++++++
5 files changed, 166 insertions(+)
diff --git a/distribution/docker-cluster-mode/Dockerfile b/distribution/docker-cluster-mode/Dockerfile
new file mode 100644
index 0000000000..d4e360865d
--- /dev/null
+++ b/distribution/docker-cluster-mode/Dockerfile
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM apache/drill:latest
+
+ENV DRILL_LOG_LEVEL=info
+
+COPY drill-override.conf $DRILL_HOME/conf/drill-override.conf
+COPY logback.xml $DRILL_HOME/conf/logback.xml
+
+ENTRYPOINT $DRILL_HOME/bin/drillbit.sh run
diff --git a/distribution/docker-cluster-mode/README b/distribution/docker-cluster-mode/README
new file mode 100644
index 0000000000..bd8bb6d040
--- /dev/null
+++ b/distribution/docker-cluster-mode/README
@@ -0,0 +1,25 @@
+This directory contains source code artifacts to launch Drill in cluster mode
+along with a ZooKeeper. The Drill image is based on a minor customisation of
+the official Drill image that switches it from an embedded to a cluster mode
+launch. Logging is redirected to stdout.
+
+In the docker-cluster-mode directory:
+
+1. docker build -t apache/drill-cluster-mode
+2. docker-compose up
+
+Then access the web UI at http://localhost:8047 or connect a JDBC client to
+jdbc:drill:drillbit=localhost or jdbc:drill:zk=localhost but note that you
+will need to make the drillbit container hostnames resolvable from the host
+to use a ZooKeeper JDBC URL.
+
+To launch a cluster of 3 Drillbits
+
+3. docker-compose up --scale drillbit=3
+
+but first note that to use docker-compose's "scale" feature to run multiple
+Drillbit containers on a single host you will need to remove the host port
+mappings from the compose file to prevent collisions (see the comments
+on the relevant lines in that file). Once the Drillbits are launched run
+`docker-compose ps` to list the ephemeral ports that have been allocated on
+the host.
diff --git a/distribution/docker-cluster-mode/docker-compose.yml b/distribution/docker-cluster-mode/docker-compose.yml
new file mode 100644
index 0000000000..d43bfa9e6f
--- /dev/null
+++ b/distribution/docker-cluster-mode/docker-compose.yml
@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "3.9"
+services:
+ zookeeper:
+ container_name: zookeeper
+ hostname: zookeeper
+ image: zookeeper:latest
+ volumes:
+ - zk_data:/data
+ - zk_datalog:/datalog
+ - zk_logs:/logs
+ ports:
+ - "2181:2181"
+
+ drillbit:
+ # container_name not set so that --scale can be used
+ image: apache/drill-cluster-mode:latest
+ volumes:
+ - drill_data:/data
+ ports:
+ - "8047:8047" # user HTTP, replace with just "8047" for --scale
+ - "31010:31010" # user RPC and JDBC, replace with just "31010" for --scale
+ depends_on: ["zookeeper"]
+ environment:
+ - DRILL_MAX_PROC_MEM=8G
+
+volumes:
+ zk_data:
+ zk_datalog:
+ zk_logs:
+ drill_data:
diff --git a/distribution/docker-cluster-mode/drill-override.conf b/distribution/docker-cluster-mode/drill-override.conf
new file mode 100644
index 0000000000..c65652a72b
--- /dev/null
+++ b/distribution/docker-cluster-mode/drill-override.conf
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This file tells Drill to consider this module when class path scanning.
+# This file can also include any supplementary configuration information.
+# This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.
+
+# See 'drill-override-example.conf' for example configurations
+
+drill.exec: {
+ cluster-id: "drillbits1",
+ zk.connect: "zookeeper:2181"
+}
diff --git a/distribution/docker-cluster-mode/logback.xml b/distribution/docker-cluster-mode/logback.xml
new file mode 100644
index 0000000000..c1cdbfdd27
--- /dev/null
+++ b/distribution/docker-cluster-mode/logback.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<configuration>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+ </encoder>
+ </appender>
+ <logger name="org.apache.drill" additivity="false">
+ <level value="${DRILL_LOG_LEVEL:-info}" />
+ <appender-ref ref="STDOUT" />
+ </logger>
+ <logger name="query.logger" additivity="false">
+ <level value="${DRILL_LOG_LEVEL:-info}" />
+ <appender-ref ref="STDOUT" />
+ </logger>
+ <root>
+ <level value="${DRILL_LOG_LEVEL:-info}" />
+ <appender-ref ref="STDOUT" />
+ </root>
+</configuration>