You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by al...@apache.org on 2018/07/03 15:41:03 UTC
nifi git commit: NIFI-5195 Added clustering to the docker image and
added a docker-compose file that builds a basic 3 node cluster.
Repository: nifi
Updated Branches:
refs/heads/master 3164759d7 -> d3222b85a
NIFI-5195 Added clustering to the docker image and added a docker-compose file that builds a basic 3 node cluster.
This closes #2828.
Signed-off-by: Aldrin Piri <al...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/d3222b85
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/d3222b85
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/d3222b85
Branch: refs/heads/master
Commit: d3222b85ab92be19e2f622228ea5f7bc757fc07d
Parents: 3164759
Author: Mike Thomsen <mi...@gmail.com>
Authored: Fri Jun 29 20:48:08 2018 -0400
Committer: Aldrin Piri <al...@apache.org>
Committed: Tue Jul 3 11:40:31 2018 -0400
----------------------------------------------------------------------
nifi-docker/docker-compose/docker-compose.yml | 74 ++++++++++++++++++++
nifi-docker/dockerhub/README.md | 23 ++++++
nifi-docker/dockerhub/sh/start.sh | 13 +++-
.../sh/update_cluster_state_management.sh | 31 ++++++++
nifi-docker/dockermaven/sh/start.sh | 13 +++-
.../sh/update_cluster_state_management.sh | 31 ++++++++
6 files changed, 183 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3222b85/nifi-docker/docker-compose/docker-compose.yml
----------------------------------------------------------------------
diff --git a/nifi-docker/docker-compose/docker-compose.yml b/nifi-docker/docker-compose/docker-compose.yml
new file mode 100644
index 0000000..63b42ea
--- /dev/null
+++ b/nifi-docker/docker-compose/docker-compose.yml
@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ zookeeper:
+ hostname: zookeeper
+ container_name: zookeeper
+ image: 'bitnami/zookeeper:latest'
+ ports:
+ - '2181:2181'
+ environment:
+ - ALLOW_ANONYMOUS_LOGIN=yes
+ node1.nifi:
+ image: apache/nifi:1.8.0-SNAPSHOT-dockermaven
+ container_name: node1.nifi
+ hostname: node1.nifi
+ links:
+ - zookeeper
+ ports:
+ - 8080:8080
+ environment:
+ - NIFI_WEB_HTTP_HOST=node1.nifi
+ - NIFI_WEB_HTTP_PORT=8080
+ - NIFI_CLUSTER_IS_NODE=true
+ - NIFI_CLUSTER_NODE_PROTOCOL_PORT=8082
+ - NIFI_ZK_CONNECT_STRING=zookeeper:2181
+ - NIFI_ELECTION_MAX_WAIT=1 min
+ node2.nifi:
+ image: apache/nifi:1.8.0-SNAPSHOT-dockermaven
+ container_name: node2.nifi
+ hostname: node2.nifi
+ links:
+ - zookeeper
+ - node1.nifi
+ ports:
+ - 8888:8080
+ environment:
+ - NIFI_WEB_HTTP_HOST=node2.nifi
+ - NIFI_WEB_HTTP_PORT=8080
+ - NIFI_CLUSTER_IS_NODE=true
+ - NIFI_CLUSTER_NODE_PROTOCOL_PORT=8082
+ - NIFI_ZK_CONNECT_STRING=zookeeper:2181
+ - NIFI_ELECTION_MAX_WAIT=1 min
+ node3.nifi:
+ image: apache/nifi:1.8.0-SNAPSHOT-dockermaven
+ container_name: node3.nifi
+ hostname: node3.nifi
+ ports:
+ - 8999:8080
+ links:
+ - zookeeper
+ - node1.nifi
+ - node2.nifi
+ environment:
+ - NIFI_WEB_HTTP_HOST=node3.nifi
+ - NIFI_WEB_HTTP_PORT=8080
+ - NIFI_CLUSTER_IS_NODE=true
+ - NIFI_CLUSTER_NODE_PROTOCOL_PORT=8082
+ - NIFI_ZK_CONNECT_STRING=zookeeper:2181
+ - NIFI_ELECTION_MAX_WAIT=1 min
+
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3222b85/nifi-docker/dockerhub/README.md
----------------------------------------------------------------------
diff --git a/nifi-docker/dockerhub/README.md b/nifi-docker/dockerhub/README.md
index 3a4cd74..4e58ed9 100644
--- a/nifi-docker/dockerhub/README.md
+++ b/nifi-docker/dockerhub/README.md
@@ -124,6 +124,29 @@ volume to provide certificates on the host system to the container instance.
-e LDAP_TLS_TRUSTSTORE_PASSWORD: ''
-e LDAP_TLS_TRUSTSTORE_TYPE: ''
+#### Clustering can be enabled by using the following properties to Docker environment variable mappings.
+
+##### nifi.properties
+
+| Property | Environment Variable |
+|-------------------------------------------|----------------------------------------|
+| nifi.cluster.is.node | NIFI_CLUSTER_IS_NODE |
+| nifi.cluster.node.address | NIFI_CLUSTER_ADDRESS |
+| nifi.cluster.node.protocol.port | NIFI_CLUSTER_NODE_PROTOCOL_PORT |
+| nifi.cluster.node.protocol.threads | NIFI_CLUSTER_NODE_PROTOCOL_THREADS |
+| nifi.cluster.node.protocol.max.threads | NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS |
+| nifi.zookeeper.connect.string | NIFI_ZK_CONNECT_STRING |
+| nifi.zookeeper.root.node | NIFI_ZK_ROOT_NODE |
+| nifi.cluster.flow.election.max.wait.time | NIFI_ELECTION_MAX_WAIT |
+| nifi.cluster.flow.election.max.candidates | NIFI_ELECTION_MAX_CANDIDATES |
+
+##### state-management.xml
+
+| Property Name | Environment Variable |
+|----------------|------------------------|
+| Connect String | NIFI_ZK_CONNECT_STRING |
+| Root Node | NIFI_ZK_ROOT_NODE |
+
## Configuration Information
The following ports are specified by default in Docker for NiFi operation within the container and
can be published to the host.
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3222b85/nifi-docker/dockerhub/sh/start.sh
----------------------------------------------------------------------
diff --git a/nifi-docker/dockerhub/sh/start.sh b/nifi-docker/dockerhub/sh/start.sh
index d43f3a1..f65a43e 100755
--- a/nifi-docker/dockerhub/sh/start.sh
+++ b/nifi-docker/dockerhub/sh/start.sh
@@ -27,6 +27,17 @@ prop_replace 'nifi.remote.input.socket.port' "${NIFI_REMOTE_INPUT_SOCKET_PORT
prop_replace 'nifi.remote.input.secure' 'false'
prop_replace 'nifi.variable.registry.properties' "${NIFI_VARIABLE_REGISTRY_PROPERTIES:-}"
+prop_replace 'nifi.cluster.is.node' "${NIFI_CLUSTER_IS_NODE:-false}"
+prop_replace 'nifi.cluster.node.address' "${NIFI_CLUSTER_ADDRESS:-$HOSTNAME}"
+prop_replace 'nifi.cluster.node.protocol.port' "${NIFI_CLUSTER_NODE_PROTOCOL_PORT:-}"
+prop_replace 'nifi.cluster.node.protocol.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_THREADS:-10}"
+prop_replace 'nifi.cluster.node.protocol.max.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS:-50}"
+prop_replace 'nifi.zookeeper.connect.string' "${NIFI_ZK_CONNECT_STRING:-}"
+prop_replace 'nifi.zookeeper.root.node' "${NIFI_ZK_ROOT_NODE:-/nifi}"
+prop_replace 'nifi.cluster.flow.election.max.wait.time' "${NIFI_ELECTION_MAX_WAIT:-5 mins}"
+prop_replace 'nifi.cluster.flow.election.max.candidates' "${NIFI_ELECTION_MAX_CANDIDATES:-}"
+
+. "${scripts_dir}/update_cluster_state_management.sh"
# Check if we are secured or unsecured
case ${AUTH} in
@@ -58,4 +69,4 @@ nifi_pid="$!"
trap "echo Received trapped signal, beginning shutdown...;" KILL TERM HUP INT EXIT;
echo NiFi running with PID ${nifi_pid}.
-wait ${nifi_pid}
\ No newline at end of file
+wait ${nifi_pid}
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3222b85/nifi-docker/dockerhub/sh/update_cluster_state_management.sh
----------------------------------------------------------------------
diff --git a/nifi-docker/dockerhub/sh/update_cluster_state_management.sh b/nifi-docker/dockerhub/sh/update_cluster_state_management.sh
new file mode 100755
index 0000000..718e52d
--- /dev/null
+++ b/nifi-docker/dockerhub/sh/update_cluster_state_management.sh
@@ -0,0 +1,31 @@
+#!/bin/sh -e
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+state_providers_file=${NIFI_HOME}/conf/state-management.xml
+property_xpath='/stateManagement/cluster-provider/property'
+
+edit_property() {
+ property_name=$1
+ property_value=$2
+
+ if [ -n "${property_value}" ]; then
+ xmlstarlet ed --inplace -u "${property_xpath}[@name='${property_name}']" -v "${property_value}" "${state_providers_file}"
+ fi
+}
+
+edit_property 'Connect String' "${NIFI_ZK_CONNECT_STRING}"
+edit_property "Root Node" "${NIFI_ZK_ROOT_NODE}"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3222b85/nifi-docker/dockermaven/sh/start.sh
----------------------------------------------------------------------
diff --git a/nifi-docker/dockermaven/sh/start.sh b/nifi-docker/dockermaven/sh/start.sh
index d43f3a1..f65a43e 100755
--- a/nifi-docker/dockermaven/sh/start.sh
+++ b/nifi-docker/dockermaven/sh/start.sh
@@ -27,6 +27,17 @@ prop_replace 'nifi.remote.input.socket.port' "${NIFI_REMOTE_INPUT_SOCKET_PORT
prop_replace 'nifi.remote.input.secure' 'false'
prop_replace 'nifi.variable.registry.properties' "${NIFI_VARIABLE_REGISTRY_PROPERTIES:-}"
+prop_replace 'nifi.cluster.is.node' "${NIFI_CLUSTER_IS_NODE:-false}"
+prop_replace 'nifi.cluster.node.address' "${NIFI_CLUSTER_ADDRESS:-$HOSTNAME}"
+prop_replace 'nifi.cluster.node.protocol.port' "${NIFI_CLUSTER_NODE_PROTOCOL_PORT:-}"
+prop_replace 'nifi.cluster.node.protocol.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_THREADS:-10}"
+prop_replace 'nifi.cluster.node.protocol.max.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS:-50}"
+prop_replace 'nifi.zookeeper.connect.string' "${NIFI_ZK_CONNECT_STRING:-}"
+prop_replace 'nifi.zookeeper.root.node' "${NIFI_ZK_ROOT_NODE:-/nifi}"
+prop_replace 'nifi.cluster.flow.election.max.wait.time' "${NIFI_ELECTION_MAX_WAIT:-5 mins}"
+prop_replace 'nifi.cluster.flow.election.max.candidates' "${NIFI_ELECTION_MAX_CANDIDATES:-}"
+
+. "${scripts_dir}/update_cluster_state_management.sh"
# Check if we are secured or unsecured
case ${AUTH} in
@@ -58,4 +69,4 @@ nifi_pid="$!"
trap "echo Received trapped signal, beginning shutdown...;" KILL TERM HUP INT EXIT;
echo NiFi running with PID ${nifi_pid}.
-wait ${nifi_pid}
\ No newline at end of file
+wait ${nifi_pid}
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3222b85/nifi-docker/dockermaven/sh/update_cluster_state_management.sh
----------------------------------------------------------------------
diff --git a/nifi-docker/dockermaven/sh/update_cluster_state_management.sh b/nifi-docker/dockermaven/sh/update_cluster_state_management.sh
new file mode 100755
index 0000000..718e52d
--- /dev/null
+++ b/nifi-docker/dockermaven/sh/update_cluster_state_management.sh
@@ -0,0 +1,31 @@
+#!/bin/sh -e
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+state_providers_file=${NIFI_HOME}/conf/state-management.xml
+property_xpath='/stateManagement/cluster-provider/property'
+
+edit_property() {
+ property_name=$1
+ property_value=$2
+
+ if [ -n "${property_value}" ]; then
+ xmlstarlet ed --inplace -u "${property_xpath}[@name='${property_name}']" -v "${property_value}" "${state_providers_file}"
+ fi
+}
+
+edit_property 'Connect String' "${NIFI_ZK_CONNECT_STRING}"
+edit_property "Root Node" "${NIFI_ZK_ROOT_NODE}"
\ No newline at end of file