You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@heron.apache.org by ni...@apache.org on 2021/04/13 04:04:07 UTC
[incubator-heron] branch nicknezis/zookeeper-helm-fix created (now
940a564)
This is an automated email from the ASF dual-hosted git repository.
nicknezis pushed a change to branch nicknezis/zookeeper-helm-fix
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git.
at 940a564 Updates to make Zookeeper configurable through the Helm chart logic
This branch includes the following new commits:
new 940a564 Updates to make Zookeeper configurable through the Helm chart logic
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-heron] 01/01: Updates to make Zookeeper configurable
through the Helm chart logic
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nicknezis pushed a commit to branch nicknezis/zookeeper-helm-fix
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
commit 940a56484474cdc9e39af5fa9cb8dc5a103b6bbe
Author: Nicholas Nezis <ni...@gmail.com>
AuthorDate: Tue Apr 13 00:03:42 2021 -0400
Updates to make Zookeeper configurable through the Helm chart logic
---
docker/base/scripts/generate-zookeeper-config.sh | 2 +
docker/base/scripts/install-zookeeper.sh | 8 +-
docker/dist/scripts/generate-zookeeper-config.sh | 203 +++++++++++++++++------
3 files changed, 160 insertions(+), 53 deletions(-)
diff --git a/docker/base/scripts/generate-zookeeper-config.sh b/docker/base/scripts/generate-zookeeper-config.sh
index 02febd1..c4dda6b 100644
--- a/docker/base/scripts/generate-zookeeper-config.sh
+++ b/docker/base/scripts/generate-zookeeper-config.sh
@@ -98,6 +98,8 @@ function create_config() {
echo "maxSessionTimeout=$ZK_MAX_SESSION_TIMEOUT" >> $ZK_CONFIG_FILE
echo "autopurge.snapRetainCount=$ZK_SNAP_RETAIN_COUNT" >> $ZK_CONFIG_FILE
echo "autopurge.purgeInteval=$ZK_PURGE_INTERVAL" >> $ZK_CONFIG_FILE
+ echo "# Enable all four letter word commands by default" >> $ZK_CONFIG_FILE
+ echo "4lw.commands.whitelist=*" >> $ZK_CONFIG_FILE
if [ $ZK_REPLICAS -gt 1 ]; then
print_servers >> $ZK_CONFIG_FILE
diff --git a/docker/base/scripts/install-zookeeper.sh b/docker/base/scripts/install-zookeeper.sh
index ea86e91..4e772fb 100644
--- a/docker/base/scripts/install-zookeeper.sh
+++ b/docker/base/scripts/install-zookeeper.sh
@@ -40,10 +40,10 @@ rm -rf \
# copy zk scripts
mkdir -p /opt/zookeeper/scripts
cp /opt/heron-docker/scripts/generate-zookeeper-config.sh /opt/zookeeper/scripts/
-chmod +x /opt/heron-docker/scripts/generate-zookeeper-config.sh
+chmod +x /opt/zookeeper/scripts/generate-zookeeper-config.sh
cp /opt/heron-docker/scripts/zookeeper-ruok.sh /opt/zookeeper/scripts/
-chmod +x /opt/heron-docker/scripts/zookeeper-ruok.sh
+chmod +x /opt/zookeeper/scripts/zookeeper-ruok.sh
cp /opt/heron-docker/scripts/start-zookeeper.sh /opt/zookeeper/scripts/
-chmod +x /opt/heron-docker/scripts/start-zookeeper.sh
+chmod +x /opt/zookeeper/scripts/start-zookeeper.sh
cp /opt/heron-docker/scripts/wait-for-zookeeper.sh /opt/zookeeper/scripts/
-chmod +x /opt/heron-docker/scripts/wait-for-zookeeper.sh
+chmod +x /opt/zookeeper/scripts/wait-for-zookeeper.sh
\ No newline at end of file
diff --git a/docker/dist/scripts/generate-zookeeper-config.sh b/docker/dist/scripts/generate-zookeeper-config.sh
index 678707a..40b5d22 100755
--- a/docker/dist/scripts/generate-zookeeper-config.sh
+++ b/docker/dist/scripts/generate-zookeeper-config.sh
@@ -1,54 +1,159 @@
#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Apply env variables to config file and start the regular command
-
-CONF_FILE=$1
-
-if [ $? != 0 ]; then
- echo "Error: Failed to apply changes to config file"
- exit 1
-fi
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+ZK_USER=${ZK_USER:-"zookeeper"}
+ZK_LOG_LEVEL=${ZK_LOG_LEVEL:-"INFO"}
+ZK_DATA_DIR=${ZK_DATA_DIR:-"/var/lib/zookeeper/data"}
+ZK_DATA_LOG_DIR=${ZK_DATA_LOG_DIR:-"/var/lib/zookeeper/log"}
+ZK_LOG_DIR=${ZK_LOG_DIR:-"var/log/zookeeper"}
+ZK_CONF_DIR=${ZK_CONF_DIR:-"/opt/zookeeper/conf"}
+ZK_CLIENT_PORT=${ZK_CLIENT_PORT:-2181}
+ZK_SERVER_PORT=${ZK_SERVER_PORT:-2888}
+ZK_ELECTION_PORT=${ZK_ELECTION_PORT:-3888}
+ZK_TICK_TIME=${ZK_TICK_TIME:-2000}
+ZK_INIT_LIMIT=${ZK_INIT_LIMIT:-10}
+ZK_SYNC_LIMIT=${ZK_SYNC_LIMIT:-5}
+ZK_HEAP_SIZE=${ZK_HEAP_SIZE:-2G}
+ZK_MAX_CLIENT_CNXNS=${ZK_MAX_CLIENT_CNXNS:-60}
+ZK_MIN_SESSION_TIMEOUT=${ZK_MIN_SESSION_TIMEOUT:-$((ZK_TICK_TIME*2))}
+ZK_MAX_SESSION_TIMEOUT=${ZK_MAX_SESSION_TIMEOUT:-$((ZK_TICK_TIME*20))}
+ZK_SNAP_RETAIN_COUNT=${ZK_SNAP_RETAIN_COUNT:-3}
+ZK_PURGE_INTERVAL=${ZK_PURGE_INTERVAL:-0}
+ID_FILE="$ZK_DATA_DIR/myid"
+ZK_CONFIG_FILE="$ZK_CONF_DIR/zookeeper.conf"
+LOGGER_PROPS_FILE="$ZK_CONF_DIR/log4j.properties"
+JAVA_ENV_FILE="$ZK_CONF_DIR/java.env"
+HOST=`hostname -s`
DOMAIN=`hostname -d`
-# Generate list of servers and detect the current server ID,
-# based on the hostname
-IDX=1
-for SERVER in $(echo $ZOOKEEPER_SERVERS | tr "," "\n")
-do
- echo "server.$IDX=$SERVER.$DOMAIN:2888:3888" >> $CONF_FILE
-
- if [ "$HOSTNAME" == "$SERVER" ]; then
- MY_ID=$IDX
- echo "Current server id $MY_ID"
- fi
-
- ((IDX++))
-done
-
-# For ZooKeeper container we need to initialize the ZK id
-if [ ! -z "$MY_ID" ]; then
- # Get ZK data dir
- DATA_DIR=`grep '^dataDir=' $CONF_FILE | awk -F= '{print $2}'`
- if [ ! -e $DATA_DIR/myid ]; then
- echo "Creating $DATA_DIR/myid with id = $MY_ID"
- mkdir -p $DATA_DIR
- echo $MY_ID > $DATA_DIR/myid
- fi
+function print_servers() {
+ for (( i=1; i<=$ZK_REPLICAS; i++ ))
+ do
+ echo "server.$i=$NAME-$((i-1)).$DOMAIN:$ZK_SERVER_PORT:$ZK_ELECTION_PORT"
+ done
+}
+
+function validate_env() {
+ echo "Validating enviornment"
+ if [ -z $ZK_REPLICAS ]; then
+ echo "ZK_REPLICAS is a mandatory environment variable"
+ exit 1
+ fi
+
+ if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then
+ NAME=${BASH_REMATCH[1]}
+ ORD=${BASH_REMATCH[2]}
+ else
+ echo "Failed to extract ordinal from hostname $HOST"
+ exit 1
+ fi
+ MY_ID=$((ORD+1))
+ echo "ZK_REPLICAS=$ZK_REPLICAS"
+ echo "MY_ID=$MY_ID"
+ echo "ZK_LOG_LEVEL=$ZK_LOG_LEVEL"
+ echo "ZK_DATA_DIR=$ZK_DATA_DIR"
+ echo "ZK_DATA_LOG_DIR=$ZK_DATA_LOG_DIR"
+ echo "ZK_LOG_DIR=$ZK_LOG_DIR"
+ echo "ZK_CLIENT_PORT=$ZK_CLIENT_PORT"
+ echo "ZK_SERVER_PORT=$ZK_SERVER_PORT"
+ echo "ZK_ELECTION_PORT=$ZK_ELECTION_PORT"
+ echo "ZK_TICK_TIME=$ZK_TICK_TIME"
+ echo "ZK_INIT_LIMIT=$ZK_INIT_LIMIT"
+ echo "ZK_SYNC_LIMIT=$ZK_SYNC_LIMIT"
+ echo "ZK_MAX_CLIENT_CNXNS=$ZK_MAX_CLIENT_CNXNS"
+ echo "ZK_MIN_SESSION_TIMEOUT=$ZK_MIN_SESSION_TIMEOUT"
+ echo "ZK_MAX_SESSION_TIMEOUT=$ZK_MAX_SESSION_TIMEOUT"
+ echo "ZK_HEAP_SIZE=$ZK_HEAP_SIZE"
+ echo "ZK_SNAP_RETAIN_COUNT=$ZK_SNAP_RETAIN_COUNT"
+ echo "ZK_PURGE_INTERVAL=$ZK_PURGE_INTERVAL"
+ echo "ENSEMBLE"
+ print_servers
+ echo "Enviorment validation successful"
+}
+
+function create_config() {
+ rm -f $ZK_CONFIG_FILE
+ echo "Creating ZooKeeper configuration"
+ echo "#This file was autogenerated by k8szk DO NOT EDIT" >> $ZK_CONFIG_FILE
+ echo "clientPort=$ZK_CLIENT_PORT" >> $ZK_CONFIG_FILE
+ echo "dataDir=$ZK_DATA_DIR" >> $ZK_CONFIG_FILE
+ echo "dataLogDir=$ZK_DATA_LOG_DIR" >> $ZK_CONFIG_FILE
+ echo "tickTime=$ZK_TICK_TIME" >> $ZK_CONFIG_FILE
+ echo "initLimit=$ZK_INIT_LIMIT" >> $ZK_CONFIG_FILE
+ echo "syncLimit=$ZK_SYNC_LIMIT" >> $ZK_CONFIG_FILE
+ echo "maxClientCnxns=$ZK_MAX_CLIENT_CNXNS" >> $ZK_CONFIG_FILE
+ echo "minSessionTimeout=$ZK_MIN_SESSION_TIMEOUT" >> $ZK_CONFIG_FILE
+ echo "maxSessionTimeout=$ZK_MAX_SESSION_TIMEOUT" >> $ZK_CONFIG_FILE
+ echo "autopurge.snapRetainCount=$ZK_SNAP_RETAIN_COUNT" >> $ZK_CONFIG_FILE
+ echo "autopurge.purgeInteval=$ZK_PURGE_INTERVAL" >> $ZK_CONFIG_FILE
+ echo "# Enable all four letter word commands by default" >> $ZK_CONFIG_FILE
+ echo "4lw.commands.whitelist=*" >> $ZK_CONFIG_FILE
+
+ if [ $ZK_REPLICAS -gt 1 ]; then
+ print_servers >> $ZK_CONFIG_FILE
+ fi
+ echo "Wrote ZooKeeper configuration file to $ZK_CONFIG_FILE"
+}
+
+function create_data_dirs() {
+ echo "Creating ZooKeeper data directories and setting permissions"
+ if [ ! -d $ZK_DATA_DIR ]; then
+ mkdir -p $ZK_DATA_DIR
+ chown -R $ZK_USER:$ZK_USER $ZK_DATA_DIR
+ fi
+
+ if [ ! -d $ZK_DATA_LOG_DIR ]; then
+ mkdir -p $ZK_DATA_LOG_DIR
+ chown -R $ZK_USER:$ZK_USER $ZK_DATA_LOG_DIR
+ fi
+
+ if [ ! -d $ZK_LOG_DIR ]; then
+ mkdir -p $ZK_LOG_DIR
+ chown -R $ZK_USER:$ZK_USER $ZK_LOG_DIR
+ fi
+ if [ ! -f $ID_FILE ]; then
+ echo $MY_ID >> $ID_FILE
+ fi
+ echo "Created ZooKeeper data directories and set permissions in $ZK_DATA_DIR"
+}
+
+function create_log_props () {
+ rm -f $LOGGER_PROPS_FILE
+ echo "Creating ZooKeeper log4j configuration"
+ echo "zookeeper.root.logger=CONSOLE" >> $LOGGER_PROPS_FILE
+ echo "zookeeper.console.threshold="$ZK_LOG_LEVEL >> $LOGGER_PROPS_FILE
+ echo "log4j.rootLogger=\${zookeeper.root.logger}" >> $LOGGER_PROPS_FILE
+ echo "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender" >> $LOGGER_PROPS_FILE
+ echo "log4j.appender.CONSOLE.Threshold=\${zookeeper.console.threshold}" >> $LOGGER_PROPS_FILE
+ echo "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout" >> $LOGGER_PROPS_FILE
+ echo "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n" >> $LOGGER_PROPS_FILE
+ echo "Wrote log4j configuration to $LOGGER_PROPS_FILE"
+}
+
+function create_java_env() {
+ rm -f $JAVA_ENV_FILE
+ echo "Creating JVM configuration file"
+ echo "ZOO_LOG_DIR=$ZK_LOG_DIR" >> $JAVA_ENV_FILE
+ echo "JVMFLAGS=\"-Xmx$ZK_HEAP_SIZE -Xms$ZK_HEAP_SIZE\"" >> $JAVA_ENV_FILE
+ echo "Wrote JVM configuration to $JAVA_ENV_FILE"
+}
+
+if [ "$#" -ne 1 ]; then
+ echo "Usage: $0 <configuration-file>"
+ exit 1
fi
+
+ZK_CONFIG_FILE=$1
+validate_env && create_config && create_log_props && create_data_dirs && create_java_env