You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by am...@apache.org on 2016/07/27 17:10:50 UTC
[1/2] incubator-trafodion git commit: TRAFODION 2001 (Elasticity) and
small changes
Repository: incubator-trafodion
Updated Branches:
refs/heads/master 522acc9a7 -> 657510219
TRAFODION 2001 (Elasticity) and small changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/07221f21
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/07221f21
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/07221f21
Branch: refs/heads/master
Commit: 07221f2114124a36e7efc01fb1d925edf72d4b8e
Parents: 0ec52bc
Author: Amanda Moran <am...@apache.com>
Authored: Mon Jul 25 20:01:31 2016 +0000
Committer: Amanda Moran <am...@apache.com>
Committed: Mon Jul 25 20:01:31 2016 +0000
----------------------------------------------------------------------
install/installer/addNode_packages | 89 ++++++
install/installer/addNode_reservePorts | 81 ++++++
install/installer/addNode_step1 | 294 +++++++++++++++++++
install/installer/addNode_step2 | 372 ++++++++++++++++++++++++
install/installer/dcs_installer | 4 +
install/installer/parseHBaseSite.py | 5 +
install/installer/rest_installer | 4 +
install/installer/tools/ambari_setup | 2 +-
install/installer/traf_apache_mods | 2 +-
install/installer/traf_cloudera_mods | 2 +-
install/installer/traf_config | 11 +
install/installer/traf_config_check | 19 ++
install/installer/traf_config_setup | 33 ++-
install/installer/traf_hortonworks_mods | 55 +++-
install/installer/traf_sqconfig | 8 +
install/installer/traf_start | 30 +-
install/installer/trafodion_config_default | 7 +
17 files changed, 975 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/07221f21/install/installer/addNode_packages
----------------------------------------------------------------------
diff --git a/install/installer/addNode_packages b/install/installer/addNode_packages
new file mode 100755
index 0000000..dc777d1
--- /dev/null
+++ b/install/installer/addNode_packages
@@ -0,0 +1,89 @@
+#!/bin/bash
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+
+#==============================================
+
+timestamp=$(date +%F-%H-%M-%S)
+export TRAF_CONFIG="/etc/trafodion/trafodion_config"
+source $TRAF_CONFIG
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --nodes)
+ MY_NEW_NODES=""
+ NEW_NODES="$2"
+
+ for node in $NEW_NODES
+ do
+ MY_NEW_NODES="$MY_NEW_NODES -w $node"
+ done
+
+ export PDSH_SSH_CMD="ssh -q -n %h"
+
+ export TRAF_NEW_PDSH="pdsh -R exec $MY_NEW_NODES $PDSH_SSH_CMD"
+ export TRAF_NEW_PDCP="pdcp -R ssh $MY_NEW_NODES"
+ shift
+ ;;
+ *)
+ echo "***ERROR: unknown parameter '$1'"
+ print_usage
+ exit -1
+ esac
+ shift
+done
+
+#========================================
+echo "***INFO: Installing required packages"
+echo "***INFO: Log file location $PWD"
+
+YUM_LOG=$PWD/trafodion_yum_$timestamp.log
+
+#===============================
+
+$TRAF_NEW_PDSH sudo yum --assumeyes install epel-release >> $YUM_LOG
+$TRAF_NEW_PDSH sudo yum --assumeyes install pdsh >> $YUM_LOG
+
+if [ $? != 0 ]; then
+ echo "***ERROR: Can't install pdsh on $NEW_NODES"
+ exit -1
+fi
+
+package_list="apr apr-util sqlite expect perl-DBD-SQLite* protobuf xerces-c perl-Params-Validate perl-Time-HiRes gzip lzo lzop unzip unixODBC libiodbc libiodbc-devel openldap-clients snappy lsof gnuplot"
+
+for package in $package_list
+do
+ echo "***INFO: Checking if $package is installed ..."
+ for node in $NEW_NODES
+ do
+ installed=`ssh -q -n $node sudo rpm -qa | grep $package | wc -l`
+ if [[ $installed -eq "0" ]]; then
+ echo "***INFO: Installing $package on all nodes"
+ ssh -q -n $node sudo yum --assumeyes install $package 2>> $YUM_LOG >> $YUM_LOG
+ if [ $? != 0 ]; then
+ echo "***ERROR: Can't install $package"
+ exit -1
+ fi
+ fi
+ done
+done
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/07221f21/install/installer/addNode_reservePorts
----------------------------------------------------------------------
diff --git a/install/installer/addNode_reservePorts b/install/installer/addNode_reservePorts
new file mode 100755
index 0000000..bde0826
--- /dev/null
+++ b/install/installer/addNode_reservePorts
@@ -0,0 +1,81 @@
+#!/bin/bash
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+
+#==============================================
+
+timestamp=$(date +%F-%H-%M-%S)
+export TRAF_CONFIG="/etc/trafodion/trafodion_config"
+source $TRAF_CONFIG
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --nodes)
+ MY_NEW_NODES=""
+ NEW_NODES="$2"
+
+ for node in $NEW_NODES
+ do
+ MY_NEW_NODES="$MY_NEW_NODES -w $node"
+ done
+
+ export PDSH_SSH_CMD="ssh -q -n %h"
+
+ export TRAF_NEW_PDSH="pdsh -R exec $MY_NEW_NODES $PDSH_SSH_CMD"
+ export TRAF_NEW_PDCP="pdcp -R ssh $MY_NEW_NODES"
+ shift
+ ;;
+ *)
+ echo "***ERROR: unknown parameter '$1'"
+ print_usage
+ exit -1
+ esac
+ shift
+done
+
+#========================================
+echo "***INFO: Reserving DCS ports"
+trafodion_ports=${DCS_MASTER_PORT:-23400},${DCS_MASTER_INFO_PORT:-24400}
+reservedTrafodionPorts="net.ipv4.ip_local_reserved_ports = ${trafodion_ports}"
+
+for node in $NEW_NODES
+do
+ redhatSystem=$(ssh -q -n $node '/sbin/sysctl net.ipv4.ip_local_reserved_ports | wc -l')
+ if [[ "$redhatSystem" -ge "1" ]]; then
+ reserved_ports=$(ssh -q -n $node '/sbin/sysctl net.ipv4.ip_local_reserved_ports' | awk -F'=' '{print $2;}' | sed 's/^ //')
+ ssh -q -n $node 'sudo chmod 777 /etc/sysctl.conf' 2>&1 > /dev/null
+ ssh -q -n $node 'sudo sed -i '/net.ipv4.ip_local_reserved_ports""\=*/d' /etc/sysctl.conf' 2>&1 > /dev/null
+
+ if [[ -z "${reserved_ports}" ]]; then
+ ssh -q -n $node "sudo echo "$reservedTrafodionPorts" >> /etc/sysctl.conf" 2>&1 > /dev/null
+ else
+ ssh -q -n $node "sudo echo "$reservedTrafodionPorts","$reserved_ports" >> /etc/sysctl.conf" 2>&1 > /dev/null
+ fi
+ ssh -q -n $node 'sudo chmod 644 /etc/sysctl.conf' 2>&1 > /dev/null
+ ssh -q -n $node 'sudo /sbin/sysctl -p' 2>&1 > /dev/null
+ else
+ echo "***ERROR: Unable to reserve DCS ports (${trafodion_ports})."
+ exit -1
+ fi
+done
+
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/07221f21/install/installer/addNode_step1
----------------------------------------------------------------------
diff --git a/install/installer/addNode_step1 b/install/installer/addNode_step1
new file mode 100755
index 0000000..e5c3dd7
--- /dev/null
+++ b/install/installer/addNode_step1
@@ -0,0 +1,294 @@
+#!/bin/bash
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+
+export TRAF_CONFIG="/etc/trafodion/trafodion_config"
+export NODES=""
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --nodes)
+ NODES=$2
+ shift
+ ;;
+ *)
+ echo "***ERROR: unknown parameter '$1'"
+ print_usage
+ exit -1
+ esac
+ shift
+done
+
+source $TRAF_CONFIG
+
+function addNewNodesConfig {
+
+source $TRAF_CONFIG
+
+MY_NEW_NODES=""
+NEW_NODES="$NODES"
+
+for node in $NODES
+do
+ MY_NEW_NODES="$MY_NEW_NODES -w $node"
+done
+
+echo $MY_NEW_NODES
+
+export PDSH_SSH_CMD="ssh -q -n %h"
+
+export TRAF_NEW_PDSH="pdsh -R exec $MY_NEW_NODES $PDSH_SSH_CMD"
+export TRAF_NEW_PDCP="pdcp -R ssh $MY_NEW_NODES"
+
+
+sudo chmod 777 $TRAF_CONFIG
+sed -i '/NEW_NODES\=/d' $TRAF_CONFIG
+echo "export NEW_NODES=\"$NEW_NODES\"" >> $TRAF_CONFIG
+sudo chmod 777 $TRAF_CONFIG
+sed -i '/MY_NEW_NODES\=/d' $TRAF_CONFIG
+echo "export MY_NEW_NODES=\"$MY_NEW_NODES\"" >> $TRAF_CONFIG
+sudo chmod 777 $TRAF_CONFIG
+sed -i '/TRAF_NEW_PDSH\=/d' $TRAF_CONFIG
+echo "export TRAF_NEW_PDSH=\"$TRAF_NEW_PDSH\"" >> $TRAF_CONFIG
+sudo chmod 777 $TRAF_CONFIG
+sed -i '/TRAF_NEW_PDCP\=/d' $TRAF_CONFIG
+echo "export TRAF_NEW_PDCP=\"$TRAF_NEW_PDCP\"" >> $TRAF_CONFIG
+sudo chmod 777 $TRAF_CONFIG
+}
+
+function copyHBaseTrx {
+
+source $TRAF_CONFIG
+
+cp $HADOOP_PATH/hbase-trx* $PWD
+cp $HADOOP_PATH/trafodion-utility* $PWD
+
+$TRAF_NEW_PDCP $PWD/hbase-trx* $HOME
+$TRAF_NEW_PDCP $PWD/trafodion-utility* $HOME
+
+$TRAF_NEW_PDSH sudo cp $HOME/hbase-trx* $HADOOP_PATH
+$TRAF_NEW_PDSH sudo cp $HOME/trafodion-utility* $HADOOP_PATH
+
+$TRAF_NEW_PDSH sudo chmod 644 $HADOOP_PATH/hbase-trx*
+$TRAF_NEW_PDSH sudo chmod 644 $HADOOP_PATH/trafodion-utility*
+
+}
+
+function createNewNodes {
+
+source $TRAF_CONFIG
+
+for node in $NODES
+do
+ NODE_LIST="$NODE_LIST $node"
+ MY_NODES="$MY_NODES -w $node"
+ ALL_NODE_LIST="$ALL_NODE_LIST $node"
+ ALL_NODES="$ALL_NODES -w $node"
+ HADOOP_NODES="$HADOOP_NODES $node"
+ MY_HADOOP_NODES="$MY_HADOOP_NODES -w $node"
+ HDFS_NODES="$HDFS_NODES $node"
+ MY_HDFS_NODES="$MY_HDFS_NODES -w $node"
+ HBASE_NODES="$HBASE_NODES $node"
+ MY_HBASE_NODES="$MY_HBASE_NODES -w $node"
+done
+
+hadoop_node_count=$(echo $HADOOP_NODES | wc -w)
+export TRAF_PDSH="pdsh -R exec $ALL_NODES $PDSH_SSH_CMD"
+export TRAF_PDCP="pdcp -R ssh $ALL_NODES"
+sudo chmod 777 $TRAF_CONFIG
+sed -i '/TRAF_PDSH\=/d' $TRAF_CONFIG
+echo "export TRAF_PDSH=\"$TRAF_PDSH\"" >> $TRAF_CONFIG
+sudo chmod 777 $TRAF_CONFIG
+sed -i '/TRAF_PDCP\=/d' $TRAF_CONFIG
+echo "export TRAF_PDCP=\"$TRAF_PDCP\"" >> $TRAF_CONFIG
+sudo chmod 777 $TRAF_CONFIG
+sed -i '/HADOOP_NODES\=/d' $TRAF_CONFIG
+echo "export HADOOP_NODES=\"$HADOOP_NODES\"" >> $TRAF_CONFIG
+sudo chmod 777 $TRAF_CONFIG
+sed -i '/MY_HADOOP_NODES\=/d' $TRAF_CONFIG
+echo "export MY_HADOOP_NODES=\"$MY_HADOOP_NODES\"" >> $TRAF_CONFIG
+sudo chmod 777 $TRAF_CONFIG
+sed -i '/HDFS_NODES\=/d' $TRAF_CONFIG
+echo "export HDFS_NODES=\"$HDFS_NODES\"" >> $TRAF_CONFIG
+sudo chmod 777 $TRAF_CONFIG
+sed -i '/MY_HDFS_NODES\=/d' $TRAF_CONFIG
+echo "export MY_HDFS_NODES=\"$MY_HDFS_NODES\"" >> $TRAF_CONFIG
+sudo chmod 777 $TRAF_CONFIG
+sed -i '/HBASE_NODES\=/d' $TRAF_CONFIG
+echo "export HBASE_NODES=\"$HBASE_NODES\"" >> $TRAF_CONFIG
+sudo chmod 777 $TRAF_CONFIG
+sed -i '/MY_HBASE_NODES\=/d' $TRAF_CONFIG
+echo "export MY_HBASE_NODES=\"$MY_HBASE_NODES\"" >> $TRAF_CONFIG
+sudo chmod 777 $TRAF_CONFIG
+sed -i '/ALL_NODES\=/d' $TRAF_CONFIG
+echo "export ALL_NODES=\"$ALL_NODES\"" >> $TRAF_CONFIG
+sudo chmod 777 $TRAF_CONFIG
+sed -i '/ALL_NODE_LIST\=/d' $TRAF_CONFIG
+echo "export ALL_NODE_LIST=\"$ALL_NODE_LIST\"" >> $TRAF_CONFIG
+sudo chmod 777 $TRAF_CONFIG
+sed -i '/MY_NODES\=/d' $TRAF_CONFIG
+echo "export MY_NODES=\"$MY_NODES\"" >> $TRAF_CONFIG
+sudo chmod 777 $TRAF_CONFIG
+sed -i '/NODE_LIST\=/d' $TRAF_CONFIG
+echo "export NODE_LIST=\"$NODE_LIST\"" >> $TRAF_CONFIG
+sudo chmod 777 $TRAF_CONFIG
+sed -i '/hadoop_node_count\=/d' $TRAF_CONFIG
+echo "export hadoop_node_count=\"$hadoop_node_count\"" >> $TRAF_CONFIG
+sudo chmod 777 $TRAF_CONFIG
+
+}
+
+function removeNodeExclude {
+
+source $TRAF_CONFIG
+temp=""
+
+for node in $TRAF_EXCLUDE_LIST
+do
+ if [[ ! $NODES =~ .*$node.* ]]; then
+ temp="$temp $node"
+ fi
+
+done
+excludeCount=$(echo $TRAF_EXCLUDE_LIST | wc -w)
+tempCount=$(echo $temp | wc -w)
+
+
+if [[ "$excludeCount" -eq "$tempCount" ]]; then
+#New node name
+ changeNodeName
+ NAME_CHANGE="true"
+ sudo chmod 777 $TRAF_CONFIG
+ sed -i '/NAME_CHANGE\=/d' $TRAF_CONFIG
+ echo "export NAME_CHANGE=\"$NAME_CHANGE\"" >> $TRAF_CONFIG
+ sudo chmod 777 $TRAF_CONFIG
+else
+ TRAF_EXCLUDE_LIST=$temp
+ sudo chmod 777 $TRAF_CONFIG
+ sed -i '/TRAF_EXCLUDE_LIST\=/d' $TRAF_CONFIG
+ echo "export TRAF_EXCLUDE_LIST=\"$TRAF_EXCLUDE_LIST\"" >> $TRAF_CONFIG
+ sudo chmod 777 $TRAF_CONFIG
+fi
+
+}
+
+function changeNodeName {
+
+source $TRAF_CONFIG
+
+for node in $NODES
+do
+
+ export oldNODENAME=$(echo $TRAF_EXCLUDE_LIST | head -n1 | awk '{print $1;}')
+ temp=""
+
+ for nodes in $TRAF_EXCLUDE_LIST
+ do
+ if [[ "$oldNODENAME" != $nodes ]]; then
+ temp="$temp $nodes"
+ fi
+ done
+ export TRAF_EXCLUDE_LIST=$temp
+
+ oldToNew="$oldNODENAME $node"
+ echo
+ echo "********************************************************************"
+ echo "***VERY IMPORTANT: User must change old user name to a new user name."
+ echo "***VERY IMPORTANT: open another session and sudo su trafodion"
+ echo "***VERY IMPORTANT: sqshell -a"
+ echo "***VERY IMPORTANT: node name $oldToNew"
+ sleep 5
+ echo -n "***VERY IMPORTANT: Press any key when complete."
+ read answer
+ echo
+ echo "********************************************************************"
+
+ sudo sed -i -e "s/\<$oldNODENAME\>/$node/g" $SQ_ROOT/tmp/cluster.conf
+ sudo sed -i -e "s/\<$oldNODENAME\>/$node/g" $SQ_ROOT/sql/scripts/sqconfig
+ sudo rm -rf $SQ_ROOT/tmp/se*
+ sudo rm -rf $SQ_ROOT/sql/scripts/se*
+ sudo chmod 777 $TRAF_CONFIG
+ sed -i '/TRAF_EXCLUDE_LIST\=/d' $TRAF_CONFIG
+ echo "export TRAF_EXCLUDE_LIST=\"$TRAF_EXCLUDE_LIST\"" >> $TRAF_CONFIG
+ sudo chmod 777 $TRAF_CONFIG
+
+done
+
+}
+
+function createConfigDir {
+
+$TRAF_NEW_PDSH sudo mkdir -p /etc/trafodion
+
+}
+
+function copyConfig {
+
+source $TRAF_CONFIG
+
+cp $TRAF_CONFIG $PWD
+$TRAF_PDCP $PWD/trafodion_config $HOME
+$TRAF_PDSH sudo cp $HOME/trafodion_config /etc/trafodion/
+$TRAF_PDSH sudo chmod 777 $TRAF_CONFIG
+
+}
+
+function checkHBaseTrx {
+
+source $TRAF_CONFIG
+
+for node in $HBASE_NODES
+do
+ copiedOver=$(ssh -q -n $node sudo ls $HADOOP_PATH/hbase-trx* | wc -l)
+ if [[ $copiedOver -ne "1" ]]; then
+ echo "***ERROR: $hbase_trx_jar was not copied on $node"
+ echo "***ERROR: Please investigate why this happened"
+ echo "***ERROR: Trafodion can not start without this. EXITING..."
+ exit -1
+ fi
+done
+
+echo "***INFO: hbase-trx file copied correctly! Huzzah."
+
+}
+
+function printMessage {
+
+source $TRAF_CONFIG
+echo
+echo "***IMPORTANT: Ready to run addNode_step2"
+echo
+
+}
+
+
+#Main
+
+addNewNodesConfig
+copyHBaseTrx
+createNewNodes
+removeNodeExclude
+createConfigDir
+copyConfig
+checkHBaseTrx
+printMessage
+
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/07221f21/install/installer/addNode_step2
----------------------------------------------------------------------
diff --git a/install/installer/addNode_step2 b/install/installer/addNode_step2
new file mode 100755
index 0000000..38551fe
--- /dev/null
+++ b/install/installer/addNode_step2
@@ -0,0 +1,372 @@
+#!/bin/bash
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+
+export TRAF_CONFIG="/etc/trafodion/trafodion_config"
+source $TRAF_CONFIG
+
+function installPackages {
+
+source $TRAF_CONFIG
+
+if [[ "$SUSE_LINUX" == "true" ]]; then
+ echo
+ echo "***WARNING: All needed RPM's must be installed in advance to install Trafodion on SUSE."
+ echo "***WARNING: pdsh apr apr-util sqlite expect perl-DBD-SQLite* xerces-c perl-Params-Validate perl-Time-HiRes gzip lzo lzop unzip unixODBC libiodbc libiodbc-devel openldap-clients snappy"
+ echo "***WARNING: If any of these packages have not been installed. Stop Trafodion installation and install."
+ echo
+ sleep 10
+else
+ $PWD/addNode_packages
+ if [[ "$?" -ne "0" ]]; then
+ echo "***ERROR: Packages were not installed."
+ echo "***ERROR: Review logs in /var/log/trafodion/.."
+ exit -1
+ fi
+fi
+
+}
+
+function copyLimits {
+
+echo "***INFO: Copying /etc/security/limits.d/trafodion.conf to new node(s)"
+echo
+
+source $TRAF_CONFIG
+
+sudo cp /etc/security/limits.d/trafodion.conf $HOME
+$TRAF_NEW_PDCP $HOME/trafodion.conf $HOME
+$TRAF_NEW_PDSH sudo cp trafodion.conf /etc/security/limits.d/
+
+}
+
+function reserveDCSPorts {
+
+source $TRAF_CONFIG
+
+$PWD/addNode_reservePorts
+
+if [[ "$?" -ne "0" ]]; then
+ echo "***ERROR: Not able to reserve DCS ports."
+ exit -1
+fi
+
+
+}
+
+function setKernelMax {
+
+echo "***INFO: Setting kernel.pid_max on new node(s)"
+echo
+
+source $TRAF_CONFIG
+
+$TRAF_NEW_PDSH sudo /sbin/sysctl -w kernel.pid_max=65535 2>&1 > /dev/null
+
+sudo cp /etc/sysctl.conf $HOME 2>&1 > /dev/null
+sudo chown $(whoami).$(whoami) $HOME/sysctl.conf
+
+$TRAF_NEW_PDCP $HOME/sysctl.conf $HOME 2>&1 > /dev/null
+$TRAF_NEW_PDSH sudo mv $HOME/sysctl.conf /etc/sysctl.conf
+$TRAF_NEW_PDSH sudo chown root.root /etc/sysctl.conf
+$TRAF_NEW_PDSH sudo chmod 644 /etc/sysctl.conf
+
+}
+
+function createTrafUserId {
+
+echo "***INFO: Creating $TRAF_USER on new node(s)"
+echo
+
+source $TRAF_CONFIG
+
+TRAF_GROUP="trafodion"
+
+traf_gid=$(id -g $TRAF_USER)
+traf_uid=$(id -u $TRAF_USER)
+
+$TRAF_NEW_PDSH sudo /usr/sbin/groupadd --gid $traf_gid $TRAF_GROUP
+return_code=$?
+if [ $return_code -ne 0 ]; then
+ # if it is not a duplicate group error then we have a problem
+ if [ $return_code -ne 9 ]; then
+ echo "***ERROR: unable to create Trafodion group: $TRAF_GROUP"
+ exit -1
+ fi
+fi
+
+
+$TRAF_NEW_PDSH sudo mkdir -p $HOME_DIR
+TRAF_USER_DIR="$HOME_DIR/$TRAF_USER"
+
+$TRAF_NEW_PDSH sudo /usr/sbin/useradd \
+ --shell /bin/bash \
+ --home $TRAF_USER_DIR -m \
+ --gid $traf_gid \
+ --uid $traf_uid \
+ $TRAF_USER
+return_code=$?
+if [ $return_code -ne 0 ]; then
+ if [ $return_code -eq 9 ]; then
+ echo "***WARNING: username already exists, will use existing userid, $TRAF_USER..."
+ else
+ echo "***ERROR: unable to create Trafodion userid: $TRAF_USER.$TRAF_GROUP, RC=$return_code"
+ exit -1
+ fi
+fi
+
+}
+
+function changeTrafPassword {
+
+source $TRAF_CONFIG
+echo
+echo "***IMPORTANT: You need to set the $TRAF_USER password on $NEW_NODES, set it to what you have on other nodes."
+echo "***IMPORTANT: sudo passwd $TRAF_USER"
+echo
+echo
+sleep 10
+
+}
+
+function editTrafbashrc {
+
+source $TRAF_CONFIG
+
+sudo chmod 777 $HOME_DIR/$TRAF_USER/.bashrc
+sudo sed -i -e "s@export NODE_LIST=.*@export NODE_LIST=\"$NODE_LIST\"@" $HOME_DIR/$TRAF_USER/.bashrc
+sudo sed -i -e "s@export MY_NODES=.*@export MY_NODES=\"$MY_NODES\"@" $HOME_DIR/$TRAF_USER/.bashrc
+
+}
+
+function copyTrafbashrc {
+
+echo "***INFO: Copying over $TRAF_USER .bashrc file"
+
+source $TRAF_CONFIG
+
+sudo cp $HOME_DIR/$TRAF_USER/.bashrc $PWD/tempBashrc
+
+$TRAF_PDCP $PWD/tempBashrc $HOME
+$TRAF_PDSH sudo cp $HOME/tempBashrc $HOME_DIR/$TRAF_USER/.bashrc
+$TRAF_PDSH sudo chmod 644 $HOME_DIR/$TRAF_USER/.bashrc
+
+}
+
+function copyClusterConf {
+
+source $TRAF_CONFIG
+echo "***INFO: Copying over new cluster.conf file"
+
+echo "***IMPORTANT*** Ignore copy errors to new node."
+echo
+sleep 5
+
+sudo cp $SQ_ROOT/tmp/cluster.conf $PWD
+sudo chown $(whoami).$(whoami) $PWD/cluster.conf
+$TRAF_PDCP $PWD/cluster.conf $HOME
+$TRAF_PDSH sudo cp $HOME/cluster.conf $SQ_ROOT/tmp/cluster.conf 2>&1 > /dev/null
+$TRAF_PDSH sudo chown $TRAF_USER.trafodion $SQ_ROOT/tmp/cluster.conf 2>&1 > /dev/null
+$TRAF_PDSH sudo rm $HOME/cluster.conf
+
+}
+
+function copySQconfig {
+
+source $TRAF_CONFIG
+echo "***INFO: Copying over new sqconfig file"
+
+echo "***IMPORTANT*** Ignore copy errors to new node."
+echo
+sleep 5
+
+sudo cp -r $SQ_ROOT/sql/scripts/sqconfig $PWD
+sudo chown $(whoami).$(whoami) $PWD/sqconfig
+$TRAF_PDCP $PWD/sqconfig $HOME
+$TRAF_PDSH sudo cp $HOME/sqconfig $SQ_ROOT/sql/scripts/sqconfig 2>&1 > /dev/null
+$TRAF_PDSH sudo chown $TRAF_USER.trafodion $SQ_ROOT/sql/scripts/sqconfig 2>&1 > /dev/null
+$TRAF_PDSH sudo rm $HOME/sqconfig
+
+
+}
+
+function addTraftoHBase {
+
+echo "***INFO: Adding $TRAF_USER userid to $HBASE_GROUP group"
+
+source $TRAF_CONFIG
+
+$TRAF_NEW_PDSH sudo usermod -a -G $HBASE_GROUP $TRAF_USER
+}
+
+function editDCSServers {
+
+echo "***INFO: Create DCS servers file"
+
+source $TRAF_CONFIG
+DCS_DIR=$(sudo ls $SQ_ROOT | grep dcs\*)
+
+sudo cp $SQ_ROOT/$DCS_DIR/conf/servers $PWD
+sudo chown $(whoami).$(whoami) $PWD/servers
+
+let per_node=$DCS_SERVERS_PARM/$node_count
+let extras=$DCS_SERVERS_PARM%$node_count
+count=1
+for node_name in $NEW_NODES
+do
+ if [ $count -le $extras ]; then
+ let node_servers=$per_node+1
+ else
+ node_servers=$per_node
+ fi
+ echo $node_name $node_servers >> $PWD/servers
+ (( count++ ))
+done
+
+
+$TRAF_PDCP $PWD/servers $HOME
+$TRAF_PDSH sudo cp $HOME/servers $SQ_ROOT/$DCS_DIR/conf
+$TRAF_PDSH sudo chown $TRAF_USER.trafodion $SQ_ROOT/$DCS_DIR/conf/servers
+
+
+
+}
+
+function copyAll {
+
+source $TRAF_CONFIG
+echo
+echo "***IMPORTANT: Copying files to $NEW_NODES"
+echo "***IMPORTANT: This will take some time, it's alot of file. Please wait...."
+echo
+sudo mkdir -p $HOME/tmpTraf/
+$TRAF_NEW_PDSH sudo rm -rf $HOME_DIR/$TRAF_USER/.ssh
+$TRAF_NEW_PDSH sudo mkdir -p $HOME_DIR/$TRAF_USER/.ssh
+
+$TRAF_NEW_PDSH mkdir -p $HOME/tmpTraf
+sudo cp -r $HOME_DIR/$TRAF_USER/.bashrc $HOME/tmpTraf/.bashrc
+sudo cp -r $HOME_DIR/$TRAF_USER/.ssh $HOME/tmpTraf
+sudo cp -r $SQ_ROOT $HOME/tmpTraf
+echo "***IMPORTANT: Keep waiting..."
+sudo chown -R $(whoami).$(whoami) $HOME/tmpTraf
+$TRAF_NEW_PDCP -r $HOME/tmpTraf $HOME
+echo "***IMPORTANT: Seriously, it's a lot of file"
+$TRAF_NEW_PDSH sudo cp -r $HOME/tmpTraf/* $HOME_DIR/$TRAF_USER
+$TRAF_NEW_PDSH sudo cp -r $HOME/tmpTraf/.ssh/* $HOME_DIR/$TRAF_USER/.ssh/
+echo "***IMPORTANT: Script got to this point, probably not a hung system."
+$TRAF_NEW_PDSH sudo chown -R $TRAF_USER.trafodion $HOME_DIR/$TRAF_USER
+$TRAF_NEW_PDSH sudo rm -rf $HOME/tmpTraf
+sudo rm -r $HOME/tmpTraf
+
+}
+
+function trafAcceptKeys {
+
+echo "***INFO: Accepting keys on all nodes for all hostname types"
+
+source $TRAF_CONFIG
+
+NODES_SHORT=""
+for node in $NODE_LIST
+do
+ newNode=$(ssh -q -n $node hostname -s)
+
+ NODES_SHORT="$NODES_SHORT $newNode"
+done
+
+NODES_LONG=""
+for node in $NODE_LIST
+do
+ newNode=$(ssh -q -n $node hostname -f)
+
+ NODES_LONG="$NODES_LONG $newNode"
+done
+./traf_createPasswordLessSSH --nodes "$NODE_LIST"
+echo "***INFO: Keep waiting...."
+./traf_createPasswordLessSSH --nodes "$NODES_SHORT"
+echo "***INFO: All is well...."
+./traf_createPasswordLessSSH --nodes "$NODES_LONG"
+
+
+sudo cp $HOME_DIR/$TRAF_USER/.ssh/known_hosts $HOME
+sudo chown -R $(whoami).$(whoami) $HOME/known_hosts
+
+$TRAF_PDCP $HOME/known_hosts $HOME
+$TRAF_PDSH sudo cp -r $HOME/known_hosts $HOME_DIR/$TRAF_USER/.ssh
+$TRAF_PDSH sudo chown -R $TRAF_USER.trafodion $HOME_DIR/$TRAF_USER/.ssh/known_hosts
+$TRAF_PDSH sudo rm -rf $HOME/known_hosts
+
+}
+
+function copySqcert {
+
+source $TRAF_CONFIG
+
+$TRAF_PDSH mkdir -p $HOME/tmpTraf
+sudo cp -r $HOME_DIR/$TRAF_USER/sqcert $HOME/tmpTraf
+sudo chown -R $(whoami).$(whoami) $HOME/tmpTraf
+$TRAF_PDCP -r $HOME/tmpTraf $HOME
+$TRAF_PDSH sudo mkdir -p $HOME_DIR/$TRAF_USER/sqcert
+$TRAF_PDSH sudo chown -R $TRAF_USER.trafodion $HOME_DIR/$TRAF_USER/sqcert 2>&1 > /dev/null
+$TRAF_PDSH sudo cp -rf $HOME/tmpTraf/sqcert $HOME_DIR/$TRAF_USER/sqcert 2>&1 > /dev/null
+$TRAF_PDSH sudo chown -R $TRAF_USER.trafodion $HOME_DIR/$TRAF_USER/sqcert 2>&1 > /dev/null
+$TRAF_PDSH rm -r $HOME/tmpTraf
+
+}
+
+
+function printMessage {
+
+source $TRAF_CONFIG
+echo
+echo "***IMPORTANT: EXIT OUT OF ALL EXISTING SHELLS, need to pick up new environment variables"
+echo "***IMPORTANT: Node is ready to be added"
+echo "***IMPORTANT: sudo su $TRAF_USER"
+echo "***IMPORTANT: sqshell -a"
+echo "***IMPORTANT: up <new node>"
+echo
+
+}
+
+
+#Main
+
+echo
+echo "***IMPORTANT: Starting installation on $NEW_NODES"
+echo
+installPackages
+copyLimits
+reserveDCSPorts
+setKernelMax
+createTrafUserId
+changeTrafPassword
+editTrafbashrc
+copyTrafbashrc
+copyAll
+trafAcceptKeys
+editDCSServers
+addTraftoHBase
+copyClusterConf
+copySQconfig
+copySqcert
+printMessage
+
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/07221f21/install/installer/dcs_installer
----------------------------------------------------------------------
diff --git a/install/installer/dcs_installer b/install/installer/dcs_installer
index 8c9d544..7033014 100755
--- a/install/installer/dcs_installer
+++ b/install/installer/dcs_installer
@@ -163,6 +163,10 @@ echo "***INFO: modifying $DCS_DIR/conf/dcs-site.xml"
# add zookeeper quorum property to end of configuration
rm dcs-site.temp 2>/dev/null
+cat dcs-site.xml | sed -e "s@</configuration>@ <property>\n <name>dcs.zookeeper.property.clientPort</name>\n <value>$ZOOKEEPER_PORT</value>\n </property>\n </configuration>@" > dcs-site.temp
+cp dcs-site.temp dcs-site.xml
+rm dcs-site.temp 2>/dev/null
+
cat dcs-site.xml | sed -e "s@</configuration>@ <property>\n <name>dcs.zookeeper.quorum</name>\n <value>$ZOOKEEPER_NODES</value>\n </property>\n </configuration>@" > dcs-site.temp
cp dcs-site.temp dcs-site.xml
rm dcs-site.temp 2>/dev/null
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/07221f21/install/installer/parseHBaseSite.py
----------------------------------------------------------------------
diff --git a/install/installer/parseHBaseSite.py b/install/installer/parseHBaseSite.py
index 4fb5905..196b85d 100755
--- a/install/installer/parseHBaseSite.py
+++ b/install/installer/parseHBaseSite.py
@@ -29,11 +29,13 @@ import xml.etree.ElementTree as ET
hbaseMaster="hbase.master.info.port"
hbaseRegion="hbase.regionserver.info.port"
zooKeeperNodes="hbase.zookeeper.quorum"
+zooKeeperClientPort="hbase.zookeeper.property.clientPort"
pathToHome= os.environ['HOME']
hbaseMasterInfoPort="60010"
regionServerInfoPort="60030"
zookeeperNodeNames=""
+zookeeperPort="2181"
tree = ET.parse( pathToHome + '/hbase-site.xml')
@@ -48,9 +50,12 @@ for x in root.findall('property'):
regionServerInfoPort = x.find('value').text
if name == zooKeeperNodes:
zookeeperNodeNames = x.find('value').text
+ if name == zooKeeperClientPort:
+ zookeeperPort = x.find('value').text
f = open( '/etc/trafodion/trafodion_config', 'a')
f.write ( 'export HBASE_MASTER_INFO_PORT="' + hbaseMasterInfoPort + '"\n' )
f.write ( 'export REGIONSERVER_INFO_PORT="' + regionServerInfoPort + '"\n' )
f.write ( 'export ZOOKEEPER_NODES="' + zookeeperNodeNames + '"\n' )
+f.write ( 'export ZOOKEEPER_PORT="' + zookeeperPort + '"\n' )
f.close()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/07221f21/install/installer/rest_installer
----------------------------------------------------------------------
diff --git a/install/installer/rest_installer b/install/installer/rest_installer
index 7e52448..511d54c 100755
--- a/install/installer/rest_installer
+++ b/install/installer/rest_installer
@@ -118,6 +118,10 @@ cd $REST_DIR/conf
echo "***INFO: modifying $REST_DIR/conf/rest-site.xml"
# add zookeeper quorum property to end of configuration
rm rest-site.temp 2>/dev/null
+cat rest-site.xml | sed -e "s@</configuration>@ <property>\n <name>rest.zookeeper.property.clientPort</name>\n <value>$ZOOKEEPER_PORT</value>\n </property>\n </configuration>@" > rest-site.temp
+cp rest-site.temp rest-site.xml
+rm rest-site.temp 2>/dev/null
+
cat rest-site.xml | sed -e "s@</configuration>@ <property>\n <name>rest.zookeeper.quorum</name>\n <value>$ZOOKEEPER_NODES</value>\n </property>\n </configuration>@" > rest-site.temp
cp rest-site.temp rest-site.xml
rm rest-site.temp 2>/dev/null
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/07221f21/install/installer/tools/ambari_setup
----------------------------------------------------------------------
diff --git a/install/installer/tools/ambari_setup b/install/installer/tools/ambari_setup
index a1d8364..aca1c41 100755
--- a/install/installer/tools/ambari_setup
+++ b/install/installer/tools/ambari_setup
@@ -83,7 +83,7 @@ if [[ $AMBARI -eq 0 ]]; then
if [[ "$SUSE_Installed" -eq "1" ]]; then
wget http://public-repo-1.hortonworks.com/ambari/suse11/2.x/updates/2.0.0/ambari.repo
else
- wget http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.1.0/ambari.repo
+ wget http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.2.2.0/ambari.repo
fi
if [ $? -ne 0 ]; then
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/07221f21/install/installer/traf_apache_mods
----------------------------------------------------------------------
diff --git a/install/installer/traf_apache_mods b/install/installer/traf_apache_mods
index 5d5e955..231e3b5 100755
--- a/install/installer/traf_apache_mods
+++ b/install/installer/traf_apache_mods
@@ -187,7 +187,7 @@ sudo su $HDFS_USER $HADOOP_PREFIX/sbin/stop-dfs.sh
echo "***IMPORTANT***********************************************************************"
echo
echo "***INFO: Settings below need to be set for Trafodion to work on Apache Hadoop/HBase"
-echo "***INFO: In hdfs-site.xml set dfs.namenode.acls.enabled to true."
+echo "***INFO: In hdfs-site.xml set dfs.namenode.acls.enabled to true and dfs.umaskmode to 0002"
echo "***INFO: In zoo.cfg set maxClientCnxns to 0"
echo "***INFO: Create $ZOO_HOME/conf/zookeeeper-env.sh and set JAVA_HOME"
echo "***INFO: In hbase-site.xml set hbase.coprocessor.region.classes to org.apache.hadoop.hbase.coprocessor.transactional.TrxRegionObserver, org.apache.hadoop.hbase.coprocessor.transactional.TrxRegionEndpoint, org.apache.hadoop.hbase.coprocessor.AggregateImplementation"
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/07221f21/install/installer/traf_cloudera_mods
----------------------------------------------------------------------
diff --git a/install/installer/traf_cloudera_mods b/install/installer/traf_cloudera_mods
index d380773..708f8dc 100755
--- a/install/installer/traf_cloudera_mods
+++ b/install/installer/traf_cloudera_mods
@@ -275,7 +275,7 @@ curl -k -X PUT -H 'Content-Type:application/json' -u $ADMIN:$PASSWORD --data \
"value" : "600000"
}, {
"name" : "hbase_regionserver_config_safety_valve",
- "value" : "<property>\r\n <name>hbase.hregion.impl</name>\r\n <value>org.apache.hadoop.hbase.regionserver.transactional.TransactionalRegion</value>\r\n</property>\r\n <property>\r\n <name>hbase.regionserver.region.split.policy</name>\r\n <value>org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy</value>\r\n</property>\r\n <property>\r\n <name>hbase.snapshot.enabled</name>\r\n <value>true</value>\r\n</property>\r\n <property>\r\n <name>hbase.bulkload.staging.dir</name>\r\n <value>/hbase-staging</value>\r\n</property>\r\n <property>\r\n <name>hbase.regionserver.region.transactional.tlog</name>\r\n <value>true</value>\r\n</property>\r\n <property>\r\n <name>hbase.snapshot.region.timeout</name>\r\n <value>600000</value>\r\n</property>\r\n "
+ "value" : "<property>\r\n <name>hbase.hregion.impl</name>\r\n <value>org.apache.hadoop.hbase.regionserver.transactional.TransactionalRegion</value>\r\n</property>\r\n <property>\r\n <name>hbase.regionserver.region.split.policy</name>\r\n <value>org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy</value>\r\n</property>\r\n <property>\r\n <name>hbase.snapshot.enabled</name>\r\n <value>true</value>\r\n</property>\r\n <property>\r\n <name>hbase.snapshot.region.timeout</name>\r\n <value>600000</value>\r\n</property>\r\n "
} ]
} ]
}' \
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/07221f21/install/installer/traf_config
----------------------------------------------------------------------
diff --git a/install/installer/traf_config b/install/installer/traf_config
index a765e53..284b772 100755
--- a/install/installer/traf_config
+++ b/install/installer/traf_config
@@ -70,6 +70,17 @@ fi
mkdir -p $SQ_ROOT
echo $SQ_ROOT
+if [[ "$UPGRADE_TRAF" == "true" ]]; then
+ echo
+ echo
+ echo "***INFO: On an upgrade, any ms.env settings will not be transferred over."
+ echo "***INFO: If previous installation directory was not deleted, it will be there for comparison"
+ echo
+ echo
+ sleep 5
+fi
+
+
if [[ -f $HOME/sqenvcom.sh ]]; then
echo "****INFO: Copying over sqenvcom.sh"
cp -rf $HOME/sqenvcom.sh $SQ_ROOT
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/07221f21/install/installer/traf_config_check
----------------------------------------------------------------------
diff --git a/install/installer/traf_config_check b/install/installer/traf_config_check
index c7da156..f504896 100755
--- a/install/installer/traf_config_check
+++ b/install/installer/traf_config_check
@@ -33,7 +33,11 @@ source $TRAF_CONFIG
function majorErrorFound {
if [[ "$errorFound" == "1" ]]; then
cat $ERROR_LOG
+ echo
+ echo
+ echo "******************************************"
cat /etc/trafodion/trafodion_config
+ echo "******************************************"
echo
exit -1
fi
@@ -207,6 +211,21 @@ else
MY_NODES="$MY_NODES -w $node"
done
+ if [[ "$ELASTICITY_ENABLED" == "Y" ]]; then
+ node_id="0"
+ TRAF_EXCLUDE_LIST=""
+ while [[ "$node_id" -lt "$NUM_ELASTIC_NODES" ]]
+ do
+ TRAF_EXCLUDE_LIST="$TRAF_EXCLUDE_LIST extranode$node_id"
+ ((node_id++))
+ done
+
+ sudo chmod 777 $TRAF_CONFIG
+ sed -i '/TRAF_EXCLUDE_LIST\=/d' $TRAF_CONFIG
+ echo "export TRAF_EXCLUDE_LIST=\"$TRAF_EXCLUDE_LIST\"" >> $TRAF_CONFIG
+ sudo chmod 777 $TRAF_CONFIG
+ fi
+
sudo chmod 777 $TRAF_CONFIG
sed -i '/MY_NODES\=/d' $TRAF_CONFIG
echo "export MY_NODES=\"$MY_NODES\"" >> $TRAF_CONFIG
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/07221f21/install/installer/traf_config_setup
----------------------------------------------------------------------
diff --git a/install/installer/traf_config_setup b/install/installer/traf_config_setup
index fd3a9ab..90143ef 100755
--- a/install/installer/traf_config_setup
+++ b/install/installer/traf_config_setup
@@ -236,6 +236,26 @@ echo "export NODE_LIST=\"$NODE_LIST\"" >> $LOCAL_TRAF_CONFIG
node_count=$(echo $NODE_LIST | wc -w)
#==============================================
+#Elasticity
+echo -n "Enable elasticity (Y/N), default is N: "
+read answer
+if [[ "${answer}" =~ ^[Yy]$ ]]; then
+ echo "export ELASTICITY_ENABLED=\"Y\"" >> $LOCAL_TRAF_CONFIG
+ echo "***INFO: When elasticity is enabled, there will be copy errors"
+ sleep 10
+ NUM_ELASTIC_NODES="$node_count"
+ echo -n "Number of elastic nodes, default is [ $NUM_ELASTIC_NODES ]: "
+ read answer
+ if [[ -z "$answer" ]]; then
+ echo "export NUM_ELASTIC_NODES=\"$NUM_ELASTIC_NODES\"" >> $LOCAL_TRAF_CONFIG
+ else
+ echo "export NUM_ELASTIC_NODES=\"$answer\"" >> $LOCAL_TRAF_CONFIG
+ fi
+else
+ echo "export ELASTICITY_ENABLED=\"N\"" >> $LOCAL_TRAF_CONFIG
+fi
+
+#==============================================
echo -n "Do you have a set of management nodes (Y/N), default is N: "
read answer
@@ -676,17 +696,8 @@ fi
#==============================================
#Install location
-if [ -z $SQ_ROOT ]; then
-
- # if we have a package file then use its name for the default install dir
- if [ ! -z "$TRAF_PACKAGE" ]; then
- traf_filename=$(basename "$TRAF_PACKAGE")
- SQ_ROOT="$HOME_DIR/$TRAF_USER/${traf_filename%.tar.gz}"
- else
- # otherwise, create a default install dir using a timestamp
- SQ_ROOT="$HOME_DIR/$TRAF_USER/traf_$(date +%F_%H%M)"
- fi
-fi
+traf_filename=$(basename "$TRAF_PACKAGE")
+SQ_ROOT="$HOME_DIR/$TRAF_USER/${traf_filename%.tar.gz}"
echo -n "Enter directory to install trafodion to, default is [$SQ_ROOT]: "
read answer
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/07221f21/install/installer/traf_hortonworks_mods
----------------------------------------------------------------------
diff --git a/install/installer/traf_hortonworks_mods b/install/installer/traf_hortonworks_mods
index a707137..4a5521f 100755
--- a/install/installer/traf_hortonworks_mods
+++ b/install/installer/traf_hortonworks_mods
@@ -135,7 +135,7 @@ fi
for node in $HBASE_NODES
do
copiedOver=$(ssh -q -n $node sudo ls $HADOOP_PATH/hbase-trx* | wc -l)
- if [[ $copiedOver -eq "0" ]]; then
+ if [[ $copiedOver -ne "1" ]]; then
echo "***ERROR: $hbase_trx_jar was not copied on $node"
echo "***ERROR: Please investigate why this happened"
echo "***ERROR: Trafodion can not start without this. EXITING..."
@@ -248,18 +248,6 @@ if [ $? != 0 ]; then
exit -1
fi
sleep 2
-ssh -q -n $AMBARI_HOST "$AMBARI_DIR"'/configs.sh -u' "$ADMIN" '-p' "$PASSWORD" '-port' "$PORT" 'set' "$AMBARI_HOST" "$CLUSTER_NAME" 'hbase-site hbase.bulkload.staging.dir /hbase-staging'
-if [ $? != 0 ]; then
- echo "***ERROR: unable to modify hbase.bulkload.staging.dir through Ambari's configs.sh script."
- exit -1
-fi
-sleep 2
-ssh -q -n $AMBARI_HOST "$AMBARI_DIR"'/configs.sh -u' "$ADMIN" '-p' "$PASSWORD" '-port' "$PORT" 'set' "$AMBARI_HOST" "$CLUSTER_NAME" 'hbase-site hbase.regionserver.region.transactional.tlog true'
-if [ $? != 0 ]; then
- echo "***ERROR: unable to modify hbase.regionserver.region.transactional.tlog through Ambari's configs.sh script."
- exit -1
-fi
-sleep 2
ssh -q -n $AMBARI_HOST "$AMBARI_DIR"'/configs.sh -u' "$ADMIN" '-p' "$PASSWORD" '-port' "$PORT" 'set' "$AMBARI_HOST" "$CLUSTER_NAME" 'hbase-site hbase.snapshot.master.timeoutMillis 600000'
if [ $? != 0 ]; then
echo "***ERROR: unable to modify hbase.snapshot.master.timeoutMillis through Ambari's configs.sh script."
@@ -337,6 +325,12 @@ while [ $completed -eq 0 ]; do
echo "***INFO: ...polling every $poll_time seconds until HBase stop is completed."
# if stop command is completed then completed will not equal 0
completed=$(cat $LOCAL_WORKDIR/hbase_restart_status_temp | grep '"request_status" : "COMPLETED"' | wc -l)
+ failed=$(cat $LOCAL_WORKDIR/hbase_restart_status_temp | grep '"request_status" : "FAILED"' | wc -l)
+ if [[ "$failed" -ge "1" ]]; then
+ echo "***ERROR: Unable to stop HBase"
+ echo "***ERROR: Please manually restart HBase through the Ambari web GUI"
+ exit -1
+ fi
done
echo "***INFO: HBase stop completed"
@@ -379,6 +373,13 @@ while [ $completed -eq 0 ]; do
echo "***INFO: ...polling every $poll_time seconds until Zookeeper stop is completed."
# if stop command is completed then completed will not equal 0
completed=$(cat $LOCAL_WORKDIR/traf_zoo_restart_temp | grep '"request_status" : "COMPLETED"' | wc -l)
+ failed=$(cat $LOCAL_WORKDIR/traf_zoo_restart_temp | grep '"request_status" : "FAILED"' | wc -l)
+ if [[ "$failed" -ge "1" ]]; then
+ echo "***ERROR: Unable to stop Zookeeper"
+ echo "***ERROR: Please manually restart Zookeeper through the Ambari web GUI"
+ exit -1
+ fi
+
done
#=====================================
@@ -420,6 +421,13 @@ while [ $completed -eq 0 ]; do
echo "***INFO: ...polling every $poll_time seconds until HDFS stop is completed."
# if stop command is completed then completed will not equal 0
completed=$(cat $LOCAL_WORKDIR/traf_hdfs_restart_temp | grep '"request_status" : "COMPLETED"' | wc -l)
+ failed=$(cat $LOCAL_WORKDIR/traf_hdfs_restart_temp | grep '"request_status" : "FAILED"' | wc -l)
+ if [[ "$failed" -ge "1" ]]; then
+ echo "***ERROR: Unable to stop HDFS"
+ echo "***ERROR: Please manually restart HDFS through the Ambari web GUI"
+ exit -1
+ fi
+
done
echo "***INFO: Starting HDFS..."
@@ -456,6 +464,13 @@ while [ $completed -eq 0 ]; do
echo "***INFO: ...polling every $poll_time seconds until HDFS start is completed."
# if start command is completed then completed will not equal 0
completed=$(cat $LOCAL_WORKDIR/traf_hdfs_restart_temp | grep '"request_status" : "COMPLETED"' | wc -l)
+ failed=$(cat $LOCAL_WORKDIR/traf_hdfs_restart_temp | grep '"request_status" : "FAILED"' | wc -l)
+ if [[ "$failed" -ge "1" ]]; then
+ echo "***ERROR: Unable to start HDFS"
+ echo "***ERROR: Please manually restart HDFS through the Ambari web GUI"
+ exit -1
+ fi
+
done
echo "***INFO: HDFS restart completed"
@@ -501,6 +516,13 @@ while [ $completed -eq 0 ]; do
echo "***INFO: ...polling every $poll_time seconds until Zookeeper start is completed."
# if start command is completed then completed will not equal 0
completed=$(cat $LOCAL_WORKDIR/traf_zoo_restart_temp | grep '"request_status" : "COMPLETED"' | wc -l)
+ failed=$(cat $LOCAL_WORKDIR/traf_zoo_restart_temp | grep '"request_status" : "FAILED"' | wc -l)
+ if [[ "$failed" -ge "1" ]]; then
+ echo "***ERROR: Unable to start Zookeeper"
+ echo "***ERROR: Please manually restart Zookeeper through the Ambari web GUI"
+ exit -1
+ fi
+
done
echo "***INFO: Zookeeper start completed"
@@ -543,6 +565,13 @@ while [ $completed -eq 0 ]; do
echo "***INFO: ...polling every $poll_time seconds until HBase start is completed."
# if start command is completed then completed will not equal 0
completed=$(cat $LOCAL_WORKDIR/hbase_restart_status_temp | grep '"request_status" : "COMPLETED"' | wc -l)
+ failed=$(cat $LOCAL_WORKDIR/hbase_restart_status_temp | grep '"request_status" : "FAILED"' | wc -l)
+ if [[ "$failed" -ge "1" ]]; then
+ echo "***ERROR: Unable to start HBase"
+ echo "***ERROR: Please manually restart HBase through the Ambari web GUI"
+ exit -1
+ fi
+
done
echo "***INFO: HBase restart completed"
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/07221f21/install/installer/traf_sqconfig
----------------------------------------------------------------------
diff --git a/install/installer/traf_sqconfig b/install/installer/traf_sqconfig
index 9a79618..02f30fb 100755
--- a/install/installer/traf_sqconfig
+++ b/install/installer/traf_sqconfig
@@ -82,6 +82,14 @@ if [ "$node_count" -ne "1" ]; then
((node_id++))
done
+ if [[ "$ELASTICITY_ENABLED" == "Y" ]]; then
+ for node in $TRAF_EXCLUDE_LIST
+ do
+ echo "node-id=$node_id;node-name=$node;cores=$cores;processors=$processors;roles=connection,aggregation,storage" >> $sqconfig
+ ((node_id++))
+ done
+ fi
+
echo "end node" >> $sqconfig
sudo cp $sqconfig $UNTAR_DIR
sudo cp $sqconfig $TRAF_USER_DIR/sqconfig
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/07221f21/install/installer/traf_start
----------------------------------------------------------------------
diff --git a/install/installer/traf_start b/install/installer/traf_start
index 53ed4a2..8fcc835 100755
--- a/install/installer/traf_start
+++ b/install/installer/traf_start
@@ -48,11 +48,20 @@ else
fi
# Do init_trafodion if specified
- if [ "$INIT_TRAFODION" == "Y" ] && [ "$UPGRADE_TRAF" == "false" ]; then
- ssh $STARTING_NODE "cd $SQ_ROOT/sql/scripts; echo \"initialize Trafodion;\" | sqlci" | tee -a $SQ_ROOT/initializeErrors.txt
- foundError=$(grep "ERROR" $SQ_ROOT/initializeErrors.txt | wc -l)
+ if [ "$INIT_TRAFODION" == "Y" ]; then
+ echo
+ echo
+ echo "***INFO: Trafodion is initializing...this will take a few minutes."
+ echo
+ echo
+ ssh $STARTING_NODE "cd $SQ_ROOT/sql/scripts; echo \"initialize Trafodion;\" | sqlci" | tee -a $SQ_ROOT/initializeErrors.txt
+ foundError=$(grep "1392" $SQ_ROOT/initializeErrors.txt | wc -l)
if [ "$foundError" -ge "1" ]; then
- ssh $STARTING_NODE "cd $SQ_ROOT/sql/scripts; echo \"initialize Trafodion, upgrade;\" | sqlci"
+ ssh $STARTING_NODE "cd $SQ_ROOT/sql/scripts; echo \"get version of metadata;\" | sqlci" | tee -a $SQ_ROOT/initalizeErrors.txt
+ metadataCurrent=$(grep "Metadata is current" $SQ_ROOT/initializeErrors.txt | wc -l)
+ if [[ "$metadataCurrent" -ne "1" ]]; then
+ ssh $STARTING_NODE "cd $SQ_ROOT/sql/scripts; echo \"initialize Trafodion, upgrade;\" | sqlci" | tee -a $SQ_ROOT/initializeErrors.txt
+ fi
fi
if [ "$?" != "0" ]; then
@@ -60,18 +69,7 @@ else
exit -1;
fi
fi
- # Do init_trafodion, upgrade if specified
- if [ "$INIT_TRAFODION" == "Y" ] && [ "$UPGRADE_TRAF" == "true" ]; then
- ssh $STARTING_NODE "cd $SQ_ROOT/sql/scripts; echo \"initialize Trafodion, upgrade;\" | sqlci" | tee -a $SQ_ROOT/initializeErrors.txt
- foundError=$(grep "ERROR" $SQ_ROOT/initializeErrors.txt | wc -l)
- if [ "$foundError" -ge "1" ];then
- ssh $STARTING_NODE "cd $SQ_ROOT/sql/scripts; echo \"initialize Trafodion;\" | sqlci"
- fi
- if [ "$?" != "0" ]; then
- echo "***ERROR: initialize Trafodion failed. Check install log files for details." | tee -a $INSTALL_LOG
- exit -1;
- fi
- fi
+
fi
#==========================================
echo "***INFO: Installation setup completed successfully." | tee -a $INSTALL_LOG
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/07221f21/install/installer/trafodion_config_default
----------------------------------------------------------------------
diff --git a/install/installer/trafodion_config_default b/install/installer/trafodion_config_default
index 6c67c0b..915828c 100755
--- a/install/installer/trafodion_config_default
+++ b/install/installer/trafodion_config_default
@@ -198,3 +198,10 @@ export CLOUD_OTHER=""
#dcs start script is run
export DCS_PRIMARY_MASTER_NODE=`hostname -f`
+
+#Ability to add nodes while trafodion is running
+export ELASTICITY_ENABLED="N"
+
+#Default is number of nodes
+export NUM_ELASTIC_NODES=""
+
[2/2] incubator-trafodion git commit: Merging [TRAFODION-2001] PR-601
Adding Installer Elasticity scripts
Posted by am...@apache.org.
Merging [TRAFODION-2001] PR-601 Adding Installer Elasticity scripts
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/65751021
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/65751021
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/65751021
Branch: refs/heads/master
Commit: 657510219c98f9254a9066bf90ce6031fe22c23f
Parents: 522acc9 07221f2
Author: Amanda Moran <am...@apache.org>
Authored: Wed Jul 27 17:08:07 2016 +0000
Committer: Amanda Moran <am...@apache.org>
Committed: Wed Jul 27 17:08:07 2016 +0000
----------------------------------------------------------------------
install/installer/addNode_packages | 89 ++++++
install/installer/addNode_reservePorts | 81 ++++++
install/installer/addNode_step1 | 294 +++++++++++++++++++
install/installer/addNode_step2 | 372 ++++++++++++++++++++++++
install/installer/dcs_installer | 4 +
install/installer/parseHBaseSite.py | 5 +
install/installer/rest_installer | 4 +
install/installer/tools/ambari_setup | 2 +-
install/installer/traf_apache_mods | 2 +-
install/installer/traf_cloudera_mods | 2 +-
install/installer/traf_config | 11 +
install/installer/traf_config_check | 19 ++
install/installer/traf_config_setup | 33 ++-
install/installer/traf_hortonworks_mods | 55 +++-
install/installer/traf_sqconfig | 8 +
install/installer/traf_start | 30 +-
install/installer/trafodion_config_default | 7 +
17 files changed, 975 insertions(+), 43 deletions(-)
----------------------------------------------------------------------