You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by db...@apache.org on 2017/07/05 22:36:12 UTC
[1/3] incubator-trafodion git commit: [TRAFODION-2658] pyinstaller -
support elastic adding nodes
Repository: incubator-trafodion
Updated Branches:
refs/heads/master 05d14d47f -> 765bb75d3
[TRAFODION-2658] pyinstaller - support elastic adding nodes
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/284415ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/284415ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/284415ff
Branch: refs/heads/master
Commit: 284415ff6f969392b939549d71f9fa8a81fc51fb
Parents: dbd078c
Author: Eason <hf...@gmail.com>
Authored: Wed Jun 21 21:10:04 2017 +0800
Committer: Eason <hf...@gmail.com>
Committed: Fri Jun 30 11:38:18 2017 +0800
----------------------------------------------------------------------
core/sqf/sql/scripts/sqregen | 9 +-
core/sqf/tools/sqtools.sh | 8 +-
install/python-installer/add_nodes.py | 216 +++++++++++++++++++
install/python-installer/configs/script.json | 25 +++
install/python-installer/db_install.py | 159 +-------------
install/python-installer/scripts/common.py | 156 +++++++++++++-
.../python-installer/scripts/traf_addnodes.py | 165 ++++++++++++++
install/python-installer/scripts/traf_dep.py | 2 +-
install/python-installer/scripts/traf_user.py | 16 +-
9 files changed, 585 insertions(+), 171 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/284415ff/core/sqf/sql/scripts/sqregen
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/sqregen b/core/sqf/sql/scripts/sqregen
index 3996bc9..e664ae1 100755
--- a/core/sqf/sql/scripts/sqregen
+++ b/core/sqf/sql/scripts/sqregen
@@ -41,8 +41,15 @@ function Usage {
function RegenConfig {
echo "Regenerating Trafodion configuration: $SQCONFIG_FILE"
+ cd $TRAF_HOME/sql/scripts
./regensq.pl $SQCONFIG_FILE
- exit $?;
+ if [ $? != 0 ]; then
+ echo "Failed to run regensq.pl."
+ exit $?
+ fi
+ echo "Copying Trafodion configuration file $SQCONFIG_FILE to all nodes"
+ pdcp `trafconf -wname` $SQCONFIG_FILE $TRAF_HOME/sql/scripts/
+ exit $?
}
while [ $# != 0 ]
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/284415ff/core/sqf/tools/sqtools.sh
----------------------------------------------------------------------
diff --git a/core/sqf/tools/sqtools.sh b/core/sqf/tools/sqtools.sh
index 1906d7e..d8e7206 100644
--- a/core/sqf/tools/sqtools.sh
+++ b/core/sqf/tools/sqtools.sh
@@ -23,12 +23,8 @@
#
function setup_my_nodes {
- if [ -z "$MY_NODES" ]; then
- echo
- else
- wnode_names=`trafconf -wname`
- export MY_NODES="$wnode_names"
- fi
+ export MY_NODES=`trafconf -wname`
+ export NODE_LIST=`trafconf -name`
}
function setup_sqpdsh {
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/284415ff/install/python-installer/add_nodes.py
----------------------------------------------------------------------
diff --git a/install/python-installer/add_nodes.py b/install/python-installer/add_nodes.py
new file mode 100755
index 0000000..92569dc
--- /dev/null
+++ b/install/python-installer/add_nodes.py
@@ -0,0 +1,216 @@
+#!/usr/bin/env python
+# -*- coding: utf8 -*-
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+import os
+import re
+import socket
+import json
+import getpass
+import time
+import sys
+reload(sys)
+sys.setdefaultencoding("utf-8")
+from optparse import OptionParser
+from glob import glob
+from collections import defaultdict
+from threading import Thread
+from scripts import wrapper
+from db_install import UserInput
+from scripts.constants import TRAF_CFG_FILE
+from scripts.common import get_sudo_prefix, Remote, run_cmd, run_cmd_as_user, info, ok, \
+ format_output, err_m, expNumRe, cmd_output
+
+TRAF_PKG_FILE = '/tmp/traf_bin.tar.gz'
+
+def get_options():
+ usage = 'usage: %prog [options]\n'
+ usage += ' Trafodion install main script.'
+ parser = OptionParser(usage=usage)
+ parser.add_option("-u", "--remote-user", dest="user", metavar="USER",
+ help="Specify ssh login user for remote server, \
+ if not provided, use current login user as default.")
+ parser.add_option("-n", "--nodes", dest="nodes", metavar="NODES",
+ help="Specify the node names you want to add, seperated by comma if more than one, \
+ support numeric RE, e.g. node[1-3].com")
+ parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
+ help="Verbose mode, will print commands.")
+ parser.add_option("--enable-pwd", action="store_true", dest="pwd", default=False,
+ help="Prompt SSH login password for remote hosts. \
+ If set, \'sshpass\' tool is required.")
+ parser.add_option("--offline", action="store_true", dest="offline", default=False,
+ help="Enable local repository for offline installing Trafodion.")
+
+ (options, args) = parser.parse_args()
+ return options
+
+def main():
+ """ add_nodes main loop """
+ cfgs = defaultdict(str)
+
+ # handle parser option
+ options = get_options()
+ if not options.nodes:
+ err_m('Must specifiy the node names using \'--nodes\' option')
+
+ # get node list from user input
+ new_nodes = expNumRe(options.nodes)
+ if not new_nodes:
+ err_m('Incorrect format')
+
+ if options.pwd:
+ pwd = getpass.getpass('Input remote host SSH Password: ')
+ else:
+ pwd = ''
+
+ u = UserInput(options, pwd)
+ g = lambda n: u.get_input(n, cfgs[n], prompt_mode=prompt_mode)
+
+ format_output('Trafodion Elastic Add Nodes Script')
+
+ ### read configs from current trafodion_config and save it to cfgs
+ if os.path.exists(TRAF_CFG_FILE):
+ with open(TRAF_CFG_FILE, 'r') as f:
+ traf_cfgs = f.readlines()
+ for traf_cfg in traf_cfgs:
+ if not traf_cfg.strip(): continue
+ key, value = traf_cfg.replace('export ', '').split('=')
+ value = value.replace('"','')
+ value = value.replace('\n','')
+ cfgs[key.lower()] = value
+ else:
+ err_m('Cannot find %s, be sure to run this script on one of trafodion nodes' % TRAF_CFG_FILE)
+
+ ### config check
+ if not cfgs['hbase_lib_path'] or not cfgs['traf_version']:
+ err_m('Missing parameters in Trafodion config file')
+
+ if not cfgs['traf_home'] or not cmd_output('%s ls %s' % (get_sudo_prefix(), cfgs['traf_home'])):
+ err_m('Cannot find trafodion binary folder')
+ # get trafodion user from traf_home path
+ cfgs['traf_user'] = cfgs['traf_home'].split('/')[-2]
+ if not cfgs['traf_user']:
+ err_m('Cannot detect trafodion user')
+
+ ### parse trafodion user's password
+ cfgs['traf_shadow'] = cmd_output("%s grep %s /etc/shadow |awk -F: '{print $2}'" % (get_sudo_prefix(), cfgs['traf_user']))
+
+ def copy_files():
+ # package trafodion binary into a tar file
+ if not os.path.exists(TRAF_PKG_FILE):
+ info('Creating trafodion packages of %s, this will take a while ...' % cfgs['traf_home'])
+ run_cmd_as_user(cfgs['traf_user'], 'cd %s; tar czf %s ./* --exclude logs/* --exclude core.* --exclude tmp/*' % (cfgs['traf_home'], TRAF_PKG_FILE))
+ else:
+ info('Using existing trafodion package %s' % TRAF_PKG_FILE)
+
+ info('Copying trafodion files to new nodes, this will take a while ...')
+ run_cmd('%s cp -rf %s/../.ssh /tmp' % (get_sudo_prefix(), cfgs['traf_home']))
+ run_cmd('%s chmod -R 755 /tmp/.ssh' % get_sudo_prefix())
+ traf_ssh_folder = '/tmp/.ssh'
+
+ hbase_trx_file = cmd_output('ls %s/hbase-trx-*' % cfgs['hbase_lib_path'])
+ trafodion_utility_file = cmd_output('ls %s/trafodion-utility-*' % cfgs['hbase_lib_path'])
+
+ files = [TRAF_CFG_FILE, TRAF_PKG_FILE, traf_ssh_folder, hbase_trx_file, trafodion_utility_file]
+
+ remote_insts = [Remote(h, pwd=pwd) for h in new_nodes]
+ threads = [Thread(target=r.copy, args=(files, '/tmp')) for r in remote_insts]
+ for thread in threads: thread.start()
+ for thread in threads: thread.join()
+
+ for r in remote_insts:
+ if r.rc != 0: err_m('Failed to copy files to %s' % r.host)
+
+ ### copy trafodion_config/trafodion-package/hbase-trx to the new nodes
+ copy_files()
+
+ ### set parameters
+ if cfgs['enable_ha'].upper() == 'true':
+ g('dcs_backup_nodes')
+ cfgs['dcs_ha'] = 'Y'
+ else:
+ cfgs['dcs_ha'] = 'N'
+
+ if cfgs['trafodion_enable_authentication'] == 'YES':
+ cfgs['ldap_security'] = 'Y'
+ else:
+ cfgs['ldap_security'] = 'N'
+
+ if cfgs['secure_hadoop'].upper() == 'Y':
+ g('kdc_server')
+ g('admin_principal')
+ g('kdcadmin_pwd')
+
+ #TODO: offline support
+ cfgs['offline_mode'] = 'N'
+
+
+ format_output('AddNode sub scripts Start')
+
+ ### run addNode script on new nodes ###
+ cfgs['node_list'] = ','.join(new_nodes)
+ info('Running add node setup on new node(s) [%s] ...' % cfgs['node_list'])
+ wrapper.run(cfgs, options, mode='addnodes_new', pwd=pwd)
+
+ ### run dcs setup script on all nodes ###
+ # get current trafodion node list
+ current_nodes = cmd_output('%s su - %s -c "trafconf -name 2>/dev/null"' % (get_sudo_prefix(), cfgs['traf_user'])).split()
+ all_nodes = list(set(new_nodes + current_nodes))
+ cfgs['node_list'] = ','.join(all_nodes)
+ info('Running dcs setup on all node(s) [%s] ...' % cfgs['node_list'])
+ wrapper.run(cfgs, options, mode='addnodes_all', pwd=pwd)
+
+ ### do sqshell node add/up, sqregen
+ # check if trafodion is running
+ mon_process = cmd_output('ps -ef|grep -v grep|grep -c "monitor COLD"')
+ if int(mon_process) > 0:
+ info('Trafodion instance is up, adding node in sqshell ...')
+
+ # cores=0-1;processors=2;roles=connection,aggregation,storage
+ sqconfig_ptr = cmd_output('%s su - %s -c "trafconf -node|sed -n 2p|cut -d\\\";\\\" -f3-5"' % (get_sudo_prefix(), cfgs['traf_user']))
+ for node in new_nodes:
+ info('adding node [%s] in sqshell ...' % node)
+ run_cmd_as_user(cfgs['traf_user'], 'echo "node add {node-name %s,%s}" | sqshell -a' % (node, sqconfig_ptr))
+ run_cmd_as_user(cfgs['traf_user'], 'echo "node up %s" | sqshell -a' % node)
+ ok('Node [%s] added!' % node)
+
+ info('Starting DCS on new nodes ...')
+ run_cmd_as_user(cfgs['traf_user'], 'dcsstart')
+ else:
+ info('Trafodion instance is not up, do sqgen ...')
+ run_cmd_as_user(cfgs['traf_user'], 'rm %s/sql/scripts/sqconfig.db' % cfgs['traf_home'])
+ run_cmd_as_user(cfgs['traf_user'], 'sqgen')
+ ok('Setup completed. You need to start trafodion manually')
+
+ ### clean up
+ run_cmd('%s rm -rf /tmp/.ssh' % get_sudo_prefix())
+ run_cmd('%s rm -rf %s' % (get_sudo_prefix(), TRAF_PKG_FILE))
+
+ format_output('AddNode Complete')
+ info('NOTICE: You need to manually restart RegionServer on newly added nodes to take effect')
+
+if __name__ == "__main__":
+ try:
+ main()
+ except (KeyboardInterrupt, EOFError):
+ print '\nAborted...'
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/284415ff/install/python-installer/configs/script.json
----------------------------------------------------------------------
diff --git a/install/python-installer/configs/script.json b/install/python-installer/configs/script.json
index f6191d5..53c1fb4 100644
--- a/install/python-installer/configs/script.json
+++ b/install/python-installer/configs/script.json
@@ -77,6 +77,31 @@
"run_as_traf": "yes"
}
],
+"addnodes_all": [
+ {
+ "script": "dcs_setup.py",
+ "desc": "DCS/REST Setup",
+ "node": "all",
+ "run_as_traf": "yes"
+ }
+],
+"addnodes_new": [
+ {
+ "script": "traf_addnodes.py",
+ "desc": "Add nodes Setup",
+ "node": "all"
+ },
+ {
+ "script": "traf_dep.py",
+ "desc": "Install Trafodion dependencies",
+ "node": "all"
+ },
+ {
+ "script": "traf_kerberos.py",
+ "desc": "Kerberos Setup",
+ "node": "all"
+ }
+],
"discover": [
{
"script": "traf_discover.py",
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/284415ff/install/python-installer/db_install.py
----------------------------------------------------------------------
diff --git a/install/python-installer/db_install.py b/install/python-installer/db_install.py
index 013e2ae..dac1ca5 100755
--- a/install/python-installer/db_install.py
+++ b/install/python-installer/db_install.py
@@ -41,168 +41,13 @@ except ImportError:
exit(1)
from scripts import wrapper
from scripts.constants import DEF_PORT_FILE, DBCFG_FILE, USER_PROMPT_FILE, DBCFG_TMP_FILE, INSTALLER_LOC, \
- DEF_HBASE_XML_FILE, PARCEL_HBASE_LIB, DEF_HBASE_LIB, HDP_HBASE_LIB, TRAF_USER
-from scripts.common import Remote, Version, ParseHttp, ParseInI, ParseJson, run_cmd, info, \
+ DEF_HBASE_XML_FILE, TRAF_USER
+from scripts.common import HadoopDiscover, Remote, Version, ParseHttp, ParseInI, ParseJson, run_cmd, info, \
http_start, http_stop, format_output, err_m, expNumRe
# init global cfgs for user input
cfgs = defaultdict(str)
-class HadoopDiscover(object):
- """ discover for hadoop related info """
- def __init__(self, user, pwd, url, cluster_name):
- self.rsnodes = []
- self.users = {}
- self.cluster_name = cluster_name
- self.hg = ParseHttp(user, pwd)
- self.url = url
- self.v1_url = '%s/api/v1/clusters' % self.url
- self.v6_url = '%s/api/v6/clusters' % self.url
- self.cluster_url = '%s/%s' % (self.v1_url, cluster_name.replace(' ', '%20'))
- self._get_distro()
- self._check_version()
- if 'CDH' in self.distro:
- self.cm = self.hg.get('%s/api/v6/cm/deployment' % self.url)
-
- def _get_distro(self):
- content = self.hg.get(self.v1_url)
-
- if content['items'][0].has_key('name'):
- # use v6 rest api for CDH to get fullversion
- content = self.hg.get(self.v6_url)
-
- # loop all managed clusters
- for cluster in content['items']:
- try:
- # HDP
- self.distro = cluster['Clusters']['version']
- except KeyError:
- # CDH
- try:
- if self.cluster_name == cluster['displayName']:
- self.distro = 'CDH' + cluster['fullVersion']
- break
- except KeyError:
- log_err('Failed to get hadoop distribution info from management url')
-
- def get_hdfs_srvname(self):
- return self._get_service_name('HDFS')
-
- def get_hbase_srvname(self):
- return self._get_service_name('HBASE')
-
- def get_zookeeper_srvname(self):
- return self._get_service_name('ZOOKEEPER')
-
- def _get_service_name(self, service):
- # CDH uses different service names in multiple clusters
- if 'CDH' in self.distro:
- services_cfgs = self.hg.get(self.cluster_url +'/services')
- for item in services_cfgs['items']:
- if item['type'] == service:
- return item['name']
- else:
- return service.lower()
-
- def _check_version(self):
- version = Version()
- if 'CDH' in self.distro: version_list = version.get_version('cdh')
- if 'HDP' in self.distro: version_list = version.get_version('hdp')
-
- has_version = 0
- for ver in version_list:
- if ver in self.distro: has_version = 1
-
- if not has_version:
- log_err('Sorry, currently Trafodion doesn\'t support %s version' % self.distro)
-
- def get_hadoop_users(self):
- if 'CDH' in self.distro:
- self._get_cdh_users()
- elif 'HDP' in self.distro or 'BigInsights' in self.distro:
- self._get_hdp_users()
- return self.users
-
- def _get_hdp_users(self):
- desired_cfg = self.hg.get('%s/?fields=Clusters/desired_configs' % (self.cluster_url))
- config_type = {'hbase-env':'hbase_user', 'hadoop-env':'hdfs_user'}
- for key, value in config_type.items():
- desired_tag = desired_cfg['Clusters']['desired_configs'][key]['tag']
- current_cfg = self.hg.get('%s/configurations?type=%s&tag=%s' % (self.cluster_url, key, desired_tag))
- self.users[value] = current_cfg['items'][0]['properties'][value]
-
- def _get_cdh_users(self):
- def _get_username(service_name, hadoop_type):
- cfg = self.hg.get('%s/services/%s/config' % (self.cluster_url, service_name))
- if cfg.has_key('items'):
- for item in cfg['items']:
- if item['name'] == 'process_username':
- return item['value']
- return hadoop_type
-
- hdfs_user = _get_username(self.get_hdfs_srvname(), 'hdfs')
- hbase_user = _get_username(self.get_hbase_srvname(), 'hbase')
-
- self.users = {'hbase_user':hbase_user, 'hdfs_user':hdfs_user}
-
- def get_rsnodes(self):
- if 'CDH' in self.distro:
- self._get_rsnodes_cdh()
- elif 'HDP' in self.distro or 'BigInsights' in self.distro:
- self._get_rsnodes_hdp()
-
- self.rsnodes.sort()
- ## use short hostname
- #try:
- # self.rsnodes = [re.match(r'([\w\-]+).*', node).group(1) for node in self.rsnodes]
- #except AttributeError:
- # pass
- return self.rsnodes
-
- def _get_rsnodes_cdh(self):
- """ get list of HBase RegionServer nodes in CDH """
- hostids = []
- for c in self.cm['clusters']:
- if c['displayName'] == self.cluster_name:
- for s in c['services']:
- if s['type'] == 'HBASE':
- for r in s['roles']:
- if r['type'] == 'REGIONSERVER': hostids.append(r['hostRef']['hostId'])
- for i in hostids:
- for h in self.cm['hosts']:
- if i == h['hostId']: self.rsnodes.append(h['hostname'])
-
- def _get_rsnodes_hdp(self):
- """ get list of HBase RegionServer nodes in HDP """
- hdp = self.hg.get('%s/services/HBASE/components/HBASE_REGIONSERVER' % self.cluster_url)
- self.rsnodes = [c['HostRoles']['host_name'] for c in hdp['host_components']]
-
- def get_hbase_lib_path(self):
- if 'CDH' in self.distro:
- for c in self.cm['clusters']:
- if c['displayName'] == self.cluster_name:
- parcels = c['parcels']
-
- cdh_parcel_enabled = False
- for parcel in parcels:
- if parcel['product'] == 'CDH':
- cdh_parcel_enabled = True
- break
-
- if cdh_parcel_enabled:
- parcel_config = self.hg.get('%s/api/v6/cm/allHosts/config' % self.url)
- # custom parcel dir exists
- if parcel_config['items'] and parcel_config['items'][0]['name'] == 'parcels_directory':
- hbase_lib_path = parcel_config['items'][0]['value'] + '/CDH/lib/hbase/lib'
- else:
- hbase_lib_path = PARCEL_HBASE_LIB
- else:
- hbase_lib_path = DEF_HBASE_LIB
- elif 'HDP' in self.distro:
- hbase_lib_path = HDP_HBASE_LIB
-
- return hbase_lib_path
-
class UserInput(object):
def __init__(self, options, pwd):
self.in_data = ParseJson(USER_PROMPT_FILE).load()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/284415ff/install/python-installer/scripts/common.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/common.py b/install/python-installer/scripts/common.py
index 98fbabc..c17cc38 100644
--- a/install/python-installer/scripts/common.py
+++ b/install/python-installer/scripts/common.py
@@ -40,7 +40,7 @@ except ImportError:
import xml.etree.ElementTree as ET
from ConfigParser import ConfigParser
from collections import defaultdict
-from constants import VERSION_FILE
+from constants import VERSION_FILE, PARCEL_HBASE_LIB, DEF_HBASE_LIB, HDP_HBASE_LIB
MARK = '[ERR]'
@@ -151,6 +151,160 @@ def write_file(template_file, string):
except IOError:
err('Failed to open file %s to write' % template_file)
+class HadoopDiscover(object):
+ """ discover for hadoop related info """
+ def __init__(self, user, pwd, url, cluster_name):
+ self.rsnodes = []
+ self.users = {}
+ self.cluster_name = cluster_name
+ self.hg = ParseHttp(user, pwd)
+ self.url = url
+ self.v1_url = '%s/api/v1/clusters' % self.url
+ self.v6_url = '%s/api/v6/clusters' % self.url
+ self.cluster_url = '%s/%s' % (self.v1_url, cluster_name.replace(' ', '%20'))
+ self._get_distro()
+ self._check_version()
+ if 'CDH' in self.distro:
+ self.cm = self.hg.get('%s/api/v6/cm/deployment' % self.url)
+
+ def _get_distro(self):
+ content = self.hg.get(self.v1_url)
+
+ if content['items'][0].has_key('name'):
+ # use v6 rest api for CDH to get fullversion
+ content = self.hg.get(self.v6_url)
+
+ # loop all managed clusters
+ for cluster in content['items']:
+ try:
+ # HDP
+ self.distro = cluster['Clusters']['version']
+ except KeyError:
+ # CDH
+ try:
+ if self.cluster_name == cluster['displayName']:
+ self.distro = 'CDH' + cluster['fullVersion']
+ break
+ except KeyError:
+ log_err('Failed to get hadoop distribution info from management url')
+
+ def get_hdfs_srvname(self):
+ return self._get_service_name('HDFS')
+
+ def get_hbase_srvname(self):
+ return self._get_service_name('HBASE')
+
+ def get_zookeeper_srvname(self):
+ return self._get_service_name('ZOOKEEPER')
+
+ def _get_service_name(self, service):
+ # CDH uses different service names in multiple clusters
+ if 'CDH' in self.distro:
+ services_cfgs = self.hg.get(self.cluster_url +'/services')
+ for item in services_cfgs['items']:
+ if item['type'] == service:
+ return item['name']
+ else:
+ return service.lower()
+
+ def _check_version(self):
+ version = Version()
+ if 'CDH' in self.distro: version_list = version.get_version('cdh')
+ if 'HDP' in self.distro: version_list = version.get_version('hdp')
+
+ has_version = 0
+ for ver in version_list:
+ if ver in self.distro: has_version = 1
+
+ if not has_version:
+ log_err('Sorry, currently Trafodion doesn\'t support %s version' % self.distro)
+
+ def get_hadoop_users(self):
+ if 'CDH' in self.distro:
+ self._get_cdh_users()
+ elif 'HDP' in self.distro or 'BigInsights' in self.distro:
+ self._get_hdp_users()
+ return self.users
+
+ def _get_hdp_users(self):
+ desired_cfg = self.hg.get('%s/?fields=Clusters/desired_configs' % (self.cluster_url))
+ config_type = {'hbase-env':'hbase_user', 'hadoop-env':'hdfs_user'}
+ for key, value in config_type.items():
+ desired_tag = desired_cfg['Clusters']['desired_configs'][key]['tag']
+ current_cfg = self.hg.get('%s/configurations?type=%s&tag=%s' % (self.cluster_url, key, desired_tag))
+ self.users[value] = current_cfg['items'][0]['properties'][value]
+
+ def _get_cdh_users(self):
+ def _get_username(service_name, hadoop_type):
+ cfg = self.hg.get('%s/services/%s/config' % (self.cluster_url, service_name))
+ if cfg.has_key('items'):
+ for item in cfg['items']:
+ if item['name'] == 'process_username':
+ return item['value']
+ return hadoop_type
+
+ hdfs_user = _get_username(self.get_hdfs_srvname(), 'hdfs')
+ hbase_user = _get_username(self.get_hbase_srvname(), 'hbase')
+
+ self.users = {'hbase_user':hbase_user, 'hdfs_user':hdfs_user}
+
+ def get_rsnodes(self):
+ if 'CDH' in self.distro:
+ self._get_rsnodes_cdh()
+ elif 'HDP' in self.distro or 'BigInsights' in self.distro:
+ self._get_rsnodes_hdp()
+
+ self.rsnodes.sort()
+ ## use short hostname
+ #try:
+ # self.rsnodes = [re.match(r'([\w\-]+).*', node).group(1) for node in self.rsnodes]
+ #except AttributeError:
+ # pass
+ return self.rsnodes
+
+ def _get_rsnodes_cdh(self):
+ """ get list of HBase RegionServer nodes in CDH """
+ hostids = []
+ for c in self.cm['clusters']:
+ if c['displayName'] == self.cluster_name:
+ for s in c['services']:
+ if s['type'] == 'HBASE':
+ for r in s['roles']:
+ if r['type'] == 'REGIONSERVER': hostids.append(r['hostRef']['hostId'])
+ for i in hostids:
+ for h in self.cm['hosts']:
+ if i == h['hostId']: self.rsnodes.append(h['hostname'])
+
+ def _get_rsnodes_hdp(self):
+ """ get list of HBase RegionServer nodes in HDP """
+ hdp = self.hg.get('%s/services/HBASE/components/HBASE_REGIONSERVER' % self.cluster_url)
+ self.rsnodes = [c['HostRoles']['host_name'] for c in hdp['host_components']]
+
+ def get_hbase_lib_path(self):
+ if 'CDH' in self.distro:
+ for c in self.cm['clusters']:
+ if c['displayName'] == self.cluster_name:
+ parcels = c['parcels']
+
+ cdh_parcel_enabled = False
+ for parcel in parcels:
+ if parcel['product'] == 'CDH':
+ cdh_parcel_enabled = True
+ break
+
+ if cdh_parcel_enabled:
+ parcel_config = self.hg.get('%s/api/v6/cm/allHosts/config' % self.url)
+ # custom parcel dir exists
+ if parcel_config['items'] and parcel_config['items'][0]['name'] == 'parcels_directory':
+ hbase_lib_path = parcel_config['items'][0]['value'] + '/CDH/lib/hbase/lib'
+ else:
+ hbase_lib_path = PARCEL_HBASE_LIB
+ else:
+ hbase_lib_path = DEF_HBASE_LIB
+ elif 'HDP' in self.distro:
+ hbase_lib_path = HDP_HBASE_LIB
+
+ return hbase_lib_path
class Version(object):
def __init__(self):
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/284415ff/install/python-installer/scripts/traf_addnodes.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_addnodes.py b/install/python-installer/scripts/traf_addnodes.py
new file mode 100755
index 0000000..917b799
--- /dev/null
+++ b/install/python-installer/scripts/traf_addnodes.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on new nodes with sudo user ###
+
+import os
+import sys
+import re
+import json
+import socket
+from constants import DEF_HBASE_HOME, TRAF_SUDOER_FILE, TRAF_CFG_FILE
+from common import err, cmd_output, run_cmd, get_default_home, mod_file, write_file
+
+def run():
+ dbcfgs = json.loads(dbcfgs_json)
+
+ if not os.path.exists(dbcfgs['hbase_lib_path']):
+ err('Cannot find HBase lib folder')
+ if not os.path.exists(dbcfgs['java_home']):
+ err('Cannot find Java, please set the JAVA_HOME on the new nodes to: %s' % dbcfgs['java_home'])
+
+ home_dir = get_default_home()
+ if dbcfgs.has_key('home_dir'):
+ home_dir = dbcfgs['home_dir']
+
+ traf_user = dbcfgs['traf_user']
+ traf_home = dbcfgs['traf_home']
+ traf_user_dir = '%s/%s' % (home_dir, traf_user)
+
+ traf_ver = dbcfgs['traf_version']
+# scratch_locs = dbcfgs['scratch_locs'].split(',')
+
+ SUDOER_CFG = """
+## Allow trafodion id to run commands needed for backup and restore
+%%%s ALL =(hbase) NOPASSWD: %s/bin/hbase"
+""" % (traf_user, DEF_HBASE_HOME)
+
+ ### add trafodion user ###
+ # create trafodion user and group
+ if cmd_output('getent passwd %s' % traf_user):
+ print 'user [%s] exists' % traf_user
+ # trafodion user exists, set actual trafodion group
+ traf_group = cmd_output('id -ng %s' % traf_user)
+ else:
+ # default trafodion group
+ traf_group = traf_user
+ if not cmd_output('getent group %s' % traf_group):
+ run_cmd('groupadd %s' % traf_group)
+ traf_shadow = dbcfgs['traf_shadow']
+ print 'Adding user [%s]' % traf_user
+ run_cmd('useradd --shell /bin/bash -m %s -g %s --home %s --password "%s"' % (traf_user, traf_group, traf_user_dir, traf_shadow))
+ print 'Added user [%s]' % traf_user
+
+ if not os.path.exists(traf_user_dir):
+ run_cmd('mkdir -p %s' % traf_user_dir)
+ run_cmd('chmod 700 %s' % traf_user_dir)
+
+ ### untar the copied trafoion binaries ###
+ TRAF_PKG_FILE = '/tmp/traf_bin.tar.gz'
+ run_cmd('mkdir -p %s' % traf_home)
+ run_cmd('mkdir -p /etc/trafodion')
+ run_cmd('tar xf %s -C %s' % (TRAF_PKG_FILE, traf_home))
+
+ run_cmd('mv -f /tmp/trafodion_config %s' % TRAF_CFG_FILE)
+ run_cmd('cp -rf /tmp/.ssh %s/..' % traf_home)
+ run_cmd('mv -f /tmp/hbase-trx-* %s' % dbcfgs['hbase_lib_path'])
+ run_cmd('mv -f /tmp/trafodion-utility-* %s' % dbcfgs['hbase_lib_path'])
+
+ ### copy trafodion bashrc ###
+ bashrc_template = '%s/sysinstall/home/trafodion/.bashrc' % traf_home
+ bashrc_file = '%s/%s/.bashrc' % (home_dir, traf_user)
+ # backup orig bashrc
+ if os.path.exists(bashrc_file):
+ run_cmd('cp -f %s %s.bak' % (bashrc_file, bashrc_file))
+ run_cmd('cp -f %s %s' % (bashrc_template, bashrc_file))
+
+ # set permission
+ run_cmd('chmod 700 %s/../.ssh' % traf_home)
+ cmd_output('chmod 600 %s/../.ssh/{id_rsa,config,authorized_keys}' % traf_home)
+ run_cmd('chmod 777 %s' % TRAF_CFG_FILE)
+ run_cmd('chown -R %s:%s /etc/trafodion' % (traf_user, traf_group))
+ run_cmd('chmod +r %s/{hbase-trx-*,trafodion-utility-*}' % dbcfgs['hbase_lib_path'])
+ run_cmd('chown -R %s:%s %s' % (traf_user, traf_group, traf_user_dir))
+
+ ### modify CLUSTERNAME ###
+ mod_file(TRAF_CFG_FILE, {'CLUSTERNAME=.*': 'CLUSTERNAME=%s' % socket.gethostname()})
+
+ ### kernel settings ###
+ run_cmd('echo "kernel.pid_max=65535" >> /etc/sysctl.conf')
+ run_cmd('echo "kernel.msgmnb=65536" >> /etc/sysctl.conf')
+ run_cmd('echo "kernel.msgmax=65536" >> /etc/sysctl.conf')
+ run_cmd('/sbin/sysctl -p /etc/sysctl.conf 2>&1 > /dev/null')
+
+ ### copy init script ###
+ init_script = '%s/sysinstall/etc/init.d/trafodion' % traf_home
+ if os.path.exists(init_script):
+ run_cmd('cp -rf %s /etc/init.d/' % init_script)
+ run_cmd('chkconfig --add trafodion')
+ run_cmd('chkconfig --level 06 trafodion on')
+
+ ### create and set permission for scratch file dir ###
+# for loc in scratch_locs:
+# # don't set permission for HOME folder
+# if not os.path.exists(loc):
+# run_cmd('mkdir -p %s' % loc)
+# if home_dir not in loc:
+# run_cmd('chmod 777 %s' % loc)
+
+ if dbcfgs['enable_ha'] == 'true':
+ # set trafodion sudoer file for specific cmds
+ SUDOER_CFG += """
+## Trafodion Floating IP commands
+Cmnd_Alias IP = /sbin/ip
+Cmnd_Alias ARP = /sbin/arping
+
+## Allow Trafodion id to run commands needed to configure floating IP
+%%%s ALL = NOPASSWD: IP, ARP
+""" % traf_user
+
+ ### write trafodion sudoer file ###
+ with open(TRAF_SUDOER_FILE, 'w') as f:
+ f.write(SUDOER_CFG)
+
+ # set ulimits for trafodion user
+ ulimits_file = '/etc/security/limits.d/%s.conf' % traf_user
+ ulimits_config = '''
+# Trafodion settings
+%s soft core unlimited
+%s hard core unlimited
+%s soft memlock unlimited
+%s hard memlock unlimited
+%s soft nofile 32768
+%s hard nofile 65536
+%s soft nproc 100000
+%s hard nproc 100000
+''' % ((traf_user,) * 8)
+
+ write_file(ulimits_file, ulimits_config)
+
+# main
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+run()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/284415ff/install/python-installer/scripts/traf_dep.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_dep.py b/install/python-installer/scripts/traf_dep.py
index 69ad489..57e9e62 100755
--- a/install/python-installer/scripts/traf_dep.py
+++ b/install/python-installer/scripts/traf_dep.py
@@ -92,7 +92,7 @@ def run():
run_cmd('yum install -y %s' % pkg)
# pdsh should not exist on single node
- if len(node_list) == 1:
+ if len(node_list) == 1 and not dbcfgs.has_key('traf_shadow'): # means we are running in adding node mode
cmd_output('yum remove -y pdsh')
# remove temp repo file
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/284415ff/install/python-installer/scripts/traf_user.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_user.py b/install/python-installer/scripts/traf_user.py
index 479405b..8229ecd 100755
--- a/install/python-installer/scripts/traf_user.py
+++ b/install/python-installer/scripts/traf_user.py
@@ -98,8 +98,6 @@ def run():
export TRAF_HOME="%s"
export MY_SQROOT=$TRAF_HOME # for compatibility
export JAVA_HOME="%s"
-export NODE_LIST="%s"
-export MY_NODES="%s"
export node_count="%s"
export HADOOP_TYPE="%s"
export ENABLE_HA="%s"
@@ -107,8 +105,17 @@ export ZOOKEEPER_NODES="%s"
export ZOOKEEPER_PORT="%s"
export SECURE_HADOOP="%s"
export CLUSTERNAME="%s"
-""" % (traf_home, dbcfgs['java_home'], ' '.join(nodes), ' -w ' + ' -w '.join(nodes),
- str(len(nodes)), hadoop_type, dbcfgs['enable_ha'], zk_nodes, zk_port, dbcfgs['secure_hadoop'], socket.gethostname())
+""" % (traf_home, dbcfgs['java_home'], str(len(nodes)), hadoop_type, dbcfgs['enable_ha'],
+ zk_nodes, zk_port, dbcfgs['secure_hadoop'], socket.gethostname())
+
+ # save additonal configs for elastic
+ trafodion_config += """
+export hbase_xml_file="%s"
+export hbase_lib_path="%s"
+export traf_user="%s"
+export traf_version="%s"
+export dcs_cnt_per_node="%s"
+""" % (dbcfgs['hbase_xml_file'], dbcfgs['hbase_lib_path'], dbcfgs['traf_user'], dbcfgs['traf_version'], dbcfgs['dcs_cnt_per_node'])
run_cmd('mkdir -p %s' % TRAF_CFG_DIR)
write_file(TRAF_CFG_FILE, trafodion_config)
@@ -125,7 +132,6 @@ export PATH=$PATH:$HADOOP_PREFIX/bin:$HADOOP_PREFIX/sbin:$HBASE_HOME/bin
# set permission
run_cmd('chown -R %s:%s %s*' % (traf_user, traf_group, TRAF_CFG_DIR))
-
# set ulimits for trafodion user
ulimits_config = '''
# Trafodion settings
[3/3] incubator-trafodion git commit: Merge [TRAFODION-2658] PR 1152
Support elastic node add in pyinstaller
Posted by db...@apache.org.
Merge [TRAFODION-2658] PR 1152 Support elastic node add in pyinstaller
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/765bb75d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/765bb75d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/765bb75d
Branch: refs/heads/master
Commit: 765bb75d3c4558f897f49122306b64a07bdc02c5
Parents: 05d14d4 6952473
Author: Dave Birdsall <db...@apache.org>
Authored: Wed Jul 5 22:35:22 2017 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Wed Jul 5 22:35:22 2017 +0000
----------------------------------------------------------------------
core/sqf/sql/scripts/sqregen | 9 +-
core/sqf/tools/sqtools.sh | 8 +-
install/python-installer/add_nodes.py | 216 +++++++++++++++++++
install/python-installer/configs/script.json | 25 +++
install/python-installer/db_install.py | 159 +-------------
install/python-installer/scripts/common.py | 156 +++++++++++++-
.../python-installer/scripts/traf_addnodes.py | 165 ++++++++++++++
install/python-installer/scripts/traf_dep.py | 2 +-
install/python-installer/scripts/traf_user.py | 16 +-
9 files changed, 585 insertions(+), 171 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/765bb75d/core/sqf/tools/sqtools.sh
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/765bb75d/install/python-installer/scripts/traf_user.py
----------------------------------------------------------------------
diff --cc install/python-installer/scripts/traf_user.py
index 6b2f85d,8229ecd..60e191c
--- a/install/python-installer/scripts/traf_user.py
+++ b/install/python-installer/scripts/traf_user.py
@@@ -96,11 -96,8 +96,9 @@@ def run()
nodes = dbcfgs['node_list'].split(',')
trafodion_config = """
export TRAF_HOME="%s"
+export TRAF_VAR=$TRAF_HOME/tmp
export MY_SQROOT=$TRAF_HOME # for compatibility
export JAVA_HOME="%s"
- export NODE_LIST="%s"
- export MY_NODES="%s"
export node_count="%s"
export HADOOP_TYPE="%s"
export ENABLE_HA="%s"
[2/3] incubator-trafodion git commit: change error msg in sqregen
Posted by db...@apache.org.
change error msg in sqregen
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/69524738
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/69524738
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/69524738
Branch: refs/heads/master
Commit: 6952473882c35d11fdab7a327af4b35e597fc8c1
Parents: 284415f
Author: Eason <hf...@gmail.com>
Authored: Tue Jul 4 08:49:40 2017 +0800
Committer: Eason <hf...@gmail.com>
Committed: Tue Jul 4 08:49:40 2017 +0800
----------------------------------------------------------------------
core/sqf/sql/scripts/sqregen | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/69524738/core/sqf/sql/scripts/sqregen
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/sqregen b/core/sqf/sql/scripts/sqregen
index e664ae1..d8c2c02 100755
--- a/core/sqf/sql/scripts/sqregen
+++ b/core/sqf/sql/scripts/sqregen
@@ -44,7 +44,7 @@ function RegenConfig {
cd $TRAF_HOME/sql/scripts
./regensq.pl $SQCONFIG_FILE
if [ $? != 0 ]; then
- echo "Failed to run regensq.pl."
+ echo "Regensq failed."
exit $?
fi
echo "Copying Trafodion configuration file $SQCONFIG_FILE to all nodes"