You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by sa...@apache.org on 2017/03/02 00:25:43 UTC
[1/2] incubator-trafodion git commit: [TRAFODION-2496] pyinstaller -
better handle constants
Repository: incubator-trafodion
Updated Branches:
refs/heads/release2.1 6cc89578a -> 58caf9fd3
[TRAFODION-2496] pyinstaller - better handle constants
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/dc58c85d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/dc58c85d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/dc58c85d
Branch: refs/heads/release2.1
Commit: dc58c85d3b0c7364461129ac7973c5f97b6576a8
Parents: 965f659
Author: Eason <hf...@gmail.com>
Authored: Thu Feb 23 16:04:48 2017 +0800
Committer: Eason <hf...@gmail.com>
Committed: Thu Feb 23 17:15:14 2017 +0800
----------------------------------------------------------------------
.../configs/db_config_default.ini | 7 +-
install/python-installer/db_install.py | 22 ++---
install/python-installer/db_uninstall.py | 14 +--
install/python-installer/discovery.py | 3 +-
install/python-installer/scripts/apache_mods.py | 3 +-
install/python-installer/scripts/common.py | 20 +----
install/python-installer/scripts/constants.py | 61 ++++++++++++++
install/python-installer/scripts/copy_files.py | 21 +----
install/python-installer/scripts/dcs_setup.py | 38 ++++-----
install/python-installer/scripts/hadoop_mods.py | 3 +-
install/python-installer/scripts/hdfs_cmds.py | 13 +--
.../python-installer/scripts/traf_discover.py | 26 +++---
.../python-installer/scripts/traf_kerberos.py | 2 +-
install/python-installer/scripts/traf_ldap.py | 26 +++---
.../python-installer/scripts/traf_package.py | 11 +--
install/python-installer/scripts/traf_setup.py | 64 +++++++-------
.../python-installer/scripts/traf_sqconfig.py | 8 +-
install/python-installer/scripts/traf_start.py | 6 +-
install/python-installer/scripts/traf_user.py | 89 ++++++++++----------
install/python-installer/scripts/wrapper.py | 23 +++--
20 files changed, 242 insertions(+), 218 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/configs/db_config_default.ini
----------------------------------------------------------------------
diff --git a/install/python-installer/configs/db_config_default.ini b/install/python-installer/configs/db_config_default.ini
index 87efb4f..918bf03 100644
--- a/install/python-installer/configs/db_config_default.ini
+++ b/install/python-installer/configs/db_config_default.ini
@@ -96,9 +96,8 @@ local_repo_dir =
# set it to 'Y' if enable LDAP security
ldap_security = N
-# LDAP user name and password to be assigned as DB admin privilege
+# LDAP user name to be assigned as DB admin privilege
db_admin_user = admin
-db_admin_pwd = traf123
# LDAP user to be assigned DB root privileges (DB__ROOT)
db_root_user = trafodion
# if LDAP security is enabled, provide the following items
@@ -112,8 +111,8 @@ ldap_certpath =
# set to Y if user info is needed
ldap_userinfo = N
# provide if ldap_userinfo = Y
-ladp_user =
-ladp_pwd =
+ldap_user =
+ldap_pwd =
##################################
# Kerberos security configuration
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/db_install.py
----------------------------------------------------------------------
diff --git a/install/python-installer/db_install.py b/install/python-installer/db_install.py
index 269ad05..d54a267 100755
--- a/install/python-installer/db_install.py
+++ b/install/python-installer/db_install.py
@@ -40,9 +40,10 @@ except ImportError:
print 'Python module prettytable is not found. Install python-prettytable first.'
exit(1)
from scripts import wrapper
-from scripts.common import DEF_PORT_FILE, DBCFG_FILE, USER_PROMPT_FILE, DBCFG_TMP_FILE, \
- INSTALLER_LOC, Remote, Version, ParseHttp, ParseInI, ParseJson, \
- http_start, http_stop, format_output, err_m, expNumRe, run_cmd, info
+from scripts.constants import DEF_PORT_FILE, DBCFG_FILE, USER_PROMPT_FILE, DBCFG_TMP_FILE, INSTALLER_LOC, \
+ DEF_HBASE_XML_FILE, PARCEL_HBASE_LIB, DEF_HBASE_LIB, HDP_HBASE_LIB, TRAF_USER
+from scripts.common import Remote, Version, ParseHttp, ParseInI, ParseJson, run_cmd, info, \
+ http_start, http_stop, format_output, err_m, expNumRe
# init global cfgs for user input
cfgs = defaultdict(str)
@@ -188,11 +189,11 @@ class HadoopDiscover(object):
if parcel_config['items'] and parcel_config['items'][0]['name'] == 'parcels_directory':
hbase_lib_path = parcel_config['items'][0]['value'] + '/CDH/lib/hbase/lib'
else:
- hbase_lib_path = '/opt/cloudera/parcels/CDH/lib/hbase/lib'
+ hbase_lib_path = PARCEL_HBASE_LIB
else:
- hbase_lib_path = '/usr/lib/hbase/lib'
+ hbase_lib_path = DEF_HBASE_LIB
elif 'HDP' in self.distro:
- hbase_lib_path = '/usr/hdp/current/hbase-regionserver/lib'
+ hbase_lib_path = HDP_HBASE_LIB
return hbase_lib_path
@@ -432,12 +433,12 @@ def user_input(options, prompt_mode=True, pwd=''):
# set some system default configs
cfgs['config_created_date'] = time.strftime('%Y/%m/%d %H:%M %Z')
- cfgs['traf_user'] = 'trafodion'
+ cfgs['traf_user'] = TRAF_USER
if apache:
cfgs['hbase_xml_file'] = cfgs['hbase_home'] + '/conf/hbase-site.xml'
cfgs['hdfs_xml_file'] = cfgs['hadoop_home'] + '/etc/hadoop/hdfs-site.xml'
else:
- cfgs['hbase_xml_file'] = '/etc/hbase/conf/hbase-site.xml'
+ cfgs['hbase_xml_file'] = DEF_HBASE_XML_FILE
### discover system settings, return a dict
system_discover = wrapper.run(cfgs, options, mode='discover', pwd=pwd)
@@ -462,6 +463,8 @@ def user_input(options, prompt_mode=True, pwd=''):
log_err('HBase is not found')
if content_dict['hbase'] == 'N/S':
log_err('HBase version is not supported')
+ else:
+ cfgs['hbase_ver'] = content_dict['hbase']
if content_dict['home_dir']: # trafodion user exists
has_home_dir += 1
cfgs['home_dir'] = content_dict['home_dir']
@@ -628,9 +631,6 @@ def main():
else:
print '\n** Loading configs from config file ... \n'
cfgs = p.load()
- # remove java home info from default config file
- if config_file == DBCFG_FILE and cfgs.has_key('java_home'):
- cfgs.pop('java_home')
if options.offline and cfgs['offline_mode'] != 'Y':
log_err('To enable offline mode, must set "offline_mode = Y" in config file')
user_input(options, prompt_mode=False, pwd=pwd)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/db_uninstall.py
----------------------------------------------------------------------
diff --git a/install/python-installer/db_uninstall.py b/install/python-installer/db_uninstall.py
index fb3f42f..5ab23e7 100755
--- a/install/python-installer/db_uninstall.py
+++ b/install/python-installer/db_uninstall.py
@@ -26,12 +26,10 @@ import os
import re
import getpass
from optparse import OptionParser
+from scripts.constants import INSTALLER_LOC, TRAF_CFG_FILE, TRAF_CFG_DIR, TRAF_USER, DBCFG_FILE
from scripts.common import run_cmd, format_output, err_m, \
expNumRe, ParseInI, Remote, info
-TRAFODION_CFG_FILE = '/etc/trafodion/trafodion_config'
-TRAF_USER = 'trafodion'
-
def get_options():
usage = 'usage: %prog [options]\n'
usage += ' Trafodion uninstall script. It will remove \n\
@@ -67,14 +65,14 @@ def main():
node_list = ''
# parse node list from trafodion_config
- if os.path.exists(TRAFODION_CFG_FILE):
- with open(TRAFODION_CFG_FILE, 'r') as f:
+ if os.path.exists(TRAF_CFG_FILE):
+ with open(TRAF_CFG_FILE, 'r') as f:
traf_cfgs = f.readlines()
try:
line = [l for l in traf_cfgs if 'NODE_LIST' in l][0]
node_list = re.search(r'NODE_LIST="(.*)"', line).groups()[0]
except Exception as e:
- err_m('Cannot find node list info from %s: %s' % (TRAFODION_CFG_FILE, e))
+ err_m('Cannot find node list info from %s: %s' % (TRAF_CFG_FILE, e))
# parse node list from installation config file
elif options.cfgfile:
if not os.path.exists(options.cfgfile):
@@ -104,10 +102,12 @@ def main():
for remote in remotes:
info('Remove Trafodion on node [%s] ...' % remote.host)
remote.execute('ps -ef|grep ^%s|awk \'{print $2}\'|xargs sudo kill -9' % TRAF_USER, chkerr=False)
+ remote.execute('ps -ef|grep krb5check|awk \'{print $2}\'|xargs sudo kill -9', chkerr=False) # traf kerberos service
remote.execute('sudo -n /usr/sbin/userdel -rf %s' % TRAF_USER, chkerr=False)
remote.execute('sudo -n /usr/sbin/groupdel %s' % TRAF_USER, chkerr=False)
- remote.execute('sudo -n rm -rf /etc/security/limits.d/trafodion.conf /etc/trafodion /tmp/hsperfdata_%s 2>/dev/null' % TRAF_USER, chkerr=False)
+ remote.execute('sudo -n rm -rf /etc/security/limits.d/%s.conf %s /tmp/hsperfdata_%s 2>/dev/null' % (TRAF_USER, TRAF_CFG_DIR, TRAF_USER), chkerr=False)
+ run_cmd('rm -f %s/*.status %s' % (INSTALLER_LOC, DBCFG_FILE))
format_output('Trafodion Uninstall Completed')
if __name__ == "__main__":
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/discovery.py
----------------------------------------------------------------------
diff --git a/install/python-installer/discovery.py b/install/python-installer/discovery.py
index fe7839d..672384a 100755
--- a/install/python-installer/discovery.py
+++ b/install/python-installer/discovery.py
@@ -32,7 +32,8 @@ try:
except ImportError:
print 'Python module prettytable is not found. Install python-prettytable first.'
exit(1)
-from scripts.common import err_m, err, ParseInI, expNumRe, format_output, DBCFG_FILE
+from scripts.constants import DBCFG_FILE
+from scripts.common import err_m, err, ParseInI, expNumRe, format_output
from scripts import wrapper
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/scripts/apache_mods.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/apache_mods.py b/install/python-installer/scripts/apache_mods.py
index 17a54d5..e7e8eed 100755
--- a/install/python-installer/scripts/apache_mods.py
+++ b/install/python-installer/scripts/apache_mods.py
@@ -26,7 +26,8 @@
import sys
import json
import socket
-from common import MODCFG_FILE, ParseJson, ParseXML, err, run_cmd
+from constants import MODCFG_FILE
+from common import ParseJson, ParseXML, err, run_cmd
def run():
dbcfgs = json.loads(dbcfgs_json)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/scripts/common.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/common.py b/install/python-installer/scripts/common.py
index daa0944..7bc4590 100644
--- a/install/python-installer/scripts/common.py
+++ b/install/python-installer/scripts/common.py
@@ -40,28 +40,10 @@ except ImportError:
import xml.etree.ElementTree as ET
from ConfigParser import ConfigParser
from collections import defaultdict
+from constants import VERSION_FILE
-INSTALLER_LOC = re.search('(.*)/\w+',os.path.dirname(os.path.abspath(__file__))).groups()[0]
-
-CONFIG_DIR = INSTALLER_LOC + '/configs'
-SCRIPTS_DIR = INSTALLER_LOC + '/scripts'
-
-USER_PROMPT_FILE = CONFIG_DIR + '/prompt.json'
-SCRCFG_FILE = CONFIG_DIR + '/script.json'
-VERSION_FILE = CONFIG_DIR + '/version.json'
-MODCFG_FILE = CONFIG_DIR + '/mod_cfgs.json'
-DEF_PORT_FILE = CONFIG_DIR + '/default_ports.ini'
-
-DBCFG_FILE = INSTALLER_LOC + '/db_config'
-DBCFG_TMP_FILE = INSTALLER_LOC + '/.db_config_temp'
-
-TMP_DIR = '/tmp/.trafodion_install_temp'
MARK = '[ERR]'
-def version():
- print 'Installer version: %s' % __VERSION__
- exit(0)
-
def ok(msg):
print '\n\33[32m***[OK]: %s \33[0m' % msg
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/scripts/constants.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/constants.py b/install/python-installer/scripts/constants.py
new file mode 100644
index 0000000..b7d3096
--- /dev/null
+++ b/install/python-installer/scripts/constants.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### The common constants ###
+
+import os
+import re
+
+INSTALLER_LOC = re.search('(.*)/\w+',os.path.dirname(os.path.abspath(__file__))).groups()[0]
+
+CONFIG_DIR = INSTALLER_LOC + '/configs'
+SCRIPTS_DIR = INSTALLER_LOC + '/scripts'
+TEMPLATES_DIR = INSTALLER_LOC + '/templates'
+
+USER_PROMPT_FILE = CONFIG_DIR + '/prompt.json'
+SCRCFG_FILE = CONFIG_DIR + '/script.json'
+VERSION_FILE = CONFIG_DIR + '/version.json'
+MODCFG_FILE = CONFIG_DIR + '/mod_cfgs.json'
+DEF_PORT_FILE = CONFIG_DIR + '/default_ports.ini'
+
+DBCFG_FILE = INSTALLER_LOC + '/db_config'
+DBCFG_TMP_FILE = INSTALLER_LOC + '/.db_config_temp'
+
+SSHKEY_FILE = '/tmp/id_rsa'
+TMP_DIR = '/tmp/.trafodion_install_temp'
+
+DEF_HBASE_HOME = '/usr'
+DEF_HBASE_XML_FILE = '/etc/hbase/conf/hbase-site.xml'
+DEF_CORE_SITE_XML = '/etc/hadoop/conf/core-site.xml'
+DEF_HDFS_BIN = '/usr/bin/hdfs'
+DEF_HBASE_LIB = '/usr/lib/hbase/lib'
+HDP_HBASE_LIB = '/usr/hdp/current/hbase-regionserver/lib'
+PARCEL_DIR = '/opt/cloudera/parcels'
+PARCEL_HBASE_LIB = PARCEL_DIR + '/CDH/lib/hbase/lib'
+PARCEL_HDFS_BIN = PARCEL_DIR + '/CDH/bin/hdfs'
+
+TRAF_HSPERFDATA_FILE = '/tmp/hsperfdata_trafodion'
+TRAF_SUDOER_FILE = '/etc/sudoers.d/trafodion'
+TRAF_CFG_DIR = '/etc/trafodion/'
+TRAF_CFG_FILE = '/etc/trafodion/trafodion_config'
+TRAF_USER = 'trafodion'
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/scripts/copy_files.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/copy_files.py b/install/python-installer/scripts/copy_files.py
index baca6c3..ef79265 100755
--- a/install/python-installer/scripts/copy_files.py
+++ b/install/python-installer/scripts/copy_files.py
@@ -26,6 +26,7 @@
import sys
import json
from threading import Thread
+from constants import SSHKEY_FILE
from common import ParseJson, Remote, run_cmd, err
def run(pwd):
@@ -36,23 +37,10 @@ def run(pwd):
hosts = dbcfgs['node_list'].split(',')
traf_package = dbcfgs['traf_package']
- # save db configs to a tmp file and copy to all trafodion nodes
- dbcfgs_file = '/tmp/dbcfgs'
- p = ParseJson(dbcfgs_file)
- # remove password from config file
- try:
- dbcfgs.pop('mgr_pwd')
- dbcfgs.pop('traf_pwd')
- dbcfgs.pop('kdcadmin_pwd')
- except KeyError:
- pass
- p.save(dbcfgs)
+ run_cmd('sudo -n rm -rf %s*' % SSHKEY_FILE)
+ run_cmd('sudo -n echo -e "y" | ssh-keygen -t rsa -N "" -f %s' % SSHKEY_FILE)
- key_file = '/tmp/id_rsa'
- run_cmd('sudo -n rm -rf %s*' % key_file)
- run_cmd('sudo -n echo -e "y" | ssh-keygen -t rsa -N "" -f %s' % key_file)
-
- files = [key_file, key_file+'.pub', traf_package, dbcfgs_file]
+ files = [SSHKEY_FILE, SSHKEY_FILE+'.pub', traf_package]
remote_insts = [Remote(h, pwd=pwd) for h in hosts]
threads = [Thread(target=r.copy, args=(files, '/tmp')) for r in remote_insts]
@@ -61,7 +49,6 @@ def run(pwd):
for r in remote_insts:
if r.rc != 0: err('Failed to copy files to %s' % r.host)
-
# main
try:
dbcfgs_json = sys.argv[1]
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/scripts/dcs_setup.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/dcs_setup.py b/install/python-installer/scripts/dcs_setup.py
index bb8a712..0d94da0 100755
--- a/install/python-installer/scripts/dcs_setup.py
+++ b/install/python-installer/scripts/dcs_setup.py
@@ -26,23 +26,23 @@
import os
import sys
import json
-from common import append_file, write_file, mod_file, cmd_output, run_cmd, \
- ParseInI, ParseXML, DEF_PORT_FILE, err
+from constants import TRAF_CFG_FILE
+from common import append_file, write_file, mod_file, cmd_output, \
+ ParseInI, ParseXML, err, run_cmd
def run():
dbcfgs = json.loads(dbcfgs_json)
- TRAF_HOME = os.environ['TRAF_HOME']
- TRAF_VER = dbcfgs['traf_version']
- HBASE_XML_FILE = dbcfgs['hbase_xml_file']
+ traf_home = os.environ['TRAF_HOME']
+ traf_ver = dbcfgs['traf_version']
+ hbase_xml_file = dbcfgs['hbase_xml_file']
- DCS_CONF_DIR = '%s/dcs-%s/conf' % (TRAF_HOME, TRAF_VER)
- DCS_SRV_FILE = DCS_CONF_DIR + '/servers'
- DCS_MASTER_FILE = DCS_CONF_DIR + '/master'
- DCS_BKMASTER_FILE = DCS_CONF_DIR + '/backup-masters'
- DCS_SITE_FILE = DCS_CONF_DIR + '/dcs-site.xml'
- REST_SITE_FILE = '%s/rest-%s/conf/rest-site.xml' % (TRAF_HOME, TRAF_VER)
- TRAFODION_CFG_FILE = '/etc/trafodion/trafodion_config'
+ dcs_conf_dir = '%s/dcs-%s/conf' % (traf_home, traf_ver)
+ dcs_srv_file = dcs_conf_dir + '/servers'
+ dcs_master_file = dcs_conf_dir + '/master'
+ dcs_bkmaster_file = dcs_conf_dir + '/backup-masters'
+ dcs_site_file = dcs_conf_dir + '/dcs-site.xml'
+ rest_site_file = '%s/rest-%s/conf/rest-site.xml' % (traf_home, traf_ver)
### dcs setting ###
# servers
@@ -52,20 +52,20 @@ def run():
for node in nodes:
dcs_servers += '%s %s\n' % (node, dcs_cnt)
- write_file(DCS_SRV_FILE, dcs_servers)
+ write_file(dcs_srv_file, dcs_servers)
### modify dcs config files ###
# modify master
dcs_master = nodes[0]
- append_file(DCS_MASTER_FILE, dcs_master)
+ append_file(dcs_master_file, dcs_master)
# modify dcs-site.xml
net_interface = run_cmd('ip route |grep default|awk \'{print $5}\'')
- hb = ParseXML(HBASE_XML_FILE)
+ hb = ParseXML(hbase_xml_file)
zk_hosts = hb.get_property('hbase.zookeeper.quorum')
zk_port = hb.get_property('hbase.zookeeper.property.clientPort')
- p = ParseXML(DCS_SITE_FILE)
+ p = ParseXML(dcs_site_file)
p.add_property('dcs.zookeeper.property.clientPort', zk_port)
p.add_property('dcs.zookeeper.quorum', zk_hosts)
p.add_property('dcs.dns.interface', net_interface)
@@ -80,16 +80,16 @@ def run():
# set DCS_MASTER_FLOATING_IP ENV for trafci
dcs_floating_ip_cfg = 'export DCS_MASTER_FLOATING_IP=%s' % dcs_floating_ip
- append_file(TRAFODION_CFG_FILE, dcs_floating_ip_cfg)
+ append_file(TRAF_CFG_FILE, dcs_floating_ip_cfg)
# modify backup_master
for dcs_backup_node in dcs_backup_nodes.split(','):
- append_file(DCS_BKMASTER_FILE, dcs_backup_node)
+ append_file(dcs_bkmaster_file, dcs_backup_node)
p.write_xml()
### rest setting ###
- p = ParseXML(REST_SITE_FILE)
+ p = ParseXML(rest_site_file)
p.add_property('rest.zookeeper.property.clientPort', zk_port)
p.add_property('rest.zookeeper.quorum', zk_hosts)
p.write_xml()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/scripts/hadoop_mods.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/hadoop_mods.py b/install/python-installer/scripts/hadoop_mods.py
index 3331673..e2ca95a 100755
--- a/install/python-installer/scripts/hadoop_mods.py
+++ b/install/python-installer/scripts/hadoop_mods.py
@@ -26,7 +26,8 @@
import time
import sys
import json
-from common import ParseHttp, ParseJson, MODCFG_FILE, err, info, retry
+from constants import MODCFG_FILE
+from common import ParseHttp, ParseJson, err, info, retry
modcfgs = ParseJson(MODCFG_FILE).load()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/scripts/hdfs_cmds.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/hdfs_cmds.py b/install/python-installer/scripts/hdfs_cmds.py
index 76a5a07..3b9a332 100755
--- a/install/python-installer/scripts/hdfs_cmds.py
+++ b/install/python-installer/scripts/hdfs_cmds.py
@@ -26,18 +26,19 @@
import os
import sys
import json
+from constants import DEF_HDFS_BIN, PARCEL_HBASE_LIB, PARCEL_HDFS_BIN
from common import err, run_cmd, cmd_output, run_cmd_as_user
def run():
- hdfs_bin = '/usr/bin/hdfs'
+ hdfs_bin = DEF_HDFS_BIN
dbcfgs = json.loads(dbcfgs_json)
- DISTRO = dbcfgs['distro']
+ distro = dbcfgs['distro']
- if 'CDH' in DISTRO:
- parcel_lib = '/opt/cloudera/parcels/CDH/lib/hbase/lib'
- if os.path.exists(parcel_lib): hdfs_bin = '/opt/cloudera/parcels/CDH/bin/hdfs'
- elif 'APACHE' in DISTRO:
+ if 'CDH' in distro:
+ parcel_lib = PARCEL_HBASE_LIB
+ if os.path.exists(parcel_lib): hdfs_bin = PARCEL_HDFS_BIN
+ elif 'APACHE' in distro:
hdfs_bin = dbcfgs['hadoop_home'] + '/bin/hdfs'
traf_loc = '/user/trafodion'
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/scripts/traf_discover.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_discover.py b/install/python-installer/scripts/traf_discover.py
index c0b5f96..28d55d1 100755
--- a/install/python-installer/scripts/traf_discover.py
+++ b/install/python-installer/scripts/traf_discover.py
@@ -29,6 +29,7 @@ import json
import sys
import platform
from glob import glob
+from constants import DEF_CORE_SITE_XML, DEF_HBASE_HOME
from common import cmd_output, err, Version, ParseXML
PREFIX = 'get_'
@@ -127,14 +128,13 @@ class Discover(object):
except AttributeError:
continue
- if not jdk_list:
- return NA
+ # auto detect JDK1.8/1.7
+ if jdk_list.has_key('1.8.0'):
+ return jdk_list['1.8.0']
+ elif jdk_list.has_key('1.7.0'):
+ return jdk_list['1.7.0']
else:
- # use JDK1.8 first
- if jdk_list.has_key('1.8.0'):
- return jdk_list['1.8.0']
- elif jdk_list.has_key('1.7.0'):
- return jdk_list['1.7.0']
+ return NA
@deco
def get_hive(self):
@@ -147,12 +147,12 @@ class Discover(object):
def _get_core_site_info(self, name):
if self.dbcfgs.has_key('hadoop_home'): # apache distro
- CORE_SITE_XML = '%s/etc/hadoop/core-site.xml' % self.dbcfgs['hadoop_home']
+ core_site_xml = '%s/etc/hadoop/core-site.xml' % self.dbcfgs['hadoop_home']
else:
- CORE_SITE_XML = '/etc/hadoop/conf/core-site.xml'
+ core_site_xml = DEF_CORE_SITE_XML
- if os.path.exists(CORE_SITE_XML):
- p = ParseXML(CORE_SITE_XML)
+ if os.path.exists(core_site_xml):
+ p = ParseXML(core_site_xml)
return p.get_property(name)
else:
return NA
@@ -166,9 +166,9 @@ class Discover(object):
""" get HBase version """
if self.dbcfgs.has_key('hbase_home'): # apache distro
hbase_home = self.dbcfgs['hbase_home']
- hbase_ver = cmd_output('%s/bin/hbase version | head -n1' % hbase_home)
else:
- hbase_ver = cmd_output('hbase version | head -n1')
+ hbase_home = DEF_HBASE_HOME
+ hbase_ver = cmd_output('%s/bin/hbase version | head -n1' % hbase_home)
support_hbase_ver = self.version.get_version('hbase')
try:
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/scripts/traf_kerberos.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_kerberos.py b/install/python-installer/scripts/traf_kerberos.py
index fc50392..b42a596 100755
--- a/install/python-installer/scripts/traf_kerberos.py
+++ b/install/python-installer/scripts/traf_kerberos.py
@@ -82,7 +82,7 @@ def run():
elif 'HDP' in distro:
hdfs_keytab = '/etc/security/keytabs/hdfs.headless.keytab'
hbase_keytab = '/etc/security/keytabs/hbase.service.keytab'
- hdfs_principal = '%s/%s@%s' % (hdfs_user, cluster_name, realm)
+ hdfs_principal = '%s-%s@%s' % (hdfs_user, cluster_name, realm)
run_cmd('sudo -u %s kinit -kt %s %s' % (hdfs_user, hdfs_keytab, hdfs_principal))
run_cmd('sudo -u %s kinit -kt %s %s' % (hbase_user, hbase_keytab, hbase_principal))
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/scripts/traf_ldap.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_ldap.py b/install/python-installer/scripts/traf_ldap.py
index dbe3417..f33531b 100755
--- a/install/python-installer/scripts/traf_ldap.py
+++ b/install/python-installer/scripts/traf_ldap.py
@@ -26,17 +26,17 @@
import os
import sys
import json
-from common import run_cmd, mod_file, err, TMP_DIR
+from common import run_cmd, mod_file, err
def run():
""" setup LDAP security """
dbcfgs = json.loads(dbcfgs_json)
- DB_ROOT_USER = dbcfgs['db_root_user']
- TRAF_HOME = os.environ['TRAF_HOME']
- SQENV_FILE = TRAF_HOME + '/sqenvcom.sh'
- TRAF_AUTH_CONFIG = '%s/sql/scripts/.traf_authentication_config' % TRAF_HOME
- TRAF_AUTH_TEMPLATE = '%s/sql/scripts/traf_authentication_config' % TRAF_HOME
+ db_root_user = dbcfgs['db_root_user']
+ traf_home = os.environ['TRAF_HOME']
+ sqenv_file = traf_home + '/sqenvcom.sh'
+ traf_auth_config = '%s/sql/scripts/.traf_authentication_config' % traf_home
+ traf_auth_template = '%s/sql/scripts/traf_authentication_config' % traf_home
# set traf_authentication_config file
change_items = {
@@ -50,19 +50,19 @@ def run():
}
print 'Modify authentication config file'
- run_cmd('cp %s %s' % (TRAF_AUTH_TEMPLATE, TRAF_AUTH_CONFIG))
- mod_file(TRAF_AUTH_CONFIG, change_items)
+ run_cmd('cp %s %s' % (traf_auth_template, traf_auth_config))
+ mod_file(traf_auth_config, change_items)
print 'Check LDAP Configuration file for errors'
- run_cmd('ldapconfigcheck -file %s' % TRAF_AUTH_CONFIG)
+ run_cmd('ldapconfigcheck -file %s' % traf_auth_config)
- print 'Verify that LDAP user %s exists' % DB_ROOT_USER
- run_cmd('ldapcheck --verbose --username=%s' % DB_ROOT_USER)
+ print 'Verify that LDAP user %s exists' % db_root_user
+ run_cmd('ldapcheck --verbose --username=%s' % db_root_user)
#if not 'Authentication successful' in ldapcheck_result:
- # err('Failed to access LDAP server with user %s' % DB_ROOT_USER)
+ # err('Failed to access LDAP server with user %s' % db_root_user)
print 'Modfiy sqenvcom.sh to turn on authentication'
- mod_file(SQENV_FILE, {'TRAFODION_ENABLE_AUTHENTICATION=.*\n':'TRAFODION_ENABLE_AUTHENTICATION=YES\n'})
+ mod_file(sqenv_file, {'TRAFODION_ENABLE_AUTHENTICATION=NO':'TRAFODION_ENABLE_AUTHENTICATION=YES'})
# main
try:
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/scripts/traf_package.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_package.py b/install/python-installer/scripts/traf_package.py
index c5bb6ec..5695103 100755
--- a/install/python-installer/scripts/traf_package.py
+++ b/install/python-installer/scripts/traf_package.py
@@ -30,15 +30,12 @@ from common import run_cmd, err
def run():
dbcfgs = json.loads(dbcfgs_json)
- TRAF_DIRNAME = dbcfgs['traf_dirname']
+ traf_dirname = dbcfgs['traf_dirname']
# untar traf package, package comes from copy_files.py
- TRAF_PACKAGE_FILE = '/tmp/' + dbcfgs['traf_package'].split('/')[-1]
- run_cmd('mkdir -p ~/%s' % TRAF_DIRNAME)
- run_cmd('tar xf %s -C ~/%s' % (TRAF_PACKAGE_FILE, TRAF_DIRNAME))
-
- # copy dbcfgs file
- run_cmd('cp -rf /tmp/dbcfgs ~/.dbcfgs')
+ traf_package_file = '/tmp/' + dbcfgs['traf_package'].split('/')[-1]
+ run_cmd('mkdir -p ~/%s' % traf_dirname)
+ run_cmd('tar xf %s -C ~/%s' % (traf_package_file, traf_dirname))
print 'Trafodion package extracted successfully!'
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/scripts/traf_setup.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_setup.py b/install/python-installer/scripts/traf_setup.py
index c0b090c..bb620b9 100755
--- a/install/python-installer/scripts/traf_setup.py
+++ b/install/python-installer/scripts/traf_setup.py
@@ -27,74 +27,72 @@ import os
import sys
import re
import json
+from constants import DEF_HBASE_HOME, TRAF_SUDOER_FILE
from common import err, cmd_output, run_cmd
def run():
dbcfgs = json.loads(dbcfgs_json)
- HOME_DIR = cmd_output('sudo cat /etc/default/useradd |grep HOME |cut -d "=" -f 2').strip()
+ home_dir = cmd_output('sudo cat /etc/default/useradd |grep HOME |cut -d "=" -f 2')
if dbcfgs.has_key('home_dir'):
- HOME_DIR = dbcfgs['home_dir']
+ home_dir = dbcfgs['home_dir']
- TRAF_USER = dbcfgs['traf_user']
- TRAF_DIRNAME = dbcfgs['traf_dirname']
- TRAF_HOME = '%s/%s/%s' % (HOME_DIR, TRAF_USER, TRAF_DIRNAME)
+ traf_user = dbcfgs['traf_user']
+ traf_dirname = dbcfgs['traf_dirname']
+ traf_home = '%s/%s/%s' % (home_dir, traf_user, traf_dirname)
- TRAF_VER = dbcfgs['traf_version']
- DISTRO = dbcfgs['distro']
- TRAF_LIB_PATH = TRAF_HOME + '/export/lib'
- SCRATCH_LOCS = dbcfgs['scratch_locs'].split(',')
+ traf_ver = dbcfgs['traf_version']
+ distro = dbcfgs['distro']
+ traf_lib_path = traf_home + '/export/lib'
+ scratch_locs = dbcfgs['scratch_locs'].split(',')
- SUDOER_FILE = '/etc/sudoers.d/trafodion'
SUDOER_CFG = """
## Allow trafodion id to run commands needed for backup and restore
-%%%s ALL =(hbase) NOPASSWD: /usr/bin/hbase"
-""" % TRAF_USER
+%%%s ALL =(hbase) NOPASSWD: %s/bin/hbase"
+""" % (traf_user, DEF_HBASE_HOME)
### kernel settings ###
run_cmd('sysctl -w kernel.pid_max=65535 2>&1 > /dev/null')
run_cmd('echo "kernel.pid_max=65535" >> /etc/sysctl.conf')
### copy trafodion bashrc ###
- BASHRC_TEMPLATE = '%s/sysinstall/home/trafodion/.bashrc' % TRAF_HOME
- BASHRC_FILE = '%s/%s/.bashrc' % (HOME_DIR, TRAF_USER)
- run_cmd('cp -f %s %s' % (BASHRC_TEMPLATE, BASHRC_FILE))
- run_cmd('chown -R %s:%s %s*' % (TRAF_USER, TRAF_USER, BASHRC_FILE))
+ bashrc_template = '%s/sysinstall/home/trafodion/.bashrc' % traf_home
+ bashrc_file = '%s/%s/.bashrc' % (home_dir, traf_user)
+ # backup orig bashrc
+ if os.path.exists(bashrc_file):
+ run_cmd('cp -f %s %s.bak' % (bashrc_file, bashrc_file))
+ run_cmd('cp -f %s %s' % (bashrc_template, bashrc_file))
+ run_cmd('chown -R %s:%s %s*' % (traf_user, traf_user, bashrc_file))
### copy init script ###
- init_script = '%s/sysinstall/etc/init.d/trafodion' % TRAF_HOME
+ init_script = '%s/sysinstall/etc/init.d/trafodion' % traf_home
if os.path.exists(init_script):
run_cmd('cp -rf %s /etc/init.d/' % init_script)
run_cmd('chkconfig --add trafodion')
run_cmd('chkconfig --level 06 trafodion on')
### create and set permission for scratch file dir ###
- for loc in SCRATCH_LOCS:
+ for loc in scratch_locs:
# don't set permission for HOME folder
if not os.path.exists(loc):
run_cmd('mkdir -p %s' % loc)
- if HOME_DIR not in loc:
+ if home_dir not in loc:
run_cmd('chmod 777 %s' % loc)
### copy jar files ###
hbase_lib_path = dbcfgs['hbase_lib_path']
- if 'APACHE' in DISTRO:
- hbase_home = dbcfgs['hbase_home']
- hbase_lib_path = hbase_home + '/lib'
- # for apache distro, get hbase version from cmdline
- hbase_ver = cmd_output('%s/bin/hbase version | head -n1' % hbase_home)
- hbase_ver = re.search(r'HBase (\d\.\d)', hbase_ver).groups()[0]
- DISTRO += hbase_ver
-
- distro, v1, v2 = re.search(r'(\w+)-*(\d)\.(\d)', DISTRO).groups()
+ if 'APACHE' in distro:
+ distro += dbcfgs['hbase_ver']
+
+ distro, v1, v2 = re.search(r'(\w+)-*(\d)\.(\d)', distro).groups()
if distro == 'CDH':
if v2 == '6': v2 = '5'
if v2 == '8': v2 = '7'
elif distro == 'HDP':
if v2 == '4': v2 = '3'
- hbase_trx_jar = 'hbase-trx-%s%s_%s-%s.jar' % (distro.lower(), v1, v2, TRAF_VER)
- traf_hbase_trx_path = '%s/%s' % (TRAF_LIB_PATH, hbase_trx_jar)
+ hbase_trx_jar = 'hbase-trx-%s%s_%s-%s.jar' % (distro.lower(), v1, v2, traf_ver)
+ traf_hbase_trx_path = '%s/%s' % (traf_lib_path, hbase_trx_jar)
hbase_trx_path = '%s/%s' % (hbase_lib_path, hbase_trx_jar)
if not os.path.exists(traf_hbase_trx_path):
err('Cannot find HBase trx jar \'%s\' for your Hadoop distribution' % hbase_trx_jar)
@@ -109,7 +107,7 @@ def run():
# copy new ones
run_cmd('cp %s %s' % (traf_hbase_trx_path, hbase_lib_path))
- run_cmd('cp %s/trafodion-utility-* %s' % (TRAF_LIB_PATH, hbase_lib_path))
+ run_cmd('cp %s/trafodion-utility-* %s' % (traf_lib_path, hbase_lib_path))
# set permission
run_cmd('chmod +r %s/{hbase-trx-*,trafodion-utility-*}' % hbase_lib_path)
@@ -123,10 +121,10 @@ Cmnd_Alias ARP = /sbin/arping
## Allow Trafodion id to run commands needed to configure floating IP
%%%s ALL = NOPASSWD: IP, ARP
-""" % TRAF_USER
+""" % traf_user
### write trafodion sudoer file ###
- with open(SUDOER_FILE, 'w') as f:
+ with open(TRAF_SUDOER_FILE, 'w') as f:
f.write(SUDOER_CFG)
# main
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/scripts/traf_sqconfig.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_sqconfig.py b/install/python-installer/scripts/traf_sqconfig.py
index 6c8c188..a07a8a5 100755
--- a/install/python-installer/scripts/traf_sqconfig.py
+++ b/install/python-installer/scripts/traf_sqconfig.py
@@ -35,9 +35,9 @@ def run():
scratch_locs = dbcfgs['scratch_locs'].split(',')
# this script is running by trafodion user, so get sqroot from env
- TRAF_HOME = os.environ['TRAF_HOME']
- if TRAF_HOME == '': err('TRAF_HOME var is empty')
- SQCONFIG_FILE = TRAF_HOME + '/sql/scripts/sqconfig'
+ traf_home = os.environ['TRAF_HOME']
+ if traf_home == '': err('TRAF_HOME var is empty')
+ sqconfig_file = traf_home + '/sql/scripts/sqconfig'
core, processor = run_cmd("lscpu|grep -E '(^CPU\(s\)|^Socket\(s\))'|awk '{print $2}'").split('\n')[:2]
core = int(core)-1 if int(core) <= 256 else 255
@@ -57,7 +57,7 @@ def run():
lines.append('end overflow\n')
- with open(SQCONFIG_FILE, 'w') as f:
+ with open(sqconfig_file, 'w') as f:
f.writelines(lines)
print 'sqconfig generated successfully!'
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/scripts/traf_start.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_start.py b/install/python-installer/scripts/traf_start.py
index 395f678..0c5f668 100755
--- a/install/python-installer/scripts/traf_start.py
+++ b/install/python-installer/scripts/traf_start.py
@@ -34,8 +34,8 @@ def run():
dbcfgs = json.loads(dbcfgs_json)
print 'Starting trafodion'
- TRAF_HOME = os.environ['TRAF_HOME']
- if os.path.exists('%s/sql/scripts/trafstart' % TRAF_HOME):
+ traf_home = os.environ['TRAF_HOME']
+ if os.path.exists('%s/sql/scripts/trafstart' % traf_home):
run_cmd('trafstart')
else:
run_cmd('sqstart')
@@ -67,7 +67,7 @@ def run():
err('Failed to setup security for trafodion:\n %s' % secure_output)
run_cmd('rm -rf %s' % tmp_file)
- if os.path.exists('%s/sql/scripts/connstart' % TRAF_HOME):
+ if os.path.exists('%s/sql/scripts/connstart' % traf_home):
run_cmd('connstart')
print 'Start trafodion successfully.'
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/scripts/traf_user.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_user.py b/install/python-installer/scripts/traf_user.py
index 3fa9237..09a346a 100755
--- a/install/python-installer/scripts/traf_user.py
+++ b/install/python-installer/scripts/traf_user.py
@@ -27,68 +27,65 @@ import os
import sys
import json
import socket
+from constants import TRAF_CFG_DIR, TRAF_CFG_FILE, TRAF_HSPERFDATA_FILE, SSHKEY_FILE
from common import ParseXML, run_cmd, append_file, mod_file, write_file, \
- cmd_output, run_cmd_as_user, err, TMP_DIR
+ cmd_output, run_cmd_as_user, err
def run():
""" create trafodion user, bashrc, setup passwordless SSH """
dbcfgs = json.loads(dbcfgs_json)
- DISTRO = dbcfgs['distro']
- if 'CDH' in DISTRO:
+ distro = dbcfgs['distro']
+ if 'CDH' in distro:
hadoop_type = 'cloudera'
- elif 'HDP' in DISTRO:
+ elif 'HDP' in distro:
hadoop_type = 'hortonworks'
- elif 'APACHE' in DISTRO:
+ elif 'APACHE' in distro:
hadoop_type = 'apache'
- TRAF_USER = dbcfgs['traf_user']
- TRAF_GROUP = TRAF_USER
- HOME_DIR = cmd_output('cat /etc/default/useradd |grep HOME |cut -d "=" -f 2').strip()
+ traf_user = dbcfgs['traf_user']
+ traf_group = traf_user
+ home_dir = cmd_output('cat /etc/default/useradd |grep HOME |cut -d "=" -f 2').strip()
# customize trafodion home dir
if dbcfgs.has_key('home_dir') and dbcfgs['home_dir']:
- HOME_DIR = dbcfgs['home_dir']
+ home_dir = dbcfgs['home_dir']
- TRAF_USER_DIR = '%s/%s' % (HOME_DIR, TRAF_USER)
- TRAF_DIRNAME = dbcfgs['traf_dirname']
- TRAF_HOME = '%s/%s' % (TRAF_USER_DIR, TRAF_DIRNAME)
+ traf_user_dir = '%s/%s' % (home_dir, traf_user)
+ traf_dirname = dbcfgs['traf_dirname']
+ traf_home = '%s/%s' % (traf_user_dir, traf_dirname)
- TRAFODION_CFG_DIR = '/etc/trafodion/'
- TRAFODION_CFG_FILE = '/etc/trafodion/trafodion_config'
- HBASE_XML_FILE = dbcfgs['hbase_xml_file']
- KEY_FILE = '/tmp/id_rsa'
- AUTH_KEY_FILE = '%s/.ssh/authorized_keys' % TRAF_USER_DIR
- SSH_CFG_FILE = '%s/.ssh/config' % TRAF_USER_DIR
- ULIMITS_FILE = '/etc/security/limits.d/%s.conf' % TRAF_USER
- HSPERFDATA_FILE = '/tmp/hsperfdata_trafodion'
+ hbase_xml_file = dbcfgs['hbase_xml_file']
+ auth_key_file = '%s/.ssh/authorized_keys' % traf_user_dir
+ ssh_cfg_file = '%s/.ssh/config' % traf_user_dir
+ ulimits_file = '/etc/security/limits.d/%s.conf' % traf_user
# create trafodion user and group
- if not cmd_output('getent group %s' % TRAF_GROUP):
- run_cmd('groupadd %s > /dev/null 2>&1' % TRAF_GROUP)
+ if not cmd_output('getent group %s' % traf_group):
+ run_cmd('groupadd %s > /dev/null 2>&1' % traf_group)
- if not cmd_output('getent passwd %s' % TRAF_USER):
- TRAF_PWD = dbcfgs['traf_pwd']
- run_cmd('useradd --shell /bin/bash -m %s -g %s --home %s --password "$(openssl passwd %s)"' % (TRAF_USER, TRAF_GROUP, TRAF_USER_DIR, TRAF_PWD))
- elif not os.path.exists(TRAF_USER_DIR):
- run_cmd('mkdir -p %s' % TRAF_USER_DIR)
- run_cmd('chmod 700 %s' % TRAF_USER_DIR)
+ if not cmd_output('getent passwd %s' % traf_user):
+ traf_pwd = dbcfgs['traf_pwd']
+ run_cmd('useradd --shell /bin/bash -m %s -g %s --home %s --password "$(openssl passwd %s)"' % (traf_user, traf_group, traf_user_dir, traf_pwd))
+ elif not os.path.exists(traf_user_dir):
+ run_cmd('mkdir -p %s' % traf_user_dir)
+ run_cmd('chmod 700 %s' % traf_user_dir)
# set ssh key
- run_cmd_as_user(TRAF_USER, 'echo -e "y" | ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa')
+ run_cmd_as_user(traf_user, 'echo -e "y" | ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa')
# the key is generated in copy_file script running on the installer node
- run_cmd('cp %s{,.pub} %s/.ssh/' % (KEY_FILE, TRAF_USER_DIR))
+ run_cmd('cp %s{,.pub} %s/.ssh/' % (SSHKEY_FILE, traf_user_dir))
- run_cmd_as_user(TRAF_USER, 'cat ~/.ssh/id_rsa.pub > %s' % AUTH_KEY_FILE)
- run_cmd('chmod 644 %s' % AUTH_KEY_FILE)
+ run_cmd_as_user(traf_user, 'cat ~/.ssh/id_rsa.pub > %s' % auth_key_file)
+ run_cmd('chmod 644 %s' % auth_key_file)
ssh_cfg = 'StrictHostKeyChecking=no\nNoHostAuthenticationForLocalhost=yes\n'
- with open(SSH_CFG_FILE, 'w') as f:
+ with open(ssh_cfg_file, 'w') as f:
f.write(ssh_cfg)
- run_cmd('chmod 600 %s' % SSH_CFG_FILE)
+ run_cmd('chmod 600 %s' % ssh_cfg_file)
- run_cmd('chown -R %s:%s %s/.ssh/' % (TRAF_USER, TRAF_GROUP, TRAF_USER_DIR))
+ run_cmd('chown -R %s:%s %s/.ssh/' % (traf_user, traf_group, traf_user_dir))
- hb = ParseXML(HBASE_XML_FILE)
+ hb = ParseXML(hbase_xml_file)
zk_nodes = hb.get_property('hbase.zookeeper.quorum')
zk_port = hb.get_property('hbase.zookeeper.property.clientPort')
# set trafodion_config
@@ -106,13 +103,13 @@ export ZOOKEEPER_NODES="%s"
export ZOOKEEPER_PORT="%s"
export SECURE_HADOOP="%s"
export CLUSTERNAME="%s"
-""" % (TRAF_HOME, dbcfgs['java_home'], ' '.join(nodes), ' -w ' + ' -w '.join(nodes),
+""" % (traf_home, dbcfgs['java_home'], ' '.join(nodes), ' -w ' + ' -w '.join(nodes),
str(len(nodes)), hadoop_type, dbcfgs['enable_ha'], zk_nodes, zk_port, dbcfgs['secure_hadoop'], socket.gethostname())
- run_cmd('mkdir -p %s' % TRAFODION_CFG_DIR)
- write_file(TRAFODION_CFG_FILE, trafodion_config)
+ run_cmd('mkdir -p %s' % TRAF_CFG_DIR)
+ write_file(TRAF_CFG_FILE, trafodion_config)
- if 'APACHE' in DISTRO:
+ if 'APACHE' in distro:
extra_config = """
export HADOOP_PREFIX=%s
export HBASE_HOME=%s
@@ -121,7 +118,7 @@ export PATH=$PATH:$HADOOP_PREFIX/bin:$HADOOP_PREFIX/sbin:$HBASE_HOME/bin
append_file(TRAFODION_CFG_FILE, extra_config)
# set permission
- run_cmd('chown -R %s:%s %s*' % (TRAF_USER, TRAF_GROUP, TRAFODION_CFG_DIR))
+ run_cmd('chown -R %s:%s %s*' % (traf_user, traf_group, TRAF_CFG_DIR))
# set ulimits for trafodion user
@@ -137,16 +134,16 @@ export PATH=$PATH:$HADOOP_PREFIX/bin:$HADOOP_PREFIX/sbin:$HBASE_HOME/bin
%s hard nproc 100000
%s soft nofile 8192
%s hard nofile 65535
-''' % ((TRAF_USER,) * 10)
+''' % ((traf_user,) * 10)
- write_file(ULIMITS_FILE, ulimits_config)
+ write_file(ulimits_file, ulimits_config)
# change permission for hsperfdata
- if os.path.exists(HSPERFDATA_FILE):
- run_cmd('chown -R %s:%s %s' % (TRAF_USER, TRAF_GROUP, HSPERFDATA_FILE))
+ if os.path.exists(TRAF_HSPERFDATA_FILE):
+ run_cmd('chown -R %s:%s %s' % (traf_user, traf_group, TRAF_HSPERFDATA_FILE))
# clean up unused key file at the last step
- run_cmd('rm -rf %s{,.pub}' % KEY_FILE)
+ run_cmd('rm -rf %s{,.pub}' % SSHKEY_FILE)
print 'Setup trafodion user successfully!'
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/dc58c85d/install/python-installer/scripts/wrapper.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/wrapper.py b/install/python-installer/scripts/wrapper.py
index cc34c3f..e72b32d 100644
--- a/install/python-installer/scripts/wrapper.py
+++ b/install/python-installer/scripts/wrapper.py
@@ -27,9 +27,8 @@ import json
import subprocess
from glob import glob
from threading import Thread
-from common import err_m, run_cmd, time_elapse, get_logger, Remote, \
- ParseJson, INSTALLER_LOC, TMP_DIR, SCRCFG_FILE, \
- CONFIG_DIR, SCRIPTS_DIR
+from constants import INSTALLER_LOC, TMP_DIR, SCRCFG_FILE, CONFIG_DIR, SCRIPTS_DIR
+from common import err_m, run_cmd, time_elapse, get_logger, Remote, ParseJson
class RemoteRun(Remote):
""" run commands or scripts remotely using ssh """
@@ -121,9 +120,9 @@ def run(dbcfgs, options, mode='install', pwd=''):
""" main entry
mode: install/discover
"""
- STAT_FILE = '%s/%s.status' % (INSTALLER_LOC, mode)
- LOG_FILE = '%s/logs/%s_%s.log' % (INSTALLER_LOC, mode, time.strftime('%Y%m%d_%H%M'))
- logger = get_logger(LOG_FILE)
+ stat_file = '%s/%s.status' % (INSTALLER_LOC, mode)
+ log_file = '%s/logs/%s_%s.log' % (INSTALLER_LOC, mode, time.strftime('%Y%m%d_%H%M'))
+ logger = get_logger(log_file)
verbose = True if hasattr(options, 'verbose') and options.verbose else False
reinstall = True if hasattr(options, 'reinstall') and options.reinstall else False
@@ -157,11 +156,11 @@ def run(dbcfgs, options, mode='install', pwd=''):
skipped_scripts += ['apache_mods', 'apache_restart']
# set ssh config file to avoid known hosts verify on current installer node
- SSH_CFG_FILE = os.environ['HOME'] + '/.ssh/config'
+ ssh_cfg_file = os.environ['HOME'] + '/.ssh/config'
ssh_cfg = 'StrictHostKeyChecking=no\nNoHostAuthenticationForLocalhost=yes\n'
- with open(SSH_CFG_FILE, 'w') as f:
+ with open(ssh_cfg_file, 'w') as f:
f.write(ssh_cfg)
- run_cmd('chmod 600 %s' % SSH_CFG_FILE)
+ run_cmd('chmod 600 %s' % ssh_cfg_file)
def run_local_script(script, json_string, req_pwd):
cmd = '%s/%s \'%s\'' % (SCRIPTS_DIR, script, json_string)
@@ -219,7 +218,7 @@ def run(dbcfgs, options, mode='install', pwd=''):
elif cfg['req_pwd'] == 'yes':
req_pwd = True
- status = Status(STAT_FILE, script)
+ status = Status(stat_file, script)
if status.get_status():
msg = 'Script [%s] had already been executed' % script
state_skip(msg)
@@ -253,7 +252,7 @@ def run(dbcfgs, options, mode='install', pwd=''):
for t in threads: t.join()
if sum([r.rc for r in parted_remote_inst]) != 0:
- err_m('Script failed to run on one or more nodes, exiting ...\nCheck log file %s for details.' % LOG_FILE)
+ err_m('Script failed to run on one or more nodes, exiting ...\nCheck log file %s for details.' % log_file)
script_output += [{r.host:r.stdout.strip()} for r in parted_remote_inst]
@@ -266,7 +265,7 @@ def run(dbcfgs, options, mode='install', pwd=''):
err_m('User quit')
# remove status file if all scripts run successfully
- os.remove(STAT_FILE)
+ os.remove(stat_file)
return script_output
[2/2] incubator-trafodion git commit: Merge Trafodion 2496 to 2.1
Posted by sa...@apache.org.
Merge Trafodion 2496 to 2.1
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/58caf9fd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/58caf9fd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/58caf9fd
Branch: refs/heads/release2.1
Commit: 58caf9fd361afd9d1bae61865032acbc0b8bd3b3
Parents: 6cc8957 dc58c85
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Thu Mar 2 00:24:38 2017 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Thu Mar 2 00:24:38 2017 +0000
----------------------------------------------------------------------
.../configs/db_config_default.ini | 7 +-
install/python-installer/db_install.py | 22 ++---
install/python-installer/db_uninstall.py | 14 +--
install/python-installer/discovery.py | 3 +-
install/python-installer/scripts/apache_mods.py | 3 +-
install/python-installer/scripts/common.py | 20 +----
install/python-installer/scripts/constants.py | 61 ++++++++++++++
install/python-installer/scripts/copy_files.py | 21 +----
install/python-installer/scripts/dcs_setup.py | 38 ++++-----
install/python-installer/scripts/hadoop_mods.py | 3 +-
install/python-installer/scripts/hdfs_cmds.py | 13 +--
.../python-installer/scripts/traf_discover.py | 26 +++---
.../python-installer/scripts/traf_kerberos.py | 2 +-
install/python-installer/scripts/traf_ldap.py | 26 +++---
.../python-installer/scripts/traf_package.py | 11 +--
install/python-installer/scripts/traf_setup.py | 64 +++++++-------
.../python-installer/scripts/traf_sqconfig.py | 8 +-
install/python-installer/scripts/traf_start.py | 6 +-
install/python-installer/scripts/traf_user.py | 89 ++++++++++----------
install/python-installer/scripts/wrapper.py | 23 +++--
20 files changed, 242 insertions(+), 218 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/58caf9fd/install/python-installer/scripts/traf_sqconfig.py
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/58caf9fd/install/python-installer/scripts/traf_start.py
----------------------------------------------------------------------