You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by li...@apache.org on 2016/12/20 08:54:43 UTC
[2/4] incubator-trafodion git commit: [TRAFODION-2393] python
installer - reorganize script directories for better user experience
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2886c023/install/python-installer/prompt.json
----------------------------------------------------------------------
diff --git a/install/python-installer/prompt.json b/install/python-installer/prompt.json
deleted file mode 100644
index 7ea0a90..0000000
--- a/install/python-installer/prompt.json
+++ /dev/null
@@ -1,227 +0,0 @@
-{
- "traf_start":
- {
- "prompt":"Start instance after installation",
- "default":"Y",
- "isYN":true
- },
- "dcs_ha":
- {
- "prompt":"Enable DCS High Avalability",
- "default":"N",
- "isYN":true
- },
- "dcs_floating_ip":
- {
- "prompt":"Enter Floating IP address for DCS HA",
- "isIP":true
- },
- "dcs_interface":
- {
- "prompt":"Enter interface for Floating IP address",
- "default":"eth0"
- },
- "dcs_backup_nodes":
- {
- "prompt":"Enter DCS Backup Master Nodes for DCS HA (comma separated)"
- },
- "kdc_server":
- {
- "prompt":"Enter KDC server address"
- },
- "admin_principal":
- {
- "prompt":"Enter admin principal (include realm)"
- },
- "kdcadmin_pwd":
- {
- "prompt":"Enter password for admin principal",
- "ispasswd":true
- },
- "hdfs_keytab":
- {
- "prompt":"Enter fully qualified name for HDFS keytab"
- },
- "hbase_keytab":
- {
- "prompt":"Enter fully qualified name for HBase keytab"
- },
- "max_lifetime":
- {
- "prompt":"Enter max lifetime for Trafodion principal (valid format required)",
- "default":"24hours"
- },
- "max_renew_lifetime":
- {
- "prompt":"Enter renew lifetime for Trafodion principal (valid format required)",
- "default":"7days"
- },
- "traf_keytab":
- {
- "prompt":"Enter Trafodion keytab name"
- },
- "ldap_security":
- {
- "prompt":"Enable LDAP security",
- "default":"N",
- "isYN":true
- },
- "ldap_hosts":
- {
- "prompt":"Enter list of LDAP Hostnames (comma separated)"
- },
- "ldap_port":
- {
- "prompt":"Enter LDAP Port number (Example: 389 for no encryption or TLS, 636 for SSL)",
- "default":"389",
- "isdigit":true
- },
- "ldap_identifiers":
- {
- "prompt":"Enter all LDAP unique identifiers (blank separated)"
- },
- "ldap_encrypt":
- {
- "prompt":"Enter LDAP Encryption Level (0: Encryption not used, 1: SSL, 2: TLS)",
- "default":"0",
- "isdigit":true
- },
- "ldap_certpath":
- {
- "prompt":"Enter full path to TLS certificate file (*.pem)",
- "isexist":true
- },
- "ldap_userinfo":
- {
- "prompt":"If Requred search user name/password",
- "default":"N",
- "isYN":true
- },
- "ldap_user":
- {
- "prompt":"Enter Search user name (if required)",
- "default":" "
- },
- "ldap_pwd":
- {
- "prompt":"Enter Search password (if required)",
- "default":" "
- },
- "scratch_locs":
- {
- "prompt":"Enter trafodion scratch file folder location(should be a large disk), if more than one folder, use comma seperated",
- "default":"$TRAF_HOME/tmp"
- },
- "local_repo_dir":
- {
- "prompt":"Enter local repository folder location to install trafodion RPM dependencies",
- "isexist":true
- },
- "java_home":
- {
- "prompt":"Specify location of Java(JDK) on trafodion nodes",
- "isremote_exist":true,
- "default":"/usr/lib/jvm/java-1.7.0-openjdk.x86_64"
- },
- "dcs_cnt_per_node":
- {
- "prompt":"Enter number of DCS client connections per node",
- "default":"4",
- "isdigit":true
- },
- "first_rsnode":
- {
- "prompt":"Enter the hostname of first Apache HBase RegionServer node"
- },
- "hdfs_user":
- {
- "prompt":"Enter hadoop user name",
- "default":"hdfs",
- "isuser":true
- },
- "hbase_user":
- {
- "prompt":"Enter hbase user name",
- "default":"hbase",
- "isuser":true
- },
- "hadoop_home":
- {
- "prompt":"Enter Apache Hadoop directory location",
- "isremote_exist":true
- },
- "hbase_home":
- {
- "prompt":"Enter Apache HBase directory location",
- "isremote_exist":true
- },
- "hive_home":
- {
- "prompt":"Enter Apache Hive directory location if exists",
- "default":"NO_HIVE"
- },
- "mgr_url":
- {
- "prompt":"Enter HDP/CDH web manager URL:port, (full URL, if no http/https prefix, default prefix is http://)"
- },
- "mgr_user":
- {
- "prompt":"Enter HDP/CDH web manager user name",
- "default":"admin",
- "isuser":true
- },
- "mgr_pwd":
- {
- "prompt":"Enter HDP/CDH web manager user password",
- "ispasswd":true
- },
- "traf_user":
- {
- "prompt":"Enter trafodion user name",
- "default":"trafodion",
- "isuser":true
- },
- "traf_pwd":
- {
- "prompt":"Enter trafodion user password",
- "ispasswd":true
- },
- "traf_package":
- {
- "prompt":"Enter full path to Trafodion tar file",
- "isexist":true
- },
- "db_root_user":
- {
- "prompt":"Enter LDAP user name to be assigned DB root privileges (DB__ROOT)",
- "default":"trafodion",
- "isuser":true
- },
- "db_admin_user":
- {
- "prompt":"Enter LDAP user name to be assigned DB Admin privileges",
- "default":"admin",
- "isuser":true
- },
- "db_admin_pwd":
- {
- "prompt":"Enter LDAP user password to be assigned DB Admin privileges",
- "default":"traf123"
- },
- "node_list":
- {
- "prompt":"Enter list of Nodes separated by comma, support simple numeric RE,\n i.e. \"n[01-12],n[21-25]\",\"n0[1-5].com\""
- },
- "cluster_no":
- {
- "prompt":"Select the above cluster number for installing Trafodion",
- "default":"1",
- "isdigit":true
- },
- "use_data_node":
- {
- "prompt":"Install Trafodion nodes on all DataNodes",
- "default":"Y",
- "isYN":true
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2886c023/install/python-installer/script.json
----------------------------------------------------------------------
diff --git a/install/python-installer/script.json b/install/python-installer/script.json
deleted file mode 100644
index f6191d5..0000000
--- a/install/python-installer/script.json
+++ /dev/null
@@ -1,87 +0,0 @@
-{
-"install": [
- {
- "script": "traf_check.py",
- "desc": "Environment Check",
- "node": "all"
- },
- {
- "script": "copy_files.py",
- "desc": "Copy Trafodion package file",
- "node": "local",
- "req_pwd": "yes"
- },
- {
- "script": "traf_user.py",
- "desc": "Trafodion user Setup",
- "node": "all"
- },
- {
- "script": "traf_dep.py",
- "desc": "Install Trafodion dependencies",
- "node": "all"
- },
- {
- "script": "traf_package.py",
- "desc": "Install Trafodion package",
- "node": "all",
- "run_as_traf": "yes"
- },
- {
- "script": "traf_setup.py",
- "desc": "Environment Setup",
- "node": "all"
- },
- {
- "script": "traf_kerberos.py",
- "desc": "Kerberos Setup",
- "node": "all"
- },
- {
- "script": "dcs_setup.py",
- "desc": "DCS/REST Setup",
- "node": "all",
- "run_as_traf": "yes"
- },
- {
- "script": "traf_ldap.py",
- "desc": "LDAP Security Setup",
- "node": "all",
- "run_as_traf": "yes"
- },
- {
- "script": "hadoop_mods.py",
- "desc": "Hadoop modification and restart",
- "node": "local"
- },
- {
- "script": "apache_mods.py",
- "desc": "Hadoop modification and restart",
- "node": "all"
- },
- {
- "script": "hdfs_cmds.py",
- "desc": "Set permission of HDFS folder for Trafodion user",
- "node": "first_rs"
- },
- {
- "script": "traf_sqconfig.py",
- "desc": "Sqconfig Setup",
- "node": "first",
- "run_as_traf": "yes"
- },
- {
- "script": "traf_start.py",
- "desc": "Start Trafodion",
- "node": "first",
- "run_as_traf": "yes"
- }
-],
-"discover": [
- {
- "script": "traf_discover.py",
- "desc": "Environment Discover",
- "node": "all"
- }
-]
-}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2886c023/install/python-installer/scripts/__init__.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/__init__.py b/install/python-installer/scripts/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2886c023/install/python-installer/scripts/apache_mods.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/apache_mods.py b/install/python-installer/scripts/apache_mods.py
new file mode 100755
index 0000000..17a54d5
--- /dev/null
+++ b/install/python-installer/scripts/apache_mods.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on all nodes with sudo user ###
+
+import sys
+import json
+import socket
+from common import MODCFG_FILE, ParseJson, ParseXML, err, run_cmd
+
+def run():
+ dbcfgs = json.loads(dbcfgs_json)
+ if 'APACHE' in dbcfgs['distro']:
+ modcfgs = ParseJson(MODCFG_FILE).load()
+ MOD_CFGS = modcfgs['MOD_CFGS']
+
+ hdfs_xml_file = dbcfgs['hdfs_xml_file']
+ hbase_xml_file = dbcfgs['hbase_xml_file']
+
+ hbasexml = ParseXML(hbase_xml_file)
+ for key, value in MOD_CFGS['hbase-site'].items():
+ hbasexml.add_property(key, value)
+ hbasexml.write_xml()
+
+ hdfsxml = ParseXML(hdfs_xml_file)
+ for key, value in MOD_CFGS['hdfs-site'].items():
+ hdfsxml.add_property(key, value)
+ hdfsxml.write_xml()
+
+ print 'Apache Hadoop modification completed'
+ first_node = dbcfgs['first_rsnode']
+ local_host = socket.gethostname()
+ if first_node in local_host:
+ hadoop_home = dbcfgs['hadoop_home']
+ hbase_home = dbcfgs['hbase_home']
+ # stop
+ run_cmd(hbase_home + '/bin/stop-hbase.sh')
+ run_cmd(hadoop_home + '/sbin/stop-dfs.sh')
+ # start
+ run_cmd(hadoop_home + '/sbin/start-dfs.sh')
+ run_cmd(hbase_home + '/bin/start-hbase.sh')
+
+ print 'Apache Hadoop restart completed'
+ else:
+ print 'no apache distribution found, skipping'
+
+# main
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+run()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2886c023/install/python-installer/scripts/common.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/common.py b/install/python-installer/scripts/common.py
new file mode 100644
index 0000000..b574c71
--- /dev/null
+++ b/install/python-installer/scripts/common.py
@@ -0,0 +1,532 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### The common functions ###
+
+import os
+import pty
+import sys
+import json
+import re
+import time
+import base64
+import subprocess
+import logging
+try:
+ import xml.etree.cElementTree as ET
+except ImportError:
+ import xml.etree.ElementTree as ET
+from ConfigParser import ConfigParser
+from collections import defaultdict
+
+INSTALLER_LOC = re.search('(.*)/\w+',os.path.dirname(os.path.abspath(__file__))).groups()[0]
+
+CONFIG_DIR = INSTALLER_LOC + '/configs'
+SCRIPTS_DIR = INSTALLER_LOC + '/scripts'
+TEMPLATES_DIR = INSTALLER_LOC + '/templates'
+
+USER_PROMPT_FILE = CONFIG_DIR + '/prompt.json'
+SCRCFG_FILE = CONFIG_DIR + '/script.json'
+VERSION_FILE = CONFIG_DIR + '/version.json'
+MODCFG_FILE = CONFIG_DIR + '/mod_cfgs.json'
+DEF_PORT_FILE = CONFIG_DIR + '/default_ports.ini'
+
+DBCFG_FILE = INSTALLER_LOC + '/db_config'
+DBCFG_TMP_FILE = INSTALLER_LOC + '/.db_config_temp'
+
+TMP_DIR = '/tmp/.trafodion_install_temp'
+MARK = '[ERR]'
+
+def version():
+ print 'Installer version: %s' % __VERSION__
+ exit(0)
+
+def ok(msg):
+ print '\n\33[32m***[OK]: %s \33[0m' % msg
+
+def info(msg):
+ print '\n\33[33m***[INFO]: %s \33[0m' % msg
+
+def err_m(msg):
+ """ used by main script """
+ sys.stderr.write('\n\33[31m***[ERROR]: %s \33[0m\n' % msg)
+ sys.exit(1)
+
+def err(msg):
+ """ used by sub script """
+ sys.stderr.write(MARK + msg)
+ sys.exit(1)
+
+def get_logger(log_file):
+
+ log_dir = os.path.dirname(log_file)
+ if not os.path.exists(log_dir): os.mkdir(log_dir)
+
+ logger = logging.getLogger()
+ logger.setLevel(logging.INFO)
+
+ formatter = logging.Formatter('[%(asctime)s %(levelname)s]: %(message)s')
+
+ fh = logging.FileHandler(log_file)
+ fh.setFormatter(formatter)
+
+ logger.addHandler(fh)
+
+ return logger
+
+def run_cmd(cmd):
+ """ check command return value and return stdout """
+ p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ stdout, stderr = p.communicate()
+ if p.returncode != 0:
+ err('Failed to run command %s: %s' % (cmd, stderr))
+ return stdout.strip()
+
+def run_cmd_as_user(user, cmd):
+ return run_cmd('sudo -n su - %s -c \'%s\'' % (user, cmd))
+
+def cmd_output(cmd):
+ """ return command output but not check return value """
+ p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ stdout, stderr = p.communicate()
+
+ return stdout.strip() if stdout else stderr
+
+def mod_file(template_file, change_items):
+ """
+ @params: change_items: a dict includes:
+ {regular_expression : replace_string}
+ """
+ try:
+ with open(template_file, 'r') as f:
+ lines = f.read()
+ except IOError:
+ err('Failed to open file %s to modify' % template_file)
+
+ for regexp, replace in change_items.iteritems():
+ lines = re.sub(regexp, replace, lines)
+
+ with open(template_file, 'w') as f:
+ f.write(lines)
+
+def append_file(template_file, string, position=''):
+ try:
+ with open(template_file, 'r') as f:
+ lines = f.readlines()
+ pos = 0
+ if position:
+ for index, line in enumerate(lines):
+ if position in line:
+ pos = index + 1
+
+ if pos == 0: pos = len(lines)
+ string_lf = string + '\n'
+ newlines = lines[:pos] + [string_lf] + lines[pos:]
+ if not string_lf in lines:
+ with open(template_file, 'w') as f:
+ f.writelines(newlines)
+ except IOError:
+ err('Failed to open file %s to append' % template_file)
+
+
+def write_file(template_file, string):
+ try:
+ with open(template_file, 'w') as f:
+ f.write(string)
+ except IOError:
+ err('Failed to open file %s to write' % template_file)
+
+
+class Version(object):
+ def __init__(self):
+ self.support_ver = ParseJson(VERSION_FILE).load()
+
+ def get_version(self, component):
+ if self.support_ver[component] == '':
+ err('Failed to get version info for "%s" from config file' % component)
+
+ return self.support_ver[component]
+
+class Remote(object):
+ """
+ copy files to/fetch files from remote host using ssh
+ can also use paramiko, but it's not a build-in module
+ """
+
+ def __init__(self, host, user='', pwd=''):
+ self.host = host
+ self.user = user
+ self.rc = 0
+ self.pwd = pwd
+ self.sshpass = self._sshpass_available()
+ self._connection_test()
+
+ @staticmethod
+ def _sshpass_available():
+ sshpass_available = True
+ try:
+ p = subprocess.Popen(['sshpass'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ p.communicate()
+ except OSError:
+ sshpass_available = False
+
+ return sshpass_available
+
+ def _commands(self, method):
+ cmd = []
+ if self.sshpass and self.pwd: cmd = ['sshpass', '-p', self.pwd]
+ cmd += [method]
+ if not (self.sshpass and self.pwd): cmd += ['-oPasswordAuthentication=no']
+ return cmd
+
+ def _execute(self, cmd, verbose=False, shell=False):
+ try:
+ if verbose: print 'cmd:', cmd
+
+ master, slave = pty.openpty()
+ if shell:
+ p = subprocess.Popen(cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ else:
+ p = subprocess.Popen(cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+ self.stdout, self.stderr = p.communicate()
+ if p.returncode:
+ self.rc = p.returncode
+ # 'ssh -tt' will overwrite stderr, so manually handle it
+ if MARK in self.stdout:
+ self.stdout, self.stderr = self.stdout.split(MARK)
+ else:
+ self.stderr = self.stdout
+ except Exception as e:
+ err_m('Failed to run commands on remote host: %s' % e)
+
+ def _connection_test(self):
+ self.execute('echo -n', chkerr=False)
+ if self.rc != 0:
+ msg = 'Host [%s]: Failed to connect using ssh. Be sure:\n' % self.host
+ msg += '1. Remote host\'s name and IP is configured correctly in /etc/hosts.\n'
+ msg += '2. Remote host\'s sshd service is running.\n'
+ msg += '3. Passwordless SSH is set if not using \'enable-pwd\' option.\n'
+ msg += '4. \'sshpass\' tool is installed and ssh password is correct if using \'enable-pwd\' option.\n'
+ err_m(msg)
+
+ def execute(self, user_cmd, verbose=False, shell=False, chkerr=True):
+ """ @params: user_cmd should be a string """
+ cmd = self._commands('ssh')
+ cmd += ['-tt'] # force tty allocation
+ if self.user:
+ cmd += ['%s@%s' % (self.user, self.host)]
+ else:
+ cmd += [self.host]
+
+ # if shell=True, cmd should be a string not list
+ if shell:
+ cmd = ' '.join(cmd) + ' '
+ cmd += user_cmd
+ else:
+ cmd += user_cmd.split()
+
+ self._execute(cmd, verbose=verbose, shell=shell)
+
+ if chkerr and self.rc != 0:
+ err_m('Failed to execute command on remote host [%s]: "%s"' % (self.host, user_cmd))
+
+ def copy(self, files, remote_folder='.'):
+ """ copy file to user's home folder """
+ for f in files:
+ if not os.path.exists(f):
+ err_m('Copy file error: %s doesn\'t exist' % f)
+
+ cmd = self._commands('scp')
+ cmd += ['-r']
+ cmd += files # files should be full path
+ if self.user:
+ cmd += ['%s@%s:%s/' % (self.user, self.host, remote_folder)]
+ else:
+ cmd += ['%s:%s/' % (self.host, remote_folder)]
+
+ self._execute(cmd)
+ if self.rc != 0: err_m('Failed to copy files to remote nodes')
+
+ def fetch(self, files, local_folder='.'):
+ """ fetch file from user's home folder """
+ cmd = self._commands('scp')
+ cmd += ['-r']
+ if self.user:
+ cmd += ['%s@%s:~/{%s}' % (self.user, self.host, ','.join(files))]
+ else:
+ cmd += ['%s:~/{%s}' % (self.host, ','.join(files))]
+ cmd += [local_folder]
+
+ self._execute(cmd)
+ if self.rc != 0: err('Failed to fetch files from remote nodes')
+
+
+class ParseHttp(object):
+ def __init__(self, user, passwd, json_type=True):
+ # httplib2 is not installed by default
+ try:
+ import httplib2
+ except ImportError:
+ err_m('Python module httplib2 is not found. Install python-httplib2 first.')
+
+ self.user = user
+ self.passwd = passwd
+ self.h = httplib2.Http(disable_ssl_certificate_validation=True)
+ self.h.add_credentials(self.user, self.passwd)
+ self.headers = {}
+ self.headers['X-Requested-By'] = 'trafodion'
+ if json_type:
+ self.headers['Content-Type'] = 'application/json'
+ self.headers['Authorization'] = 'Basic %s' % (base64.b64encode('%s:%s' % (self.user, self.passwd)))
+
+ def _request(self, url, method, body=None):
+ try:
+ resp, content = self.h.request(url, method, headers=self.headers, body=body)
+ # return code is not 2xx and 409(for ambari blueprint)
+ if not (resp.status == 409 or 200 <= resp.status < 300):
+ err_m('Error return code {0} when {1}ting configs: {2}'.format(resp.status, method.lower(), content))
+ return content
+ except Exception as exc:
+ err_m('Error with {0}ting configs using URL {1}. Reason: {2}'.format(method.lower(), url, exc))
+
+ def get(self, url):
+ try:
+ return defaultdict(str, json.loads(self._request(url, 'GET')))
+ except ValueError:
+ err_m('Failed to get data from URL, check password if URL requires authentication')
+
+ def put(self, url, config):
+ if not isinstance(config, dict): err_m('Wrong HTTP PUT parameter, should be a dict')
+ result = self._request(url, 'PUT', body=json.dumps(config))
+ if result: return defaultdict(str, json.loads(result))
+
+ def post(self, url, config=None):
+ try:
+ if config:
+ if not isinstance(config, dict): err_m('Wrong HTTP POST parameter, should be a dict')
+ body = json.dumps(config)
+ else:
+ body = None
+
+ result = self._request(url, 'POST', body=body)
+ if result: return defaultdict(str, json.loads(result))
+
+ except ValueError as ve:
+ err_m('Failed to send command to URL: %s' % ve)
+
+
+class ParseXML(object):
+ """ handle *-site.xml with format
+ <property><name></name><value></value></proerty>
+ """
+ def __init__(self, xml_file):
+ self.__xml_file = xml_file
+ if not os.path.exists(self.__xml_file): err_m('Cannot find xml file %s' % self.__xml_file)
+ try:
+ self._tree = ET.parse(self.__xml_file)
+ except Exception as e:
+ err_m('failed to parsing xml: %s' % e)
+
+ self._root = self._tree.getroot()
+ self._properties = self._root.findall('property')
+ # name, value list
+ self._nvlist = [[elem.text for elem in p] for p in self._properties]
+
+ def __indent(self, elem):
+ """Return a pretty-printed XML string for the Element."""
+ if len(elem):
+ if not elem.text: elem.text = '\n' + ' '
+ if not elem.tail: elem.tail = '\n'
+ for subelem in elem:
+ self.__indent(subelem)
+ else:
+ if not elem.tail: elem.tail = '\n' + ' '
+
+ def get_property(self, name):
+ try:
+ return [x[1] for x in self._nvlist if x[0] == name][0]
+ except:
+ return ''
+
+ def rm_property(self, name):
+ for p in self._properties:
+ if p[0].text == name:
+ self._root.remove(p)
+
+ def add_property(self, name, value):
+ # don't add property if already exists
+ if self.get_property(name): return
+
+ elem_p = ET.Element('property')
+ elem_name = ET.Element('name')
+ elem_value = ET.Element('value')
+
+ elem_name.text = name
+ elem_value.text = value
+ elem_p.append(elem_name)
+ elem_p.append(elem_value)
+
+ self._nvlist.append([name, value])
+ self._root.append(elem_p)
+
+ def write_xml(self):
+ self.__indent(self._root)
+ self._tree.write(self.__xml_file)
+
+ def print_xml(self):
+ for name, value in self._nvlist:
+ print name, value
+
+class ParseJson(object):
+ def __init__(self, js_file):
+ self.__js_file = js_file
+
+ def load(self):
+ """ load json file to a dict """
+ if not os.path.exists(self.__js_file): err_m('Cannot find json file %s' % self.__js_file)
+ with open(self.__js_file, 'r') as f:
+ tmparray = f.readlines()
+ content = ''
+ for t in tmparray:
+ content += t
+
+ try:
+ return defaultdict(str, json.loads(content))
+ except ValueError:
+ err_m('No json format found in config file %s' % self.__js_file)
+
+ def save(self, dic):
+ """ save dict to json file with pretty format """
+ with open(self.__js_file, 'w') as f:
+ f.write(json.dumps(dic, indent=4))
+ return 0
+
+
+class ParseInI(object):
+ def __init__(self, ini_file, section):
+ self.__ini_file = ini_file
+ self.section = section
+
+ def load(self):
+ """ load content from ini file and return a dict """
+ if not os.path.exists(self.__ini_file):
+ err_m('Cannot find ini file %s' % self.__ini_file)
+
+ cfgs = {}
+ cf = ConfigParser()
+ cf.read(self.__ini_file)
+
+ if not cf.has_section(self.section):
+ return {}
+
+ for cfg in cf.items(self.section):
+ cfgs[cfg[0]] = cfg[1]
+
+ return defaultdict(str, cfgs)
+
+ def save(self, dic):
+ """ save a dict as an ini file """
+ cf = ConfigParser()
+ cf.add_section(self.section)
+ for key, value in dic.iteritems():
+ cf.set(self.section, key, value)
+
+ with open(self.__ini_file, 'w') as f:
+ cf.write(f)
+
+def http_start(repo_dir, repo_port):
+ info('Starting temporary python http server')
+ os.system("cd %s; python -m SimpleHTTPServer %s > /dev/null 2>&1 &" % (repo_dir, repo_port))
+
+def http_stop():
+ #info('Stopping temporary python http server')
+ os.system("ps -ef|grep SimpleHTTPServer |grep -v grep | awk '{print $2}' |xargs kill -9 >/dev/null 2>&1")
+
+def format_output(text):
+ num = len(text) + 4
+ print '*' * num
+ print ' ' + text
+ print '*' * num
+
+def expNumRe(text):
+ """
+ expand numeric regular expression to list
+ e.g. 'n[01-03],n1[0-1]': ['n01','n02','n03','n10','n11']
+ e.g. 'n[09-11].com': ['n09.com','n10.com','n11.com']
+ """
+ explist = []
+ for regex in text.split(','):
+ regex = regex.strip()
+ r = re.match(r'(.*)\[(\d+)-(\d+)\](.*)', regex)
+ if r:
+ h = r.group(1)
+ d1 = r.group(2)
+ d2 = r.group(3)
+ t = r.group(4)
+
+ convert = lambda d: str(('%0' + str(min(len(d1), len(d2))) + 'd') % d)
+ if d1 > d2: d1, d2 = d2, d1
+ explist.extend([h + convert(c) + t for c in range(int(d1), int(d2)+1)])
+
+ else:
+ # keep original value if not matched
+ explist.append(regex)
+
+ return explist
+
+def time_elapse(func):
+ """ time elapse decorator """
+ def wrapper(*args, **kwargs):
+ start_time = time.time()
+ output = func(*args, **kwargs)
+ end_time = time.time()
+ seconds = end_time - start_time
+ hours = seconds / 3600
+ seconds = seconds % 3600
+ minutes = seconds / 60
+ seconds = seconds % 60
+ print '\nTime Cost: %d hour(s) %d minute(s) %d second(s)' % (hours, minutes, seconds)
+ return output
+ return wrapper
+
+def retry(func, maxcnt, interval, msg):
+ """ retry timeout function """
+ retry_cnt = 0
+ rc = False
+ try:
+ while not rc:
+ retry_cnt += 1
+ rc = func()
+ flush_str = '.' * retry_cnt
+ print '\rCheck %s status (timeout: %d secs) %s' % (msg, maxcnt * interval, flush_str),
+ sys.stdout.flush()
+ time.sleep(interval)
+ if retry_cnt == maxcnt:
+ err_m('Timeout exit')
+ except KeyboardInterrupt:
+ err_m('user quit')
+ ok('%s successfully!' % msg)
+
+if __name__ == '__main__':
+ exit(0)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2886c023/install/python-installer/scripts/copy_files.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/copy_files.py b/install/python-installer/scripts/copy_files.py
new file mode 100755
index 0000000..baca6c3
--- /dev/null
+++ b/install/python-installer/scripts/copy_files.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on local node ###
+
+import sys
+import json
+from threading import Thread
+from common import ParseJson, Remote, run_cmd, err
+
+def run(pwd):
+ """ gen ssh key on local and copy to all nodes
+ copy traf package file from local to all nodes
+ """
+ dbcfgs = json.loads(dbcfgs_json)
+ hosts = dbcfgs['node_list'].split(',')
+ traf_package = dbcfgs['traf_package']
+
+ # save db configs to a tmp file and copy to all trafodion nodes
+ dbcfgs_file = '/tmp/dbcfgs'
+ p = ParseJson(dbcfgs_file)
+ # remove password from config file
+ try:
+ dbcfgs.pop('mgr_pwd')
+ dbcfgs.pop('traf_pwd')
+ dbcfgs.pop('kdcadmin_pwd')
+ except KeyError:
+ pass
+ p.save(dbcfgs)
+
+ key_file = '/tmp/id_rsa'
+ run_cmd('sudo -n rm -rf %s*' % key_file)
+ run_cmd('sudo -n echo -e "y" | ssh-keygen -t rsa -N "" -f %s' % key_file)
+
+ files = [key_file, key_file+'.pub', traf_package, dbcfgs_file]
+
+ remote_insts = [Remote(h, pwd=pwd) for h in hosts]
+ threads = [Thread(target=r.copy, args=(files, '/tmp')) for r in remote_insts]
+ for thread in threads: thread.start()
+ for thread in threads: thread.join()
+ for r in remote_insts:
+ if r.rc != 0: err('Failed to copy files to %s' % r.host)
+
+
+# main
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+
+try:
+ pwd = sys.argv[2]
+except IndexError:
+ pwd = ''
+
+run(pwd)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2886c023/install/python-installer/scripts/dcs_setup.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/dcs_setup.py b/install/python-installer/scripts/dcs_setup.py
new file mode 100755
index 0000000..d971af5
--- /dev/null
+++ b/install/python-installer/scripts/dcs_setup.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on all nodes with trafodion user ###
+
+import os
+import sys
+import json
+from common import append_file, write_file, mod_file, cmd_output, run_cmd, \
+ ParseInI, ParseXML, DEF_PORT_FILE, err
+
+def run():
+ dbcfgs = json.loads(dbcfgs_json)
+
+ TRAF_HOME = os.environ['TRAF_HOME']
+ TRAF_VER = dbcfgs['traf_version']
+ HBASE_XML_FILE = dbcfgs['hbase_xml_file']
+
+ DCS_INSTALL_ENV = 'export DCS_INSTALL_DIR=%s/dcs-%s' % (TRAF_HOME, TRAF_VER)
+ REST_INSTALL_ENV = 'export REST_INSTALL_DIR=%s/rest-%s' % (TRAF_HOME, TRAF_VER)
+
+ DCS_CONF_DIR = '%s/dcs-%s/conf' % (TRAF_HOME, TRAF_VER)
+ DCS_SRV_FILE = DCS_CONF_DIR + '/servers'
+ DCS_MASTER_FILE = DCS_CONF_DIR + '/master'
+ DCS_BKMASTER_FILE = DCS_CONF_DIR + '/backup-masters'
+ DCS_ENV_FILE = DCS_CONF_DIR + '/dcs-env.sh'
+ DCS_SITE_FILE = DCS_CONF_DIR + '/dcs-site.xml'
+ REST_SITE_FILE = '%s/rest-%s/conf/rest-site.xml' % (TRAF_HOME, TRAF_VER)
+ TRAFCI_FILE = TRAF_HOME + '/trafci/bin/trafci'
+ SQENV_FILE = TRAF_HOME + '/sqenvcom.sh'
+
+ ### dcs setting ###
+ # servers
+ nodes = dbcfgs['node_list'].split(',')
+ dcs_cnt = dbcfgs['dcs_cnt_per_node']
+ dcs_servers = ''
+ for node in nodes:
+ dcs_servers += '%s %s\n' % (node, dcs_cnt)
+
+ write_file(DCS_SRV_FILE, dcs_servers)
+
+ ### modify dcs config files ###
+ # modify master
+ dcs_master = nodes[0]
+ append_file(DCS_MASTER_FILE, dcs_master)
+
+ # modify sqenvcom.sh
+ append_file(SQENV_FILE, DCS_INSTALL_ENV)
+ append_file(SQENV_FILE, REST_INSTALL_ENV)
+
+ # modify dcs-env.sh
+ mod_file(DCS_ENV_FILE, {'.*DCS_MANAGES_ZK=.*':'export DCS_MANAGES_ZK=false'})
+
+ ports = ParseInI(DEF_PORT_FILE, 'ports').load()
+ dcs_master_port = ports['dcs_master_port']
+ # modify trafci
+ mod_file(TRAFCI_FILE, {'HNAME=.*':'HNAME=%s:%s' % (dcs_master, dcs_master_port)})
+
+ # modify dcs-site.xml
+ net_interface = run_cmd('ip route |grep default|awk \'{print $5}\'')
+ hb = ParseXML(HBASE_XML_FILE)
+ zk_hosts = hb.get_property('hbase.zookeeper.quorum')
+ zk_port = hb.get_property('hbase.zookeeper.property.clientPort')
+
+ p = ParseXML(DCS_SITE_FILE)
+ p.add_property('dcs.zookeeper.property.clientPort', zk_port)
+ p.add_property('dcs.zookeeper.quorum', zk_hosts)
+ p.add_property('dcs.dns.interface', net_interface)
+
+ if dbcfgs['dcs_ha'] == 'Y':
+ dcs_floating_ip = dbcfgs['dcs_floating_ip']
+ dcs_backup_nodes = dbcfgs['dcs_backup_nodes']
+ p.add_property('dcs.master.floating.ip', 'true')
+ p.add_property('dcs.master.floating.ip.external.interface', net_interface)
+ p.add_property('dcs.master.floating.ip.external.ip.address', dcs_floating_ip)
+ p.rm_property('dcs.dns.interface')
+
+ # modify backup_master
+ write_file(DCS_BKMASTER_FILE, dcs_backup_nodes)
+
+ p.write_xml()
+
+ ### rest setting ###
+ p = ParseXML(REST_SITE_FILE)
+ p.add_property('rest.zookeeper.property.clientPort', zk_port)
+ p.add_property('rest.zookeeper.quorum', zk_hosts)
+ p.write_xml()
+
+ ### run sqcertgen ###
+ run_cmd('sqcertgen')
+
+# main
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+run()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2886c023/install/python-installer/scripts/hadoop_mods.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/hadoop_mods.py b/install/python-installer/scripts/hadoop_mods.py
new file mode 100755
index 0000000..c065d15
--- /dev/null
+++ b/install/python-installer/scripts/hadoop_mods.py
@@ -0,0 +1,187 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on local node ###
+
+import time
+import sys
+import json
+from common import ParseHttp, ParseJson, MODCFG_FILE, err, info, retry
+
+modcfgs = ParseJson(MODCFG_FILE).load()
+
+MOD_CFGS = modcfgs['MOD_CFGS']
+HBASE_MASTER_CONFIG = modcfgs['HBASE_MASTER_CONFIG']
+HBASE_RS_CONFIG = modcfgs['HBASE_RS_CONFIG']
+HDFS_CONFIG = modcfgs['HDFS_CONFIG']
+ZK_CONFIG = modcfgs['ZK_CONFIG']
+
+CLUSTER_URL_PTR = '%s/api/v1/clusters/%s'
+RESTART_URL_PTR = CLUSTER_URL_PTR + '/commands/restart'
+RESTART_SRV_URL_PTR = CLUSTER_URL_PTR + '/services/%s/commands/restart'
+SRVCFG_URL_PTR = CLUSTER_URL_PTR + '/services/%s/config'
+RSGRP_BASEURL_PTR = '%s/api/v6/clusters/%s/services/%s/roleConfigGroups'
+DEPLOY_CFG_URL_PTR = '%s/api/v6/clusters/%s/commands/deployClientConfig'
+CMD_STAT_URL_PTR = '%s/api/v1/commands/%s'
+
+class CDHMod(object):
+ """ Modify CDH configs for trafodion and restart CDH services """
+ def __init__(self, user, passwd, url, cluster_name):
+ self.url = url
+ self.cluster_name = cluster_name
+ self.p = ParseHttp(user, passwd)
+
+ def mod(self):
+ hdfs_service = dbcfgs['hdfs_service_name']
+ hbase_service = dbcfgs['hbase_service_name']
+ zk_service = dbcfgs['zookeeper_service_name']
+ services = {hdfs_service:HDFS_CONFIG, hbase_service:HBASE_MASTER_CONFIG, zk_service:ZK_CONFIG}
+
+ for srv, cfg in services.iteritems():
+ srvcfg_url = SRVCFG_URL_PTR % (self.url, self.cluster_name, srv)
+ self.p.put(srvcfg_url, cfg)
+
+ # set configs in each regionserver group
+ rsgrp_baseurl = RSGRP_BASEURL_PTR % (self.url, self.cluster_name, hbase_service)
+ rscfg = self.p.get(rsgrp_baseurl)
+ rsgrp_urls = ['%s/%s/config' % (rsgrp_baseurl, r['name']) for r in rscfg['items'] if r['roleType'] == 'REGIONSERVER']
+
+ for rsgrp_url in rsgrp_urls:
+ self.p.put(rsgrp_url, HBASE_RS_CONFIG)
+
+ def restart(self):
+ restart_url = RESTART_URL_PTR % (self.url, self.cluster_name)
+ deploy_cfg_url = DEPLOY_CFG_URL_PTR % (self.url, self.cluster_name)
+
+ def __retry(url, maxcnt, interval, msg):
+ rc = self.p.post(url)
+ stat_url = CMD_STAT_URL_PTR % (self.url, rc['id'])
+ get_stat = lambda: self.p.get(stat_url)['success'] is True and self.p.get(stat_url)['active'] is False
+ retry(get_stat, maxcnt, interval, msg)
+
+ info('Restarting CDH services ...')
+ __retry(restart_url, 40, 15, 'CDH services restart')
+
+ info('Deploying CDH client configs ...')
+ __retry(deploy_cfg_url, 30, 10, 'CDH services deploy')
+
+
+class HDPMod(object):
+ """ Modify HDP configs for trafodion and restart HDP services """
+ def __init__(self, user, passwd, url, cluster_name):
+ self.url = url
+ self.cluster_name = cluster_name
+ self.p = ParseHttp(user, passwd, json_type=False)
+
+ def mod(self):
+ cluster_url = CLUSTER_URL_PTR % (self.url, self.cluster_name)
+ desired_cfg_url = cluster_url + '?fields=Clusters/desired_configs'
+ cfg_url = cluster_url + '/configurations?type={0}&tag={1}'
+ desired_cfg = self.p.get(desired_cfg_url)
+
+ hdp = self.p.get('%s/services/HBASE/components/HBASE_REGIONSERVER' % cluster_url)
+ rsnodes = [c['HostRoles']['host_name'] for c in hdp['host_components']]
+
+ hbase_hregion_property = 'hbase.hregion.impl'
+ hbase_config_group = {
+ 'ConfigGroup': {
+ 'cluster_name': self.cluster_name,
+ 'group_name': 'hbase-regionserver',
+ 'tag': 'HBASE',
+ 'description': 'HBase Regionserver configs for Trafodion',
+ 'hosts': [{'host_name': host} for host in rsnodes],
+ 'desired_configs': [
+ {
+ 'type': 'hbase-site',
+ 'tag': 'traf_cg',
+ 'properties': {hbase_hregion_property: MOD_CFGS['hbase-site'].pop(hbase_hregion_property)}
+ }
+ ]
+ }
+ }
+ self.p.post('%s/config_groups' % cluster_url, hbase_config_group)
+
+ for config_type in MOD_CFGS.keys():
+ desired_tag = desired_cfg['Clusters']['desired_configs'][config_type]['tag']
+ current_cfg = self.p.get(cfg_url.format(config_type, desired_tag))
+ tag = 'version' + str(int(time.time() * 1000000))
+ new_properties = current_cfg['items'][0]['properties']
+ new_properties.update(MOD_CFGS[config_type])
+ config = {
+ 'Clusters': {
+ 'desired_config': {
+ 'type': config_type,
+ 'tag': tag,
+ 'properties': new_properties
+ }
+ }
+ }
+ self.p.put(cluster_url, config)
+
+ def restart(self):
+ srv_baseurl = CLUSTER_URL_PTR % (self.url, self.cluster_name) + '/services/'
+ srvs = ['HBASE', 'ZOOKEEPER', 'HDFS']
+
+ # Stop
+ info('Restarting HDP services ...')
+ for srv in srvs:
+ srv_url = srv_baseurl + srv
+ config = {'RequestInfo': {'context' :'Stop %s services' % srv}, 'ServiceInfo': {'state' : 'INSTALLED'}}
+ rc = self.p.put(srv_url, config)
+
+ # check stop status
+ if rc:
+ get_stat = lambda: self.p.get(srv_url)['ServiceInfo']['state'] == 'INSTALLED'
+ retry(get_stat, 30, 5, 'HDP service %s stop' % srv)
+ else:
+ info('HDP service %s had already been stopped' % srv)
+
+ time.sleep(5)
+ # Start
+ config = {'RequestInfo': {'context' :'Start All services'}, 'ServiceInfo': {'state' : 'STARTED'}}
+ rc = self.p.put(srv_baseurl, config)
+
+ # check start status
+ if rc:
+ result_url = rc['href']
+ get_stat = lambda: self.p.get(result_url)['Requests']['request_status'] == 'COMPLETED'
+ retry(get_stat, 120, 5, 'HDP services start')
+ else:
+ info('HDP services had already been started')
+
+def run():
+ if 'CDH' in dbcfgs['distro']:
+ hadoop_mod = CDHMod(dbcfgs['mgr_user'], dbcfgs['mgr_pwd'], dbcfgs['mgr_url'], dbcfgs['cluster_name'])
+ elif 'HDP' in dbcfgs['distro']:
+ hadoop_mod = HDPMod(dbcfgs['mgr_user'], dbcfgs['mgr_pwd'], dbcfgs['mgr_url'], dbcfgs['cluster_name'])
+
+ hadoop_mod.mod()
+ hadoop_mod.restart()
+
+# main
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+dbcfgs = json.loads(dbcfgs_json)
+run()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2886c023/install/python-installer/scripts/hdfs_cmds.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/hdfs_cmds.py b/install/python-installer/scripts/hdfs_cmds.py
new file mode 100755
index 0000000..dc08b7e
--- /dev/null
+++ b/install/python-installer/scripts/hdfs_cmds.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on first node with trafodion user ###
+
+import os
+import sys
+import json
+from common import err, run_cmd, cmd_output, run_cmd_as_user
+
+def run():
+ hdfs_bin = '/usr/bin/hdfs'
+
+ dbcfgs = json.loads(dbcfgs_json)
+ DISTRO = dbcfgs['distro']
+
+ if 'CDH' in DISTRO:
+ parcel_lib = '/opt/cloudera/parcels/CDH/lib/hbase/lib'
+ if os.path.exists(parcel_lib): hdfs_bin = '/opt/cloudera/parcels/CDH/bin/hdfs'
+ elif 'APACHE' in DISTRO:
+ hdfs_bin = dbcfgs['hadoop_home'] + '/bin/hdfs'
+
+ traf_loc = '/user/trafodion'
+ traf_user = dbcfgs['traf_user']
+ hdfs_user = dbcfgs['hdfs_user']
+ hbase_user = dbcfgs['hbase_user']
+
+ run_cmd_as_user(hdfs_user, '%s dfsadmin -safemode wait' % hdfs_bin)
+ run_cmd_as_user(hdfs_user, '%s dfs -mkdir -p %s/{trafodion_backups,bulkload,lobs} /hbase/archive /hbase-staging' % (hdfs_bin, traf_loc))
+ run_cmd_as_user(hdfs_user, '%s dfs -chown -R %s:%s /hbase/archive /hbase-staging' % (hdfs_bin, hbase_user, hbase_user))
+ run_cmd_as_user(hdfs_user, '%s dfs -chown -R %s:%s %s/{trafodion_backups,bulkload,lobs}' % (hdfs_bin, traf_user, traf_user, traf_loc))
+ run_cmd_as_user(hdfs_user, '%s dfs -setfacl -R -m user:%s:rwx /hbase/archive' % (hdfs_bin, traf_user))
+ run_cmd_as_user(hdfs_user, '%s dfs -setfacl -R -m default:user:%s:rwx /hbase/archive' % (hdfs_bin, traf_user))
+ run_cmd_as_user(hdfs_user, '%s dfs -setfacl -R -m mask::rwx /hbase/archive' % hdfs_bin)
+
+ # Grant all privileges to the Trafodion principal in HBase
+ if dbcfgs['secure_hadoop'] == 'Y':
+ run_cmd('grant "%s", "RWXC" | sudo -u %s hbase shell > /tmp/hbase_shell.out' % (traf_user, hbase_user))
+ has_err = cmd_output('grep -c ERROR /tmp/hbase_shell.out')
+ if int(has_err):
+ err('Failed to grant HBase privileges to %s' % traf_user)
+ run_cmd('rm /tmp/hbase_shell.out')
+# main
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+run()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2886c023/install/python-installer/scripts/traf_check.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_check.py b/install/python-installer/scripts/traf_check.py
new file mode 100755
index 0000000..31d62a9
--- /dev/null
+++ b/install/python-installer/scripts/traf_check.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on all nodes with sudo user ###
+
+import re
+import json
+import sys
+import os
+from common import run_cmd, cmd_output, err, Version
+
+class Check(object):
+ """ check system envs """
+
+ def __init__(self, dbcfgs_json):
+ self.dbcfgs = json.loads(dbcfgs_json)
+ self.version = Version()
+
+ def check_sudo(self):
+ """ check sudo access """
+ run_cmd('sudo -n echo -n "check sudo access" > /dev/null 2>&1')
+
+ def check_hbase_xml(self):
+ """ check if hbase-site.xml file exists """
+ hbase_xml_file = self.dbcfgs['hbase_xml_file']
+ if not os.path.exists(hbase_xml_file):
+ err('HBase xml file is not found')
+
+ def check_java(self):
+ """ check JDK version """
+ jdk_path = self.dbcfgs['java_home']
+ jdk_ver = cmd_output('%s/bin/javac -version' % jdk_path)
+ try:
+ jdk_ver, sub_ver = re.search(r'javac (\d\.\d).\d_(\d+)', jdk_ver).groups()
+ except AttributeError:
+ err('No JDK found')
+
+ if self.dbcfgs['req_java8'] == 'Y': # only allow JDK1.8
+ support_java = '1.8'
+ else:
+ support_java = self.version.get_version('java')
+
+ if jdk_ver == '1.7' and int(sub_ver) < 65:
+ err('Unsupported JDK1.7 version, sub version should be higher than 65')
+ if jdk_ver not in support_java:
+ err('Unsupported JDK version %s, supported version: %s' % (jdk_ver, support_java))
+
+ #def check_scratch_loc(self):
+ # """ check if scratch file folder exists """
+ # scratch_locs = self.dbcfgs['scratch_locs'].split(',')
+ # for loc in scratch_locs:
+ # if not os.path.exists(loc):
+ # err('Scratch file location \'%s\' doesn\'t exist' % loc)
+
+def run():
+ PREFIX = 'check_'
+ check = Check(dbcfgs_json)
+
+ # call method
+ [getattr(check, m)() for m in dir(check) if m.startswith(PREFIX)]
+
+# main
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+run()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2886c023/install/python-installer/scripts/traf_dep.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_dep.py b/install/python-installer/scripts/traf_dep.py
new file mode 100755
index 0000000..69ad489
--- /dev/null
+++ b/install/python-installer/scripts/traf_dep.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on all nodes with sudo user ###
+
+import re
+import os
+import sys
+import json
+import platform
+from common import run_cmd, cmd_output, err
+
+LOCAL_REPO_PTR = """
+[traflocal]
+name=trafodion local repo
+baseurl=http://%s:%s/
+enabled=1
+gpgcheck=0
+"""
+
+REPO_FILE = '/etc/yum.repos.d/traflocal.repo'
+EPEL_FILE = '/etc/yum.repos.d/epel.repo'
+
+def run():
+ """ install Trafodion dependencies """
+
+ dbcfgs = json.loads(dbcfgs_json)
+
+ node_list = dbcfgs['node_list'].split(',')
+
+ offline = True if dbcfgs['offline_mode'] == 'Y' else False
+
+ if offline:
+ repo_content = LOCAL_REPO_PTR % (dbcfgs['repo_ip'], dbcfgs['repo_http_port'])
+ with open(REPO_FILE, 'w') as f:
+ f.write(repo_content)
+
+ if not offline and not os.path.exists(EPEL_FILE):
+ run_cmd('yum install -y epel-release')
+
+ package_list = [
+ 'apr',
+ 'apr-util',
+ 'expect',
+ 'gzip',
+ 'libiodbc-devel',
+ 'lzo',
+ 'lzop',
+ 'pdsh', # epel
+ 'perl-DBD-SQLite',
+ 'perl-Params-Validate',
+ 'perl-Time-HiRes',
+ 'protobuf', # epel
+ 'sqlite',
+ 'snappy',
+ 'unixODBC-devel',
+ 'unzip'
+ ]
+
+ if dbcfgs['ldap_security'].upper() == 'Y':
+ package_list += ['openldap-clients']
+
+ all_pkg_list = run_cmd('rpm -qa')
+ for pkg in package_list:
+ if pkg in all_pkg_list:
+ print 'Package %s had already been installed' % pkg
+ else:
+ print 'Installing %s ...' % pkg
+ if offline:
+ run_cmd('yum install -y --disablerepo=\* --enablerepo=traflocal %s' % pkg)
+ else:
+ run_cmd('yum install -y %s' % pkg)
+
+ # pdsh should not exist on single node
+ if len(node_list) == 1:
+ cmd_output('yum remove -y pdsh')
+
+ # remove temp repo file
+ if offline: os.remove(REPO_FILE)
+
+# main
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+run()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2886c023/install/python-installer/scripts/traf_discover.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_discover.py b/install/python-installer/scripts/traf_discover.py
new file mode 100755
index 0000000..d662d47
--- /dev/null
+++ b/install/python-installer/scripts/traf_discover.py
@@ -0,0 +1,270 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on all nodes with sudo user ###
+
+import re
+import json
+import sys
+import platform
+from glob import glob
+from common import cmd_output, err, Version, ParseXML
+
+PREFIX = 'get_'
+NA = 'N/A' # not available
+NS = 'N/S' # not supported
+OK = 'OK'
+
+def deco(func):
+ def wrapper(self):
+ if PREFIX in func.__name__:
+ name = func.__name__.replace(PREFIX, '')
+ return name, func(self)
+ else:
+ return
+ return wrapper
+
+
+class Discover(object):
+ """ discover functions, to add a new discover function,
+ simply add a new def with name get_xx and decorated
+ by 'deco', then return result in string format:
+
+ @deco
+ def get_xx(self):
+ # do something
+ return result
+ """
+
+ def __init__(self, dbcfgs):
+ self.CPUINFO = cmd_output('cat /proc/cpuinfo')
+ self.MEMINFO = cmd_output('cat /proc/meminfo')
+ self.SYSCTLINFO = cmd_output('sysctl -a')
+ self.version = Version()
+ self.dbcfgs = dbcfgs
+
+ def _parse_string(self, info, string):
+ try:
+ info = info.split('\n')
+ string_line = [line for line in info if string in line][0]
+ except IndexError:
+ err('Cannot get %s info' % string)
+
+ return string_line
+
+ def _get_cpu_info(self, string):
+ return self._parse_string(self.CPUINFO, string).split(':')[1].strip()
+
+ def _get_mem_info(self, string):
+ return self._parse_string(self.MEMINFO, string).split(':')[1].split()[0]
+
+ def _get_sysctl_info(self, string):
+ return self._parse_string(self.SYSCTLINFO, string).split('=')[1].strip()
+
+ @deco
+ def get_linux(self):
+ """ get linux version """
+ os_dist, os_ver = platform.dist()[:2]
+ if os_dist not in self.version.get_version('linux'):
+ return NA
+ else:
+ if not os_ver.split('.')[0] in self.version.get_version(os_dist):
+ return NA
+ return '%s-%s' % (os_dist, os_ver)
+
+ @deco
+ def get_firewall_status(self):
+ """ get firewall running status """
+ iptables_stat = cmd_output('iptables -nL|grep -vE "(Chain|target)"').strip()
+ if iptables_stat:
+ return 'Running'
+ else:
+ return 'Stopped'
+
+ @deco
+ def get_pidmax(self):
+ """ get kernel pid max setting """
+ return self._get_sysctl_info('kernel.pid_max')
+
+ @deco
+ def get_default_java(self):
+ """ get default java version """
+ jdk_path = glob('/usr/java/*') + \
+ glob('/usr/jdk64/*') + \
+ glob('/usr/lib/jvm/java-*-openjdk.x86_64')
+
+ jdk_list = {} # {jdk_version: jdk_path}
+ for path in jdk_path:
+ jdk_ver = cmd_output('%s/bin/javac -version' % path)
+
+ try:
+ main_ver, sub_ver = re.search(r'(\d\.\d\.\d)_(\d+)', jdk_ver).groups()
+ # don't support JDK version less than 1.7.0_65
+ if main_ver == '1.7.0' and int(sub_ver) < 65:
+ continue
+ jdk_list[main_ver] = path
+ except AttributeError:
+ continue
+
+ if not jdk_list:
+ return NA
+ else:
+ # use JDK1.8 first
+ if jdk_list.has_key('1.8.0'):
+ return jdk_list['1.8.0']
+ elif jdk_list.has_key('1.7.0'):
+ return jdk_list['1.7.0']
+
+ @deco
+ def get_hive(self):
+ """ get Hive status """
+ hive_stat = cmd_output('which hive')
+ if 'no hive' in hive_stat:
+ return NA
+ else:
+ return OK
+
+ def _get_core_site_xml(self):
+ if self.dbcfgs.has_key('hadoop_home'): # apache distro
+ CORE_SITE_XML = '%s/etc/hadoop/core-site.xml' % self.dbcfgs['hadoop_home']
+ else:
+ CORE_SITE_XML = '/etc/hadoop/conf/core-site.xml'
+ p = ParseXML(CORE_SITE_XML)
+ return p
+
+ @deco
+ def get_hadoop_authentication(self):
+ p = self._get_core_site_xml()
+ return p.get_property('hadoop.security.authentication')
+
+ @deco
+ def get_hadoop_authorization(self):
+ p = self._get_core_site_xml()
+ return p.get_property('hadoop.security.authorization')
+
+ @deco
+ def get_hbase(self):
+ """ get HBase version """
+ if self.dbcfgs.has_key('hbase_home'): # apache distro
+ hbase_home = self.dbcfgs['hbase_home']
+ hbase_ver = cmd_output('%s/bin/hbase version | head -n1' % hbase_home)
+ else:
+ hbase_ver = cmd_output('hbase version | head -n1')
+
+ support_hbase_ver = self.version.get_version('hbase')
+ try:
+ hbase_ver = re.search(r'HBase (\d\.\d)', hbase_ver).groups()[0]
+ except AttributeError:
+ return NA
+ if hbase_ver not in support_hbase_ver:
+ return NS
+ return hbase_ver
+
+ @deco
+ def get_cpu_model(self):
+ """ get CPU model """
+ return self._get_cpu_info('model name')
+
+ @deco
+ def get_cpu_cores(self):
+ """ get CPU cores """
+ return self.CPUINFO.count('processor')
+
+ @deco
+ def get_arch(self):
+ """ get CPU architecture """
+ arch = platform.processor()
+ if not arch:
+ arch = 'Unknown'
+ return arch
+
+ @deco
+ def get_mem_total(self):
+ """ get total memory size """
+ mem = self._get_mem_info('MemTotal')
+ memsize = mem.split()[0]
+
+ return "%0.1f GB" % round(float(memsize) / (1024 * 1024), 2)
+
+ @deco
+ def get_mem_free(self):
+ """ get current free memory size """
+ free = self._get_mem_info('MemFree')
+ buffers = self._get_mem_info('Buffers')
+ cached = self._get_mem_info('Cached')
+ memfree = float(free) + float(buffers) + float(cached)
+
+ return "%0.1f GB" % round(memfree / (1024 * 1024), 2)
+
+ @deco
+ def get_ext_interface(self):
+ """ get external network interface """
+ return cmd_output('ip route |grep default|awk \'{print $5}\'')
+
+ @deco
+ def get_rootdisk_free(self):
+ """ get root disk space left """
+ space = cmd_output('df -h|grep "\/$" | awk \'{print $4}\'')
+ return space.strip()
+
+ @deco
+ def get_python_ver(self):
+ """ get python version """
+ return platform.python_version()
+
+ @deco
+ def get_home_dir(self):
+ if self.dbcfgs.has_key('traf_user'): # apache distro
+ traf_user = self.dbcfgs['traf_user']
+ return cmd_output("getent passwd %s | awk -F: '{print $6}' | sed 's/\/%s//g'" % (traf_user, traf_user))
+ else:
+ return ''
+
+ @deco
+ def get_traf_status(self):
+ """ get trafodion running status """
+ mon_process = cmd_output('ps -ef|grep -v grep|grep -c "monitor COLD"')
+ if int(mon_process) > 0:
+ return 'Running'
+ else:
+ return 'Stopped'
+
+def run():
+ try:
+ dbcfgs_json = sys.argv[1]
+ except IndexError:
+ err('No db config found')
+ dbcfgs = json.loads(dbcfgs_json)
+ discover = Discover(dbcfgs)
+ methods = [m for m in dir(discover) if m.startswith(PREFIX)]
+ result = {}
+ for method in methods:
+ key, value = getattr(discover, method)() # call method
+ result[key] = value
+
+ print json.dumps(result)
+
+
+# main
+if __name__ == '__main__':
+ run()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2886c023/install/python-installer/scripts/traf_kerberos.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_kerberos.py b/install/python-installer/scripts/traf_kerberos.py
new file mode 100755
index 0000000..2e89d2c
--- /dev/null
+++ b/install/python-installer/scripts/traf_kerberos.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on all nodes with sudo user ###
+
+import sys
+import re
+import json
+import socket
+from common import run_cmd, cmd_output, err
+
+def run():
+ """ setup Kerberos security """
+ dbcfgs = json.loads(dbcfgs_json)
+
+ distro = dbcfgs['distro']
+ admin_principal = dbcfgs['admin_principal']
+ admin_passwd = dbcfgs['kdcadmin_pwd']
+ kdc_server = dbcfgs['kdc_server']
+ # maxlife = dbcfgs['max_lifetime']
+ # max_renewlife = dbcfgs['max_renew_lifetime']
+ maxlife = '24hours'
+ max_renewlife = '7days'
+ kadmin_cmd = 'kadmin -p %s -w %s -s %s -q' % (admin_principal, admin_passwd, kdc_server)
+
+ host_name = socket.getfqdn()
+ traf_user = dbcfgs['traf_user']
+ hdfs_user = 'hdfs'
+ hbase_user = 'hbase'
+ realm = re.match('.*@(.*)', admin_principal).groups()[0]
+ traf_keytab_dir = '/etc/%s/keytab' % traf_user
+ traf_keytab = '%s/%s.keytab' % (traf_keytab_dir, traf_user)
+ traf_principal = '%s/%s@%s' % (traf_user, host_name, realm)
+ hdfs_principal = '%s/%s@%s' % (hdfs_user, host_name, realm)
+ hbase_principal = '%s/%s@%s' % (hbase_user, host_name, realm)
+
+ ### setting start ###
+ print 'Checking KDC server connection'
+ run_cmd('%s listprincs' % kadmin_cmd)
+
+ # create principals and keytabs for trafodion user
+ principal_exists = cmd_output('%s listprincs | grep -c %s' % (kadmin_cmd, traf_principal))
+ if int(principal_exists) == 0: # not exist
+ run_cmd('%s \'addprinc -randkey %s\'' % (kadmin_cmd, traf_principal))
+ # Adjust principal's maxlife and maxrenewlife
+ run_cmd('%s \'modprinc -maxlife %s -maxrenewlife %s\' %s >/dev/null 2>&1' % (kadmin_cmd, maxlife, max_renewlife, traf_principal))
+
+ run_cmd('mkdir -p %s' % traf_keytab_dir)
+
+ # TODO: need skip add keytab if exist?
+ print 'Create keytab file for trafodion user'
+ run_cmd('%s \'ktadd -k %s %s\'' % (kadmin_cmd, traf_keytab, traf_principal))
+ run_cmd('chown %s %s' % (traf_user, traf_keytab))
+ run_cmd('chmod 400 %s' % traf_keytab)
+
+ # create principals for hdfs/hbase user
+ print 'Create principals for hdfs/hbase user'
+ if 'CDH' in distro:
+ hdfs_keytab = cmd_output('find /var/run/cloudera-scm-agent/process/ -name hdfs.keytab | head -n 1')
+ hbase_keytab = cmd_output('find /var/run/cloudera-scm-agent/process/ -name hbase.keytab | head -n 1')
+ elif 'HDP' in distro:
+ hdfs_keytab = '/etc/security/keytabs/hdfs.headless.keytab'
+ hbase_keytab = '/etc/security/keytabs/hbase.service.keytab'
+
+ run_cmd('sudo -u %s kinit -kt %s %s' % (hdfs_user, hdfs_keytab, hdfs_principal))
+ run_cmd('sudo -u %s kinit -kt %s %s' % (hbase_user, hbase_keytab, hbase_principal))
+
+ print 'Done creating principals and keytabs'
+
+ kinit_bashrc = """
+
+# ---------------------------------------------------------------
+# if needed obtain and cache the Kerberos ticket-granting ticket
+# start automatic ticket renewal process
+# ---------------------------------------------------------------
+klist -s >/dev/null 2>&1
+if [[ $? -eq 1 ]]; then
+ kinit -kt %s %s >/dev/null 2>&1
+fi
+
+# ---------------------------------------------------------------
+# Start trafodion kerberos ticket manager process
+# ---------------------------------------------------------------
+$TRAF_HOME/sql/scripts/krb5service start >/dev/null 2>&1
+""" % (traf_keytab, traf_principal)
+
+ traf_bashrc = '/home/%s/.bashrc' % traf_user
+ with open(traf_bashrc, 'a') as f:
+ f.write(kinit_bashrc)
+
+# main
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+run()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2886c023/install/python-installer/scripts/traf_ldap.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_ldap.py b/install/python-installer/scripts/traf_ldap.py
new file mode 100755
index 0000000..9013482
--- /dev/null
+++ b/install/python-installer/scripts/traf_ldap.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on all nodes with trafodion user ###
+
+import os
+import sys
+import json
+from common import run_cmd, mod_file, err, TMP_DIR
+
+def run():
+ """ setup LDAP security """
+ dbcfgs = json.loads(dbcfgs_json)
+
+ DB_ROOT_USER = dbcfgs['db_root_user']
+ TRAF_HOME = os.environ['TRAF_HOME']
+ SQENV_FILE = TRAF_HOME + '/sqenvcom.sh'
+ TRAF_AUTH_CONFIG = '%s/sql/scripts/.traf_authentication_config' % TRAF_HOME
+ TRAF_AUTH_TEMPLATE = '%s/templates/traf_authentication_conf.template' % TMP_DIR
+
+ # set traf_authentication_config file
+ change_items = {
+ '{{ ldap_hosts }}': dbcfgs['ldap_hosts'],
+ '{{ ldap_port }}': dbcfgs['ldap_port'],
+ '{{ ldap_identifiers }}': dbcfgs['ldap_identifiers'],
+ '{{ ldap_encrypt }}': dbcfgs['ldap_encrypt'],
+ '{{ ldap_certpath }}': dbcfgs['ldap_certpath'],
+ '{{ ldap_user }}': dbcfgs['ldap_user'],
+ '{{ ldap_pwd }}': dbcfgs['ldap_pwd']
+ }
+
+ print 'Modify authentication config file'
+ run_cmd('cp %s %s' % (TRAF_AUTH_TEMPLATE, TRAF_AUTH_CONFIG))
+ mod_file(TRAF_AUTH_CONFIG, change_items)
+
+
+ print 'Check LDAP Configuration file for errors'
+ run_cmd('ldapconfigcheck -file %s' % TRAF_AUTH_CONFIG)
+
+ print 'Verify that LDAP user %s exists' % DB_ROOT_USER
+ run_cmd('ldapcheck --verbose --username=%s' % DB_ROOT_USER)
+ #if not 'Authentication successful' in ldapcheck_result:
+ # err('Failed to access LDAP server with user %s' % DB_ROOT_USER)
+
+ print 'Modfiy sqenvcom.sh to turn on authentication'
+ mod_file(SQENV_FILE, {'TRAFODION_ENABLE_AUTHENTICATION=NO':'TRAFODION_ENABLE_AUTHENTICATION=YES'})
+
+# main
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+run()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2886c023/install/python-installer/scripts/traf_package.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_package.py b/install/python-installer/scripts/traf_package.py
new file mode 100755
index 0000000..c5bb6ec
--- /dev/null
+++ b/install/python-installer/scripts/traf_package.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+## This script should be run on all nodes with trafodion user ##
+
+import sys
+import json
+from common import run_cmd, err
+
+def run():
+ dbcfgs = json.loads(dbcfgs_json)
+
+ TRAF_DIRNAME = dbcfgs['traf_dirname']
+
+ # untar traf package, package comes from copy_files.py
+ TRAF_PACKAGE_FILE = '/tmp/' + dbcfgs['traf_package'].split('/')[-1]
+ run_cmd('mkdir -p ~/%s' % TRAF_DIRNAME)
+ run_cmd('tar xf %s -C ~/%s' % (TRAF_PACKAGE_FILE, TRAF_DIRNAME))
+
+ # copy dbcfgs file
+ run_cmd('cp -rf /tmp/dbcfgs ~/.dbcfgs')
+
+ print 'Trafodion package extracted successfully!'
+
+# main
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+run()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2886c023/install/python-installer/scripts/traf_setup.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_setup.py b/install/python-installer/scripts/traf_setup.py
new file mode 100755
index 0000000..5a66827
--- /dev/null
+++ b/install/python-installer/scripts/traf_setup.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on all nodes with sudo user ###
+
+import os
+import sys
+import re
+import json
+from common import err, cmd_output, run_cmd
+
+def run():
+ dbcfgs = json.loads(dbcfgs_json)
+
+ HOME_DIR = cmd_output('cat /etc/default/useradd |grep HOME |cut -d "=" -f 2').strip()
+ if dbcfgs.has_key('home_dir'):
+ HOME_DIR = dbcfgs['home_dir']
+
+ TRAF_USER = dbcfgs['traf_user']
+ TRAF_DIRNAME = dbcfgs['traf_dirname']
+ TRAF_HOME = '%s/%s/%s' % (HOME_DIR, TRAF_USER, TRAF_DIRNAME)
+
+ TRAF_VER = dbcfgs['traf_version']
+ DISTRO = dbcfgs['distro']
+ TRAF_LIB_PATH = TRAF_HOME + '/export/lib'
+ SCRATCH_LOCS = dbcfgs['scratch_locs'].split(',')
+
+ SUDOER_FILE = '/etc/sudoers.d/trafodion'
+ SUDOER_CFG = """
+## Allow trafodion id to run commands needed for backup and restore
+%%%s ALL =(hbase) NOPASSWD: /usr/bin/hbase"
+""" % TRAF_USER
+
+ ### kernel settings ###
+ run_cmd('sysctl -w kernel.pid_max=65535 2>&1 > /dev/null')
+ run_cmd('echo "kernel.pid_max=65535" >> /etc/sysctl.conf')
+
+ ### copy init script ###
+ init_script = '%s/sysinstall/etc/init.d/trafodion' % TRAF_HOME
+ if os.path.exists(init_script):
+ run_cmd('cp -rf %s /etc/init.d/' % init_script)
+ run_cmd('chkconfig --add trafodion')
+ run_cmd('chkconfig --level 06 trafodion on')
+
+ ### create and set permission for scratch file dir ###
+ for loc in SCRATCH_LOCS:
+ # don't set permission for HOME folder
+ if not os.path.exists(loc):
+ run_cmd('mkdir -p %s' % loc)
+ if HOME_DIR not in loc:
+ run_cmd('chmod 777 %s' % loc)
+
+ ### copy jar files ###
+ hbase_lib_path = dbcfgs['hbase_lib_path']
+ if 'APACHE' in DISTRO:
+ hbase_home = dbcfgs['hbase_home']
+ hbase_lib_path = hbase_home + '/lib'
+ # for apache distro, get hbase version from cmdline
+ hbase_ver = cmd_output('%s/bin/hbase version | head -n1' % hbase_home)
+ hbase_ver = re.search(r'HBase (\d\.\d)', hbase_ver).groups()[0]
+ DISTRO += hbase_ver
+
+ distro, v1, v2 = re.search(r'(\w+)-*(\d)\.(\d)', DISTRO).groups()
+ if distro == 'CDH':
+ if v2 == '6': v2 = '5'
+ if v2 == '8': v2 = '7'
+ elif distro == 'HDP':
+ if v2 == '4': v2 = '3'
+
+ hbase_trx_jar = 'hbase-trx-%s%s_%s-%s.jar' % (distro.lower(), v1, v2, TRAF_VER)
+ traf_hbase_trx_path = '%s/%s' % (TRAF_LIB_PATH, hbase_trx_jar)
+ hbase_trx_path = '%s/%s' % (hbase_lib_path, hbase_trx_jar)
+ if not os.path.exists(traf_hbase_trx_path):
+ err('Cannot find HBase trx jar \'%s\' for your Hadoop distribution' % hbase_trx_jar)
+
+ # upgrade mode, check if existing trx jar doesn't match the new trx jar file
+ if dbcfgs.has_key('upgrade') and dbcfgs['upgrade'].upper() == 'Y':
+ if not os.path.exists(hbase_trx_path):
+ err('The trx jar \'%s\' doesn\'t exist in hbase lib path, cannot do upgrade, please do regular install' % hbase_trx_jar)
+ else:
+ # remove old trx and trafodion-utility jar files
+ run_cmd('rm -rf %s/{hbase-trx-*,trafodion-utility-*}' % hbase_lib_path)
+
+ # copy new ones
+ run_cmd('cp %s %s' % (traf_hbase_trx_path, hbase_lib_path))
+ run_cmd('cp %s/trafodion-utility-* %s' % (TRAF_LIB_PATH, hbase_lib_path))
+
+ # set permission
+ run_cmd('chmod +r %s/{hbase-trx-*,trafodion-utility-*}' % hbase_lib_path)
+
+ if dbcfgs['dcs_ha'] == 'Y':
+ # set trafodion sudoer file for specific cmds
+ SUDOER_CFG += """
+## Trafodion Floating IP commands
+Cmnd_Alias IP = /sbin/ip
+Cmnd_Alias ARP = /sbin/arping
+
+## Allow Trafodion id to run commands needed to configure floating IP
+%%%s ALL = NOPASSWD: IP, ARP
+""" % TRAF_USER
+
+ ### write trafodion sudoer file ###
+ with open(SUDOER_FILE, 'w') as f:
+ f.write(SUDOER_CFG)
+
+# main
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+run()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2886c023/install/python-installer/scripts/traf_sqconfig.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_sqconfig.py b/install/python-installer/scripts/traf_sqconfig.py
new file mode 100755
index 0000000..6c8c188
--- /dev/null
+++ b/install/python-installer/scripts/traf_sqconfig.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on first node with trafodion user ###
+
+import os
+import sys
+import json
+from common import run_cmd, err
+
+def run():
+ dbcfgs = json.loads(dbcfgs_json)
+
+ nodes = dbcfgs['node_list'].split(',')
+ scratch_locs = dbcfgs['scratch_locs'].split(',')
+
+ # this script is running by trafodion user, so get sqroot from env
+ TRAF_HOME = os.environ['TRAF_HOME']
+ if TRAF_HOME == '': err('TRAF_HOME var is empty')
+ SQCONFIG_FILE = TRAF_HOME + '/sql/scripts/sqconfig'
+
+ core, processor = run_cmd("lscpu|grep -E '(^CPU\(s\)|^Socket\(s\))'|awk '{print $2}'").split('\n')[:2]
+ core = int(core)-1 if int(core) <= 256 else 255
+
+ lines = ['begin node\n']
+ for node_id, node in enumerate(nodes):
+ line = 'node-id=%s;node-name=%s;cores=0-%d;processors=%s;roles=connection,aggregation,storage\n' % (node_id, node, core, processor)
+ lines.append(line)
+
+ lines.append('end node\n')
+ lines.append('\n')
+ lines.append('begin overflow\n')
+
+ for scratch_loc in scratch_locs:
+ line = 'hdd %s\n' % scratch_loc
+ lines.append(line)
+
+ lines.append('end overflow\n')
+
+ with open(SQCONFIG_FILE, 'w') as f:
+ f.writelines(lines)
+
+ print 'sqconfig generated successfully!'
+
+ run_cmd('sqgen')
+
+ print 'sqgen ran successfully!'
+
+# main
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+run()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2886c023/install/python-installer/scripts/traf_start.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_start.py b/install/python-installer/scripts/traf_start.py
new file mode 100755
index 0000000..395f678
--- /dev/null
+++ b/install/python-installer/scripts/traf_start.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on first node with trafodion user ###
+
+import sys
+import time
+import os
+import json
+from common import cmd_output, run_cmd, err
+
+def run():
+ """ start trafodion instance """
+ dbcfgs = json.loads(dbcfgs_json)
+
+ print 'Starting trafodion'
+ TRAF_HOME = os.environ['TRAF_HOME']
+ if os.path.exists('%s/sql/scripts/trafstart' % TRAF_HOME):
+ run_cmd('trafstart')
+ else:
+ run_cmd('sqstart')
+
+ # set a uniq file name
+ tmp_file = '/tmp/initialize.out.' + str(int(time.time()))
+ print 'Initialize trafodion'
+ run_cmd('echo "initialize trafodion;" | sqlci > %s' % tmp_file)
+ init_output = cmd_output('cat %s' % tmp_file)
+ # error 1392, 1395
+ if '1392' in init_output or '1395' in init_output:
+ run_cmd('echo "get version of metadata;" | sqlci > %s' % tmp_file)
+ meta_current = cmd_output('grep \'Metadata is current\' %s | wc -l' % tmp_file)
+ if meta_current != "1":
+ print 'Initialize trafodion, upgrade'
+ run_cmd('echo "initialize trafodion, upgrade;" | sqlci > %s' % tmp_file)
+ # other errors
+ elif 'ERROR' in init_output:
+ err('Failed to initialize trafodion:\n %s' % init_output)
+
+ run_cmd('rm -rf %s' % tmp_file)
+ if dbcfgs['ldap_security'] == 'Y':
+ run_cmd('echo "initialize authorization; alter user DB__ROOT set external name \"%s\";" | sqlci > %s' % (dbcfgs['db_root_user'], tmp_file))
+ if dbcfgs.has_key('db_admin_user'):
+ run_cmd('echo "alter user DB__ADMIN set external name \"%s\";" | sqlci >> %s' % (dbcfgs['db_admin_user'], tmp_file))
+
+ secure_output = cmd_output('cat %s' % tmp_file)
+ if 'ERROR' in secure_output:
+ err('Failed to setup security for trafodion:\n %s' % secure_output)
+
+ run_cmd('rm -rf %s' % tmp_file)
+ if os.path.exists('%s/sql/scripts/connstart' % TRAF_HOME):
+ run_cmd('connstart')
+
+ print 'Start trafodion successfully.'
+
+# main
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+run()