You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by sv...@apache.org on 2016/11/09 18:55:06 UTC
[2/6] incubator-trafodion git commit: [TRAFODION-1839] Trafodion
Installer Evolution
[TRAFODION-1839] Trafodion Installer Evolution
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/904b53df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/904b53df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/904b53df
Branch: refs/heads/master
Commit: 904b53df5c75f0dd5b0bd84e483c7d060023a7d6
Parents: 6ce6a8e
Author: Eason <hf...@gmail.com>
Authored: Tue Nov 1 16:53:27 2016 +0800
Committer: Eason <hf...@gmail.com>
Committed: Tue Nov 1 16:53:27 2016 +0800
----------------------------------------------------------------------
install/python-installer/README.md | 21 +
install/python-installer/apache_mods.py | 72 +++
install/python-installer/bashrc.template | 78 +++
install/python-installer/common.py | 479 ++++++++++++++
install/python-installer/copy_files.py | 64 ++
install/python-installer/db_config_default | 120 ++++
install/python-installer/db_install.py | 643 +++++++++++++++++++
install/python-installer/dcs_setup.py | 114 ++++
install/python-installer/discovery.py | 144 +++++
install/python-installer/hadoop_mods.py | 203 ++++++
install/python-installer/hdfs_cmds.py | 68 ++
install/python-installer/mod_cfgs.json | 73 +++
install/python-installer/prompt.json | 227 +++++++
install/python-installer/script.json | 87 +++
.../traf_authentication_conf.template | 71 ++
install/python-installer/traf_check.py | 87 +++
install/python-installer/traf_dep.py | 119 ++++
install/python-installer/traf_discover.py | 253 ++++++++
install/python-installer/traf_kerberos.py | 116 ++++
install/python-installer/traf_ldap.py | 73 +++
install/python-installer/traf_package.py | 47 ++
install/python-installer/traf_setup.py | 125 ++++
install/python-installer/traf_sqconfig.py | 74 +++
install/python-installer/traf_start.py | 70 ++
install/python-installer/traf_user.py | 145 +++++
install/python-installer/version.json | 10 +
install/python-installer/wrapper.py | 301 +++++++++
27 files changed, 3884 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/904b53df/install/python-installer/README.md
----------------------------------------------------------------------
diff --git a/install/python-installer/README.md b/install/python-installer/README.md
new file mode 100644
index 0000000..e60a331
--- /dev/null
+++ b/install/python-installer/README.md
@@ -0,0 +1,21 @@
+# Apache Trafodion Python Installer
+
+## Prerequisite:
+
+- CDH/HDP is installed on Trafodion nodes with web UI enabled, or Apache Hadoop, HBase is installed on the same directory on all nodes
+- /etc/hosts contains hostname info for all Trafodion nodes on **installer node**
+- python version 2.6/2.7, and python library `httplib2`, `prettytable`
+- Trafodion server package file is stored on **installer node**
+- Passwordless SSH login, two ways:
+ - Set SSH key pairs against **installer node** and Trafodion nodes
+ - Install `sshpass` tool on **installer node**, then input the SSH password during installation using `--enable-pwd` option
+
+> **installer node** can be any nodes as long as it can ssh to Trafodion nodes, it also can be one of the Trafodion nodes
+
+## How to use:
+- Two ways:
+ - Simply invoke `./db_install.py` to start the installation in guided mode
+ - Copy the `db_config_default` file to `your_db_config` and modify it, then invoke `./db_config.py --config-file your_db_config` to start installation in config mode
+- For a quick install with default settings, you only need to put Trafodion package file in installer's directory, provide CDH/HDP web URL in `your_db_config` file and then it's ready to go!
+- Use `./db_install.py --help` for more options
+- Invoke `./discovery.py` to get the system basic info on all nodes
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/904b53df/install/python-installer/apache_mods.py
----------------------------------------------------------------------
diff --git a/install/python-installer/apache_mods.py b/install/python-installer/apache_mods.py
new file mode 100755
index 0000000..17a54d5
--- /dev/null
+++ b/install/python-installer/apache_mods.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on all nodes with sudo user ###
+
+import sys
+import json
+import socket
+from common import MODCFG_FILE, ParseJson, ParseXML, err, run_cmd
+
+def run():
+ dbcfgs = json.loads(dbcfgs_json)
+ if 'APACHE' in dbcfgs['distro']:
+ modcfgs = ParseJson(MODCFG_FILE).load()
+ MOD_CFGS = modcfgs['MOD_CFGS']
+
+ hdfs_xml_file = dbcfgs['hdfs_xml_file']
+ hbase_xml_file = dbcfgs['hbase_xml_file']
+
+ hbasexml = ParseXML(hbase_xml_file)
+ for key, value in MOD_CFGS['hbase-site'].items():
+ hbasexml.add_property(key, value)
+ hbasexml.write_xml()
+
+ hdfsxml = ParseXML(hdfs_xml_file)
+ for key, value in MOD_CFGS['hdfs-site'].items():
+ hdfsxml.add_property(key, value)
+ hdfsxml.write_xml()
+
+ print 'Apache Hadoop modification completed'
+ first_node = dbcfgs['first_rsnode']
+ local_host = socket.gethostname()
+ if first_node in local_host:
+ hadoop_home = dbcfgs['hadoop_home']
+ hbase_home = dbcfgs['hbase_home']
+ # stop
+ run_cmd(hbase_home + '/bin/stop-hbase.sh')
+ run_cmd(hadoop_home + '/sbin/stop-dfs.sh')
+ # start
+ run_cmd(hadoop_home + '/sbin/start-dfs.sh')
+ run_cmd(hbase_home + '/bin/start-hbase.sh')
+
+ print 'Apache Hadoop restart completed'
+ else:
+ print 'no apache distribution found, skipping'
+
+# main
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+run()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/904b53df/install/python-installer/bashrc.template
----------------------------------------------------------------------
diff --git a/install/python-installer/bashrc.template b/install/python-installer/bashrc.template
new file mode 100644
index 0000000..ceed82a
--- /dev/null
+++ b/install/python-installer/bashrc.template
@@ -0,0 +1,78 @@
+#!/bin/sh
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+
+# This is the .bashrc for the Trafodion environment
+#
+#-------------------------------------------
+# Execute the system's default .bashrc first
+#-------------------------------------------
+if [ -f /etc/bashrc ]; then
+ . /etc/bashrc
+fi
+
+cd $HOME
+
+#-------------------------------------------
+# full path of your Trafodion installation
+#-------------------------------------------
+export JAVA_HOME="{{ java_home }}"
+export SQ_HOME="{{ sq_home }}"
+export MY_SQROOT=$SQ_HOME
+
+#-------------------------------------------
+# other env vars needed by Trafodion
+#-------------------------------------------
+
+# These env vars define all nodes in the cluster
+export NODE_LIST="{{ node_list }}"
+export MY_NODES="{{ my_nodes }}"
+export node_count="{{ node_count }}"
+export HADOOP_TYPE="{{ hadoop_type }}"
+
+#-------------------------------------------
+# Execute the sqenv.sh script if it exists.
+#-------------------------------------------
+PATH=".:$PATH"
+if [ -f $MY_SQROOT/sqenv.sh ]; then
+ pushd . >/dev/null
+ cd $MY_SQROOT
+ source ./sqenv.sh
+ popd >/dev/null
+ export MANPATH=$MANPATH:$MPI_ROOT/share/man
+fi
+
+#-------------------------------------------
+# additional settings for Trafodion environment
+#-------------------------------------------
+ETC_SECURITY_MSG="***ERROR: To fix this please configure /etc/security/limits.conf properly on $HOSTNAME."
+
+# set core file size
+ulimit -c unlimited
+
+# set max open files
+ulimit -n 32768
+if [ $? -ne 0 ]; then
+ echo "***ERROR: Unable to set max open files. Current value $(ulimit -n)"
+ echo $ETC_SECURITY_MSG
+fi
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/904b53df/install/python-installer/common.py
----------------------------------------------------------------------
diff --git a/install/python-installer/common.py b/install/python-installer/common.py
new file mode 100644
index 0000000..3713da8
--- /dev/null
+++ b/install/python-installer/common.py
@@ -0,0 +1,479 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### The common functions ###
+
+import os
+import pty
+import sys
+import json
+import re
+import time
+import base64
+import subprocess
+import logging
+try:
+ import xml.etree.cElementTree as ET
+except ImportError:
+ import xml.etree.ElementTree as ET
+from ConfigParser import ConfigParser
+from collections import defaultdict
+
+__VERSION__ = 'v1.0.0'
+INSTALLER_LOC = sys.path[0]
+
+USER_PROMPT_FILE = INSTALLER_LOC + '/prompt.json'
+SCRCFG_FILE = INSTALLER_LOC + '/script.json'
+VERSION_FILE = INSTALLER_LOC + '/version.json'
+MODCFG_FILE = INSTALLER_LOC + '/mod_cfgs.json'
+
+DBCFG_FILE = INSTALLER_LOC + '/db_config'
+DBCFG_TMP_FILE = INSTALLER_LOC + '/.db_config_temp'
+
+TMP_DIR = '/tmp/.install'
+MARK = '[ERR]'
+
+def version():
+ print 'Installer version: %s' % __VERSION__
+ exit(0)
+
+def ok(msg):
+ print '\n\33[32m***[OK]: %s \33[0m' % msg
+
+def info(msg):
+ print '\n\33[33m***[INFO]: %s \33[0m' % msg
+
+def err_m(msg):
+ """ used by main script """
+ sys.stderr.write('\n\33[31m***[ERROR]: %s \33[0m\n' % msg)
+ sys.exit(1)
+
+def err(msg):
+ """ used by sub script """
+ sys.stderr.write(MARK + msg)
+ sys.exit(1)
+
+def get_logger(log_file):
+
+ log_dir = os.path.dirname(log_file)
+ if not os.path.exists(log_dir): os.mkdir(log_dir)
+
+ logger = logging.getLogger()
+ logger.setLevel(logging.INFO)
+
+ formatter = logging.Formatter('[%(asctime)s %(levelname)s]: %(message)s')
+
+ fh = logging.FileHandler(log_file)
+ fh.setFormatter(formatter)
+
+ logger.addHandler(fh)
+
+ return logger
+
+def run_cmd(cmd):
+ """ check command return value and return stdout """
+ p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ stdout, stderr = p.communicate()
+ if p.returncode != 0:
+ err('Failed to run command %s: %s' % (cmd, stderr))
+ return stdout.strip()
+
+def run_cmd_as_user(user, cmd):
+ return run_cmd('sudo su - %s -c \'%s\'' % (user, cmd))
+
+def cmd_output(cmd):
+ """ return command output but not check return value """
+ p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ stdout, stderr = p.communicate()
+
+ return stdout.strip() if stdout else stderr
+
+def mod_file(template_file, change_items):
+ """
+ @params: change_items: a dict includes:
+ {regular_expression : replace_string}
+ """
+ try:
+ with open(template_file, 'r') as f:
+ lines = f.read()
+ except IOError:
+ err('Failed to open file %s to modify' % template_file)
+
+ for regexp, replace in change_items.iteritems():
+ lines = re.sub(regexp, replace, lines)
+
+ with open(template_file, 'w') as f:
+ f.write(lines)
+
+def append_file(template_file, string, position=''):
+ try:
+ with open(template_file, 'r') as f:
+ lines = f.readlines()
+ pos = 0
+ if position:
+ for index, line in enumerate(lines):
+ if position in line:
+ pos = index + 1
+
+ if pos == 0: pos = len(lines)
+ newlines = lines[:pos] + [string + '\n'] + lines[pos:]
+ if not string in lines:
+ with open(template_file, 'w') as f:
+ f.writelines(newlines)
+ except IOError:
+ err('Failed to open file %s to append' % template_file)
+
+
+def write_file(template_file, string):
+ try:
+ with open(template_file, 'w') as f:
+ f.write(string)
+ except IOError:
+ err('Failed to open file %s to write' % template_file)
+
+
+class Version(object):
+ def __init__(self):
+ self.support_ver = ParseJson(VERSION_FILE).load()
+
+ def get_version(self, component):
+ if self.support_ver[component] == '':
+ err('Failed to get version info for "%s" from config file' % component)
+
+ return self.support_ver[component]
+
+class Remote(object):
+ """
+ copy files to/fetch files from remote host using ssh
+ can also use paramiko, but it's not a build-in module
+ """
+
+ def __init__(self, host, user='', pwd=''):
+ self.host = host
+ self.user = user
+ self.rc = 0
+ self.pwd = pwd
+ self.sshpass = self._sshpass_available()
+
+ @staticmethod
+ def _sshpass_available():
+ sshpass_available = True
+ try:
+ p = subprocess.Popen(['sshpass'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ p.communicate()
+ except OSError:
+ sshpass_available = False
+
+ return sshpass_available
+
+ def _commands(self, method):
+ cmd = []
+ if self.sshpass and self.pwd: cmd = ['sshpass', '-p', self.pwd]
+ cmd += [method]
+ if not (self.sshpass and self.pwd): cmd += ['-oPasswordAuthentication=no']
+ return cmd
+
+ def _execute(self, cmd, verbose=False, shell=False):
+ try:
+ if verbose: print 'cmd:', cmd
+
+ master, slave = pty.openpty()
+ if shell:
+ p = subprocess.Popen(cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ else:
+ p = subprocess.Popen(cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+ self.stdout, self.stderr = p.communicate()
+ if p.returncode:
+ self.rc = p.returncode
+ # 'ssh -tt' will overwrite stderr, so manually handle it
+ if MARK in self.stdout:
+ self.stdout, self.stderr = self.stdout.split(MARK)
+ else:
+ self.stderr = self.stdout
+ except Exception as e:
+ err_m('Failed to run commands on remote host: %s' % e)
+
+ def execute(self, user_cmd):
+ cmd = self._commands('ssh')
+ if self.user:
+ cmd += ['%s@%s' % (self.user, self.host)]
+ else:
+ cmd += [self.host]
+
+ cmd += user_cmd.split()
+ self._execute(cmd)
+
+ def copy(self, files, remote_folder='.'):
+ """ copy file to user's home folder """
+ for f in files:
+ if not os.path.exists(f):
+ err_m('Copy file error: %s doesn\'t exist' % f)
+
+ cmd = self._commands('scp')
+ cmd += ['-r']
+ cmd += files # files should be full path
+ if self.user:
+ cmd += ['%s@%s:%s/' % (self.user, self.host, remote_folder)]
+ else:
+ cmd += ['%s:%s/' % (self.host, remote_folder)]
+
+ self._execute(cmd)
+ if self.rc != 0: err('Failed to copy files to remote nodes')
+
+ def fetch(self, files, local_folder='.'):
+ """ fetch file from user's home folder """
+ cmd = self._commands('scp')
+ cmd += ['-r']
+ if self.user:
+ cmd += ['%s@%s:~/{%s}' % (self.user, self.host, ','.join(files))]
+ else:
+ cmd += ['%s:~/{%s}' % (self.host, ','.join(files))]
+ cmd += [local_folder]
+
+ self._execute(cmd)
+ if self.rc != 0: err('Failed to fetch files from remote nodes')
+
+
+class ParseHttp(object):
+ def __init__(self, user, passwd, json_type=True):
+ # httplib2 is not installed by default
+ try:
+ import httplib2
+ except ImportError:
+ err_m('Python module httplib2 is not found. Install python-httplib2 first.')
+
+ self.user = user
+ self.passwd = passwd
+ self.h = httplib2.Http(disable_ssl_certificate_validation=True)
+ self.h.add_credentials(self.user, self.passwd)
+ self.headers = {}
+ self.headers['X-Requested-By'] = 'trafodion'
+ if json_type:
+ self.headers['Content-Type'] = 'application/json'
+ self.headers['Authorization'] = 'Basic %s' % (base64.b64encode('%s:%s' % (self.user, self.passwd)))
+
+ def _request(self, url, method, body=None):
+ try:
+ resp, content = self.h.request(url, method, headers=self.headers, body=body)
+ # return code is not 2xx
+ if not 200 <= resp.status < 300:
+ err_m('Error return code {0} when {1}ting configs'.format(resp.status, method.lower()))
+ return content
+ except Exception as exc:
+ err_m('Error with {0}ting configs using URL {1}. Reason: {2}'.format(method.lower(), url, exc))
+
+ def get(self, url):
+ try:
+ return defaultdict(str, json.loads(self._request(url, 'GET')))
+ except ValueError:
+ err_m('Failed to get data from URL, check password if URL requires authentication')
+
+ def put(self, url, config):
+ if not isinstance(config, dict): err_m('Wrong HTTP PUT parameter, should be a dict')
+ result = self._request(url, 'PUT', body=json.dumps(config))
+ if result: return defaultdict(str, json.loads(result))
+
+ def post(self, url):
+ try:
+ return defaultdict(str, json.loads(self._request(url, 'POST')))
+ except ValueError:
+ err_m('Failed to send command to URL')
+
+
+class ParseXML(object):
+ """ handle *-site.xml with format
+ <property><name></name><value></value></proerty>
+ """
+ def __init__(self, xml_file):
+ self.__xml_file = xml_file
+ if not os.path.exists(self.__xml_file): err_m('Cannot find xml file %s' % self.__xml_file)
+ try:
+ self._tree = ET.parse(self.__xml_file)
+ except Exception as e:
+ err_m('failed to parsing xml: %s' % e)
+
+ self._root = self._tree.getroot()
+ self._properties = self._root.findall('property')
+ # name, value list
+ self._nvlist = [[elem.text for elem in p] for p in self._properties]
+
+ def __indent(self, elem):
+ """Return a pretty-printed XML string for the Element."""
+ if len(elem):
+ if not elem.text: elem.text = '\n' + ' '
+ if not elem.tail: elem.tail = '\n'
+ for subelem in elem:
+ self.__indent(subelem)
+ else:
+ if not elem.tail: elem.tail = '\n' + ' '
+
+ def get_property(self, name):
+ try:
+ return [x[1] for x in self._nvlist if x[0] == name][0]
+ except:
+ return ''
+
+ def rm_property(self, name):
+ for p in self._properties:
+ if p[0].text == name:
+ self._root.remove(p)
+
+ def add_property(self, name, value):
+ # don't add property if already exists
+ if self.get_property(name): return
+
+ elem_p = ET.Element('property')
+ elem_name = ET.Element('name')
+ elem_value = ET.Element('value')
+
+ elem_name.text = name
+ elem_value.text = value
+ elem_p.append(elem_name)
+ elem_p.append(elem_value)
+
+ self._nvlist.append([name, value])
+ self._root.append(elem_p)
+
+ def write_xml(self):
+ self.__indent(self._root)
+ self._tree.write(self.__xml_file)
+
+ def print_xml(self):
+ for name, value in self._nvlist:
+ print name, value
+
+class ParseJson(object):
+ def __init__(self, js_file):
+ self.__js_file = js_file
+
+ def load(self):
+ """ load json file to a dict """
+ if not os.path.exists(self.__js_file): err_m('Cannot find json file %s' % self.__js_file)
+ with open(self.__js_file, 'r') as f:
+ tmparray = f.readlines()
+ content = ''
+ for t in tmparray:
+ content += t
+
+ try:
+ return defaultdict(str, json.loads(content))
+ except ValueError:
+ err_m('No json format found in config file %s' % self.__js_file)
+
+ def save(self, dic):
+ """ save dict to json file with pretty format """
+ with open(self.__js_file, 'w') as f:
+ f.write(json.dumps(dic, indent=4))
+ return 0
+
+
+class ParseInI(object):
+ def __init__(self, ini_file):
+ self.__ini_file = ini_file
+ self.section = 'def'
+
+ def load(self):
+ """ load content from ini file and return a dict """
+ if not os.path.exists(self.__ini_file):
+ err_m('Cannot find ini file %s' % self.__ini_file)
+
+ cfgs = {}
+ cf = ConfigParser()
+ cf.read(self.__ini_file)
+
+ if not cf.has_section(self.section):
+ err_m('Cannot find the default section [%s]' % self.section)
+
+ for cfg in cf.items(self.section):
+ cfgs[cfg[0]] = cfg[1]
+
+ return defaultdict(str, cfgs)
+
+ def save(self, dic):
+ """ save a dict as an ini file """
+ cf = ConfigParser()
+ cf.add_section(self.section)
+ for key, value in dic.iteritems():
+ cf.set(self.section, key, value)
+
+ with open(self.__ini_file, 'w') as f:
+ cf.write(f)
+
+def http_start(repo_dir, repo_port):
+ info('Starting temporary python http server')
+ os.system("cd %s; python -m SimpleHTTPServer %s > /dev/null 2>&1 &" % (repo_dir, repo_port))
+
+def http_stop():
+ #info('Stopping temporary python http server')
+ os.system("ps -ef|grep SimpleHTTPServer |grep -v grep | awk '{print $2}' |xargs kill -9 >/dev/null 2>&1")
+
+def format_output(text):
+ num = len(text) + 4
+ print '*' * num
+ print ' ' + text
+ print '*' * num
+
+def expNumRe(text):
+ """
+ expand numeric regular expression to list
+ e.g. 'n[01-03],n1[0-1]': ['n01','n02','n03','n10','n11']
+ e.g. 'n[09-11].com': ['n09.com','n10.com','n11.com']
+ """
+ explist = []
+ for regex in text.split(','):
+ regex = regex.strip()
+ r = re.match(r'(.*)\[(\d+)-(\d+)\](.*)', regex)
+ if r:
+ h = r.group(1)
+ d1 = r.group(2)
+ d2 = r.group(3)
+ t = r.group(4)
+
+ convert = lambda d: str(('%0' + str(min(len(d1), len(d2))) + 'd') % d)
+ if d1 > d2: d1, d2 = d2, d1
+ explist.extend([h + convert(c) + t for c in range(int(d1), int(d2)+1)])
+
+ else:
+ # keep original value if not matched
+ explist.append(regex)
+
+ return explist
+
+def time_elapse(func):
+ """ time elapse decorator """
+ def wrapper(*args, **kwargs):
+ start_time = time.time()
+ output = func(*args, **kwargs)
+ end_time = time.time()
+ seconds = end_time - start_time
+ hours = seconds / 3600
+ seconds = seconds % 3600
+ minutes = seconds / 60
+ seconds = seconds % 60
+ print '\nTime Cost: %d hour(s) %d minute(s) %d second(s)' % (hours, minutes, seconds)
+ return output
+ return wrapper
+
+if __name__ == '__main__':
+ exit(0)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/904b53df/install/python-installer/copy_files.py
----------------------------------------------------------------------
diff --git a/install/python-installer/copy_files.py b/install/python-installer/copy_files.py
new file mode 100755
index 0000000..73d0237
--- /dev/null
+++ b/install/python-installer/copy_files.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on local node ###
+
+import sys
+import json
+from threading import Thread
+from common import Remote, run_cmd, err
+
+def run(pwd):
+ """ gen ssh key on local and copy to all nodes
+ copy traf package file from local to all nodes
+ """
+ dbcfgs = json.loads(dbcfgs_json)
+ hosts = dbcfgs['node_list'].split(',')
+ traf_package = dbcfgs['traf_package']
+
+ key_file = '/tmp/id_rsa'
+ run_cmd('sudo rm -rf %s*' % key_file)
+ run_cmd('sudo echo -e "y" | ssh-keygen -t rsa -N "" -f %s' % key_file)
+
+ files = [key_file, key_file+'.pub', traf_package]
+
+ remote_insts = [Remote(h, pwd=pwd) for h in hosts]
+ threads = [Thread(target=r.copy, args=(files, '/tmp')) for r in remote_insts]
+ for thread in threads: thread.start()
+ for thread in threads: thread.join()
+ for r in remote_insts:
+ if r.rc != 0: err('Failed to copy files to %s' % r.host)
+
+
+# main
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+
+try:
+ pwd = sys.argv[2]
+except IndexError:
+ pwd = ''
+
+run(pwd)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/904b53df/install/python-installer/db_config_default
----------------------------------------------------------------------
diff --git a/install/python-installer/db_config_default b/install/python-installer/db_config_default
new file mode 100644
index 0000000..c4a2c4e
--- /dev/null
+++ b/install/python-installer/db_config_default
@@ -0,0 +1,120 @@
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+[def]
+# NOTICE: if you are using CDH/HDP hadoop distro,
+# you can only specifiy management url address for a quick install
+
+##################################
+# Common Settings
+##################################
+
+# trafodion username and password
+traf_user = trafodion
+traf_pwd = traf123
+
+# trafodion used java(JDK) path on trafodion nodes
+# if not provided, installer will auto detect installed JDK
+java_home =
+
+# cloudera/ambari management url(i.e. http://192.168.0.1:7180 or just 192.168.0.1)
+# if 'http' or 'https' prefix is not provided, the default one is 'http'
+# if port is not provided, the default port is cloudera port '7180'
+mgr_url =
+# user name for cloudera/ambari management url
+mgr_user = admin
+# password for cloudera/ambari management url
+mgr_pwd = admin
+# set the cluster number if multiple clusters managed by one Cloudera manager
+# ignore it if only one cluster being managed
+cluster_no = 1
+
+# trafodion tar package file location
+# no need to provide it if the package can be found in current installer's directory
+traf_package =
+
+# the number of dcs servers on each node
+dcs_cnt_per_node = 4
+
+# scratch file location, seperated by comma if more than one
+scratch_locs = $MY_SQROOT/tmp
+
+# start trafodion instance after installation completed
+traf_start = Y
+
+
+##################################
+# DCS HA configuration
+##################################
+
+# set it to 'Y' if enable DCS HA
+dcs_ha = N
+# if HA is enabled, provide floating ip, network interface and the hostname of backup dcs master nodes
+dcs_floating_ip =
+# network interface that dcs used
+dcs_interface =
+# backup dcs master nodes, seperated by comma if more than one
+dcs_backup_nodes =
+
+
+##################################
+# Offline installation setting
+##################################
+
+# set offline mode to Y if no internet connection
+offline_mode = N
+# if offline mode is set, you must provide a local repository directory with all needed RPMs
+local_repo_dir =
+
+
+##################################
+# LDAP security configuration
+##################################
+
+# set it to 'Y' if enable LDAP security
+ldap_security = N
+# LDAP user name and password to be assigned as DB admin privilege
+db_admin_user = admin
+db_admin_pwd = traf123
+# LDAP user to be assigned DB root privileges (DB__ROOT)
+db_root_user = trafodion
+# if LDAP security is enabled, provide the following items
+ldap_hosts =
+# 389 for no encryption or TLS, 636 for SSL
+ldap_port = 389
+ldap_identifiers =
+ldap_encrypt = 0
+ldap_certpath =
+# provide if have
+ladp_user =
+ladp_pwd =
+
+##################################
+# Kerberos security configuration
+##################################
+# if kerberos is enabled in your hadoop system, provide below info
+
+# KDC server address
+kdc_server =
+# include realm, i.e. admin/admin@EXAMPLE.COM
+admin_principal =
+# admin password for admin principal, it is used to create trafodion user's principal and keytab
+kdcadmin_pwd =
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/904b53df/install/python-installer/db_install.py
----------------------------------------------------------------------
diff --git a/install/python-installer/db_install.py b/install/python-installer/db_install.py
new file mode 100755
index 0000000..9d88f8d
--- /dev/null
+++ b/install/python-installer/db_install.py
@@ -0,0 +1,643 @@
+#!/usr/bin/env python
+# -*- coding: utf8 -*-
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+import os
+import re
+import socket
+import json
+import getpass
+import time
+import sys
+reload(sys)
+sys.setdefaultencoding("utf-8")
+from optparse import OptionParser
+from glob import glob
+from collections import defaultdict
+import wrapper
+try:
+ from prettytable import PrettyTable
+except ImportError:
+ print 'Python module prettytable is not found. Install python-prettytable first.'
+ exit(1)
+from common import *
+
+# init global cfgs for user input
+cfgs = defaultdict(str)
+
+class HadoopDiscover(object):
+ """ discover for hadoop related info """
+ def __init__(self, user, pwd, url, cluster_name):
+ self.rsnodes = []
+ self.users = {}
+ self.cluster_name = cluster_name
+ self.hg = ParseHttp(user, pwd)
+ self.v1_url = '%s/api/v1/clusters' % url
+ self.v6_url = '%s/api/v6/clusters' % url
+ self.cluster_url = '%s/%s' % (self.v1_url, cluster_name.replace(' ', '%20'))
+ self._get_distro()
+ self._check_version()
+
+ def _get_distro(self):
+ content = self.hg.get(self.v1_url)
+
+ if content['items'][0].has_key('name'):
+ # use v6 rest api for CDH to get fullversion
+ content = self.hg.get(self.v6_url)
+
+ # loop all managed clusters
+ for cluster in content['items']:
+ try:
+ # HDP
+ self.distro = cluster['Clusters']['version']
+ except KeyError:
+ # CDH
+ try:
+ if self.cluster_name == cluster['displayName']:
+ self.distro = 'CDH' + cluster['fullVersion']
+ break
+ except KeyError:
+ log_err('Failed to get hadoop distribution info from management url')
+
+ def get_hdfs_srvname(self):
+ return self._get_service_name('HDFS')
+
+ def get_hbase_srvname(self):
+ return self._get_service_name('HBASE')
+
+ def get_zookeeper_srvname(self):
+ return self._get_service_name('ZOOKEEPER')
+
+ def _get_service_name(self, service):
+ # CDH uses different service names in multiple clusters
+ if 'CDH' in self.distro:
+ services_cfgs = self.hg.get(self.cluster_url +'/services')
+ for item in services_cfgs['items']:
+ if item['type'] == service:
+ return item['name']
+ else:
+ return service.lower()
+
+ def _check_version(self):
+ version = Version()
+ if 'CDH' in self.distro: version_list = version.get_version('cdh')
+ if 'HDP' in self.distro: version_list = version.get_version('hdp')
+
+ has_version = 0
+ for ver in version_list:
+ if ver in self.distro: has_version = 1
+
+ if not has_version:
+ log_err('Sorry, currently EsgynDB doesn\'t support %s version' % self.distro)
+
+ def get_hadoop_users(self):
+ if 'CDH' in self.distro:
+ self._get_cdh_users()
+ elif 'HDP' in self.distro or 'BigInsights' in self.distro:
+ self._get_hdp_users()
+ return self.users
+
+ def _get_hdp_users(self):
+ desired_cfg = self.hg.get('%s/?fields=Clusters/desired_configs' % (self.cluster_url))
+ config_type = {'hbase-env':'hbase_user', 'hadoop-env':'hdfs_user'}
+ for key, value in config_type.items():
+ desired_tag = desired_cfg['Clusters']['desired_configs'][key]['tag']
+ current_cfg = self.hg.get('%s/configurations?type=%s&tag=%s' % (self.cluster_url, key, desired_tag))
+ self.users[value] = current_cfg['items'][0]['properties'][value]
+
+ def _get_cdh_users(self):
+ def _get_username(service_name, hadoop_type):
+ cfg = self.hg.get('%s/services/%s/config' % (self.cluster_url, service_name))
+ if cfg.has_key('items'):
+ for item in cfg['items']:
+ if item['name'] == 'process_username':
+ return item['value']
+ return hadoop_type
+
+ hdfs_user = _get_username(self.get_hdfs_srvname(), 'hdfs')
+ hbase_user = _get_username(self.get_hbase_srvname(), 'hbase')
+
+ self.users = {'hbase_user':hbase_user, 'hdfs_user':hdfs_user}
+
+ def get_rsnodes(self):
+ if 'CDH' in self.distro:
+ self._get_rsnodes_cdh()
+ elif 'HDP' in self.distro or 'BigInsights' in self.distro:
+ self._get_rsnodes_hdp()
+
+ self.rsnodes.sort()
+ # use short hostname
+ try:
+ self.rsnodes = [re.match(r'([\w\-]+).*', node).group(1) for node in self.rsnodes]
+
+ except AttributeError:
+ pass
+ return self.rsnodes
+
+ def _get_rsnodes_cdh(self):
+ """ get list of HBase RegionServer nodes in CDH """
+ cm = self.hg.get('%s/api/v6/cm/deployment' % cfgs['mgr_url'])
+
+ hostids = []
+ for c in cm['clusters']:
+ if c['displayName'] == self.cluster_name:
+ for s in c['services']:
+ if s['type'] == 'HBASE':
+ for r in s['roles']:
+ if r['type'] == 'REGIONSERVER': hostids.append(r['hostRef']['hostId'])
+ for i in hostids:
+ for h in cm['hosts']:
+ if i == h['hostId']: self.rsnodes.append(h['hostname'])
+
+ def _get_rsnodes_hdp(self):
+ """ get list of HBase RegionServer nodes in HDP """
+ hdp = self.hg.get('%s/services/HBASE/components/HBASE_REGIONSERVER' % self.cluster_url)
+ self.rsnodes = [c['HostRoles']['host_name'] for c in hdp['host_components']]
+
+
+class UserInput(object):
+ def __init__(self, options, pwd):
+ self.in_data = ParseJson(USER_PROMPT_FILE).load()
+ self.pwd = pwd
+
+ def _basic_check(self, name, answer):
+ isYN = self.in_data[name].has_key('isYN')
+ isdigit = self.in_data[name].has_key('isdigit')
+ isexist = self.in_data[name].has_key('isexist')
+ isremote_exist = self.in_data[name].has_key('isremote_exist')
+ isIP = self.in_data[name].has_key('isIP')
+ isuser = self.in_data[name].has_key('isuser')
+
+ # check answer value basicly
+ answer = answer.rstrip()
+ if answer:
+ if isYN:
+ answer = answer.upper()
+ if answer != 'Y' and answer != 'N':
+ log_err('Invalid parameter for %s, should be \'Y|y|N|n\'' % name)
+ elif isdigit:
+ if not answer.isdigit():
+ log_err('Invalid parameter for %s, should be a number' % name)
+ elif isexist:
+ if not os.path.exists(answer):
+ log_err('%s path \'%s\' doesn\'t exist' % (name, answer))
+ elif isremote_exist:
+ hosts = cfgs['node_list'].split(',')
+ remotes = [Remote(host, pwd=self.pwd) for host in hosts]
+
+ nodes = ''
+ for remote in remotes:
+ # check if directory exists on remote host
+ remote.execute('ls %s 2>&1 >/dev/null' % answer)
+ if remote.rc != 0:
+ nodes += ' ' + remote.host
+ if nodes:
+ log_err('%s path \'%s\' doesn\'t exist on node(s) \'%s\'' % (name, answer, nodes))
+ elif isIP:
+ try:
+ socket.inet_pton(socket.AF_INET, answer)
+ except:
+ log_err('Invalid IP address \'%s\'' % answer)
+ elif isuser:
+ if re.match(r'\w+', answer).group() != answer:
+ log_err('Invalid user name \'%s\'' % answer)
+
+ else:
+ log_err('Empty value for \'%s\'' % name)
+
+ def _handle_prompt(self, name, user_defined):
+ prompt = self.in_data[name]['prompt']
+ default = user_defined
+
+ if (not default) and self.in_data[name].has_key('default'):
+ default = self.in_data[name]['default']
+
+ ispasswd = self.in_data[name].has_key('ispasswd')
+ isYN = self.in_data[name].has_key('isYN')
+
+ # no default value for password
+ if ispasswd: default = ''
+
+ if isYN:
+ prompt = prompt + ' (Y/N) '
+
+ if default:
+ prompt = prompt + ' [' + default + ']: '
+ else:
+ prompt = prompt + ': '
+
+ # no default value for password
+ if ispasswd:
+ orig = getpass.getpass(prompt)
+ confirm = getpass.getpass('Confirm ' + prompt)
+ if orig == confirm:
+ answer = confirm
+ else:
+ log_err('Password mismatch')
+ else:
+ try:
+ answer = raw_input(prompt)
+ except UnicodeEncodeError:
+ log_err('Character Encode error, check user input')
+ if not answer and default: answer = default
+
+ return answer
+
+ def get_input(self, name, user_defined='', prompt_mode=True):
+ if self.in_data.has_key(name):
+ if prompt_mode:
+ # save configs to global dict
+ cfgs[name] = self._handle_prompt(name, user_defined)
+
+ # check basic values from global configs
+ self._basic_check(name, cfgs[name])
+ else:
+ # should not go to here, just in case
+ log_err('Invalid prompt')
+
+ def get_confirm(self):
+ answer = raw_input('Confirm result (Y/N) [N]: ')
+ if not answer: answer = 'N'
+
+ answer = answer.upper()
+ if answer != 'Y' and answer != 'N':
+ log_err('Invalid parameter, should be \'Y|y|N|n\'')
+ return answer
+
+ def notify_user(self):
+ """ show the final configs to user """
+ format_output('Final Configs')
+ title = ['config type', 'value']
+ pt = PrettyTable(title)
+ for item in title:
+ pt.align[item] = 'l'
+
+ for key, value in sorted(cfgs.items()):
+ if self.in_data.has_key(key) and value:
+ if self.in_data[key].has_key('ispasswd'): continue
+ pt.add_row([key, value])
+ print pt
+ confirm = self.get_confirm()
+ if confirm != 'Y':
+ if os.path.exists(DBCFG_FILE): os.remove(DBCFG_FILE)
+ log_err('User quit')
+
+
+def log_err(errtext):
+ # save tmp config files
+ tp = ParseInI(DBCFG_TMP_FILE)
+ tp.save(cfgs)
+ err_m(errtext)
+
+
+def user_input(options, prompt_mode=True, pwd=''):
+ """ get user's input and check input value """
+ global cfgs
+
+ apache = True if hasattr(options, 'apache') and options.apache else False
+ offline = True if hasattr(options, 'offline') and options.offline else False
+ silent = True if hasattr(options, 'silent') and options.silent else False
+
+ # load from temp config file if in prompt mode
+ if os.path.exists(DBCFG_TMP_FILE) and prompt_mode == True:
+ tp = ParseInI(DBCFG_TMP_FILE)
+ cfgs = tp.load()
+
+ u = UserInput(options, pwd)
+ g = lambda n: u.get_input(n, cfgs[n], prompt_mode=prompt_mode)
+
+ ### begin user input ###
+ if apache:
+ g('node_list')
+ node_lists = expNumRe(cfgs['node_list'])
+
+ # check if node list is expanded successfully
+ if len([1 for node in node_lists if '[' in node]):
+ log_err('Failed to expand node list, please check your input.')
+ cfgs['node_list'] = ','.join(node_lists)
+ g('hadoop_home')
+ g('hbase_home')
+ g('hive_home')
+ g('hdfs_user')
+ g('hbase_user')
+ g('first_rsnode')
+ cfgs['distro'] = 'APACHE'
+ else:
+ g('mgr_url')
+ if not ('http:' in cfgs['mgr_url'] or 'https:' in cfgs['mgr_url']):
+ cfgs['mgr_url'] = 'http://' + cfgs['mgr_url']
+
+ # set cloudera default port 7180 if not provided by user
+ if not re.search(r':\d+', cfgs['mgr_url']):
+ cfgs['mgr_url'] += ':7180'
+
+ g('mgr_user')
+ g('mgr_pwd')
+
+ validate_url_v1 = '%s/api/v1/clusters' % cfgs['mgr_url']
+ content = ParseHttp(cfgs['mgr_user'], cfgs['mgr_pwd']).get(validate_url_v1)
+
+ # currently only CDH support multiple clusters
+ # so if condition is true, it must be CDH cluster
+ if len(content['items']) > 1:
+ cluster_names = []
+ # loop all managed clusters
+ for cluster in content['items']:
+ cluster_names.append(cluster['name'])
+
+ for index, name in enumerate(cluster_names):
+ print str(index + 1) + '. ' + name
+ g('cluster_no')
+ c_index = int(cfgs['cluster_no']) - 1
+ if c_index < 0 or c_index >= len(cluster_names):
+ log_err('Incorrect number')
+ cluster_name = cluster_names[int(c_index)]
+ else:
+ try:
+ cluster_name = content['items'][0]['name']
+ except (IndexError, KeyError):
+ cluster_name = content['items'][0]['Clusters']['cluster_name']
+
+ discover = HadoopDiscover(cfgs['mgr_user'], cfgs['mgr_pwd'], cfgs['mgr_url'], cluster_name)
+ rsnodes = discover.get_rsnodes()
+ hadoop_users = discover.get_hadoop_users()
+
+ cfgs['distro'] = discover.distro
+ cfgs['hbase_service_name'] = discover.get_hbase_srvname()
+ cfgs['hdfs_service_name'] = discover.get_hdfs_srvname()
+ cfgs['zookeeper_service_name'] = discover.get_zookeeper_srvname()
+
+ cfgs['cluster_name'] = cluster_name.replace(' ', '%20')
+ cfgs['hdfs_user'] = hadoop_users['hdfs_user']
+ cfgs['hbase_user'] = hadoop_users['hbase_user']
+ cfgs['node_list'] = ','.join(rsnodes)
+ cfgs['first_rsnode'] = rsnodes[0] # first regionserver node
+
+ # check node connection
+ for node in cfgs['node_list'].split(','):
+ rc = os.system('ping -c 1 %s >/dev/null 2>&1' % node)
+ if rc: log_err('Cannot ping %s, please check network connection and /etc/hosts' % node)
+
+ ### discover system settings, return a dict
+ discover_results = wrapper.run(cfgs, options, mode='discover', pwd=pwd)
+
+ # check discover results, return error if fails on any sinlge node
+ need_java_home = 0
+ for result in discover_results:
+ host, content = result.items()[0]
+ content_dict = json.loads(content)
+
+ java_home = content_dict['default_java']
+ if java_home == 'N/A':
+ need_java_home += 1
+ if content_dict['linux'] == 'N/A':
+ log_err('Unsupported Linux version')
+ if content_dict['firewall_status'] == 'Running':
+ log_err('Firewall should be stopped')
+ if content_dict['traf_status'] == 'Running':
+ log_err('Trafodion process is found, please stop it first')
+ if content_dict['hbase'] == 'N/A':
+ log_err('HBase is not found')
+ if content_dict['hbase'] == 'N/S':
+ log_err('HBase version is not supported')
+
+ if content_dict['secure_hadoop'] == 'kerberos':
+ cfgs['secure_hadoop'] = 'Y'
+ else:
+ cfgs['secure_hadoop'] = 'N'
+
+ if offline:
+ g('local_repo_dir')
+ if not glob('%s/repodata' % cfgs['local_repo_dir']):
+ log_err('repodata directory not found, this is not a valid repository directory')
+ cfgs['offline_mode'] = 'Y'
+ cfgs['repo_ip'] = socket.gethostbyname(socket.gethostname())
+ cfgs['repo_port'] = '9900'
+
+ pkg_list = ['apache-trafodion']
+ # find tar in installer folder, if more than one found, use the first one
+ for pkg in pkg_list:
+ tar_loc = glob('%s/*%s*.tar.gz' % (INSTALLER_LOC, pkg))
+ if tar_loc:
+ cfgs['traf_package'] = tar_loc[0]
+ break
+
+ g('traf_package')
+
+ # get basename and version from tar filename
+ try:
+ pattern = '|'.join(pkg_list)
+ cfgs['traf_basename'], cfgs['traf_version'] = re.search(r'.*(%s).*-(\d\.\d\.\d).*' % pattern, cfgs['traf_package']).groups()
+ except:
+ log_err('Invalid package tar file')
+
+ #if float(cfgs['traf_version'][:3]) >= 2.2:
+ # cfgs['req_java8'] = 'Y'
+ #else:
+ # cfgs['req_java8'] = 'N'
+
+ g('traf_pwd')
+ g('dcs_cnt_per_node')
+ g('scratch_locs')
+ g('traf_start')
+
+ # kerberos
+ if cfgs['secure_hadoop'].upper() == 'Y':
+ g('kdc_server')
+ g('admin_principal')
+ g('kdcadmin_pwd')
+
+ # ldap security
+ g('ldap_security')
+ if cfgs['ldap_security'].upper() == 'Y':
+ g('db_root_user')
+ g('db_admin_user')
+ g('db_admin_pwd')
+ g('ldap_hosts')
+ g('ldap_port')
+ g('ldap_identifiers')
+ g('ldap_encrypt')
+ if cfgs['ldap_encrypt'] == '1' or cfgs['ldap_encrypt'] == '2':
+ g('ldap_certpath')
+ elif cfgs['ldap_encrypt'] == '0':
+ cfgs['ldap_certpath'] = ''
+ else:
+ log_err('Invalid ldap encryption level')
+
+ g('ldap_userinfo')
+ if cfgs['ldap_userinfo'] == 'Y':
+ g('ldap_user')
+ g('ldap_pwd')
+ else:
+ cfgs['ldap_user'] = ''
+ cfgs['ldap_pwd'] = ''
+
+ # DCS HA
+ g('dcs_ha')
+ if cfgs['dcs_ha'].upper() == 'Y':
+ g('dcs_floating_ip')
+ g('dcs_interface')
+ g('dcs_backup_nodes')
+ # check dcs backup nodes should exist in node list
+ if sorted(list(set((cfgs['dcs_backup_nodes'] + ',' + cfgs['node_list']).split(',')))) != sorted(cfgs['node_list'].split(',')):
+ log_err('Invalid DCS backup nodes, please pick up from node list')
+
+ if need_java_home:
+ g('java_home')
+ else:
+ # don't overwrite user input java home
+ if not cfgs['java_home']:
+ cfgs['java_home'] = java_home
+
+ # set other config to cfgs
+ if apache:
+ cfgs['hbase_xml_file'] = cfgs['hbase_home'] + '/conf/hbase-site.xml'
+ cfgs['hdfs_xml_file'] = cfgs['hadoop_home'] + '/etc/hadoop/hdfs-site.xml'
+ else:
+ cfgs['hbase_xml_file'] = '/etc/hbase/conf/hbase-site.xml'
+
+ cfgs['req_java8'] = 'N'
+ cfgs['traf_user'] = 'trafodion'
+ cfgs['config_created_date'] = time.strftime('%Y/%m/%d %H:%M %Z')
+
+ if not silent:
+ u.notify_user()
+
+def get_options():
+ usage = 'usage: %prog [options]\n'
+ usage += ' Trafodion install main script.'
+ parser = OptionParser(usage=usage)
+ parser.add_option("-c", "--config-file", dest="cfgfile", metavar="FILE",
+ help="Json format file. If provided, all install prompts \
+ will be taken from this file and not prompted for.")
+ parser.add_option("-u", "--remote-user", dest="user", metavar="USER",
+ help="Specify ssh login user for remote server, \
+ if not provided, use current login user as default.")
+ parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
+ help="Verbose mode, will print commands.")
+ parser.add_option("--silent", action="store_true", dest="silent", default=False,
+ help="Do not ask user to confirm configuration result")
+ parser.add_option("--enable-pwd", action="store_true", dest="pwd", default=False,
+ help="Prompt SSH login password for remote hosts. \
+ If set, \'sshpass\' tool is required.")
+ parser.add_option("--build", action="store_true", dest="build", default=False,
+ help="Build the config file in guided mode only.")
+ parser.add_option("--upgrade", action="store_true", dest="upgrade", default=False,
+ help="Upgrade install, it is useful when reinstalling Trafodion.")
+ parser.add_option("--apache-hadoop", action="store_true", dest="apache", default=False,
+ help="Install Trafodion on top of Apache Hadoop.")
+ parser.add_option("--offline", action="store_true", dest="offline", default=False,
+ help="Enable local repository for offline installing Trafodion.")
+ parser.add_option("--version", action="store_true", dest="version", default=False,
+ help="Show the installer version.")
+
+ (options, args) = parser.parse_args()
+ return options
+
+def main():
+ """ db_installer main loop """
+ global cfgs
+ format_output('Trafodion Installation ToolKit')
+
+ # handle parser option
+ options = get_options()
+
+ if options.version: version()
+
+ if options.build and options.cfgfile:
+ log_err('Wrong parameter, cannot specify both --build and --config-file')
+
+ if options.build and options.offline:
+ log_err('Wrong parameter, cannot specify both --build and --offline')
+
+ if options.cfgfile:
+ if not os.path.exists(options.cfgfile):
+ log_err('Cannot find config file \'%s\'' % options.cfgfile)
+ config_file = options.cfgfile
+ else:
+ config_file = DBCFG_FILE
+
+ if options.pwd:
+ pwd = getpass.getpass('Input remote host SSH Password: ')
+ else:
+ pwd = ''
+
+ # not specified config file and default config file doesn't exist either
+ p = ParseInI(config_file)
+ if options.build or (not os.path.exists(config_file)):
+ if options.build: format_output('DryRun Start')
+ user_input(options, prompt_mode=True, pwd=pwd)
+
+ # save config file as json format
+ print '\n** Generating config file to save configs ... \n'
+ p.save(cfgs)
+ # config file exists
+ else:
+ print '\n** Loading configs from config file ... \n'
+ cfgs = p.load()
+ if options.offline and cfgs['offline_mode'] != 'Y':
+ log_err('To enable offline mode, must set "offline_mode = Y" in config file')
+ user_input(options, prompt_mode=False, pwd=pwd)
+
+ if options.upgrade:
+ cfgs['upgrade'] = 'Y'
+
+ if options.offline:
+ http_start(cfgs['local_repo_dir'], cfgs['repo_port'])
+ else:
+ cfgs['offline_mode'] = 'N'
+
+ if not options.build:
+ format_output('Installation Start')
+
+ ### perform actual installation ###
+ wrapper.run(cfgs, options, pwd=pwd)
+
+ format_output('Installation Complete')
+
+ if options.offline: http_stop()
+
+ # rename default config file when successfully installed
+ # so next time user can input new variables for a new install
+ # or specify the backup config file to install again
+ try:
+ # only rename default config file
+ ts = time.strftime('%y%m%d_%H%M')
+ if config_file == DBCFG_FILE and os.path.exists(config_file):
+ os.rename(config_file, config_file + '.bak' + ts)
+ except OSError:
+ log_err('Cannot rename config file')
+ else:
+ format_output('DryRun Complete')
+
+ # remove temp config file
+ if os.path.exists(DBCFG_TMP_FILE): os.remove(DBCFG_TMP_FILE)
+
+if __name__ == "__main__":
+ try:
+ main()
+ except (KeyboardInterrupt, EOFError):
+ tp = ParseInI(DBCFG_TMP_FILE)
+ tp.save(cfgs)
+ http_stop()
+ print '\nAborted...'
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/904b53df/install/python-installer/dcs_setup.py
----------------------------------------------------------------------
diff --git a/install/python-installer/dcs_setup.py b/install/python-installer/dcs_setup.py
new file mode 100755
index 0000000..818a933
--- /dev/null
+++ b/install/python-installer/dcs_setup.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on all nodes with trafodion user ###
+
+import os
+import sys
+import json
+from common import ParseXML, append_file, write_file, mod_file, cmd_output, run_cmd, err
+
+def run():
+ dbcfgs = json.loads(dbcfgs_json)
+
+ SQ_ROOT = os.environ['MY_SQROOT']
+ TRAF_VER = dbcfgs['traf_version']
+ HBASE_XML_FILE = dbcfgs['hbase_xml_file']
+
+ DCS_INSTALL_ENV = 'export DCS_INSTALL_DIR=%s/dcs-%s' % (SQ_ROOT, TRAF_VER)
+ REST_INSTALL_ENV = 'export REST_INSTALL_DIR=%s/rest-%s' % (SQ_ROOT, TRAF_VER)
+
+ DCS_CONF_DIR = '%s/dcs-%s/conf' % (SQ_ROOT, TRAF_VER)
+ DCS_SRV_FILE = DCS_CONF_DIR + '/servers'
+ DCS_MASTER_FILE = DCS_CONF_DIR + '/master'
+ DCS_BKMASTER_FILE = DCS_CONF_DIR + '/backup-masters'
+ DCS_ENV_FILE = DCS_CONF_DIR + '/dcs-env.sh'
+ DCS_SITE_FILE = DCS_CONF_DIR + '/dcs-site.xml'
+ REST_SITE_FILE = '%s/rest-%s/conf/rest-site.xml' % (SQ_ROOT, TRAF_VER)
+ TRAFCI_FILE = SQ_ROOT + '/trafci/bin/trafci'
+ SQENV_FILE = SQ_ROOT + '/sqenvcom.sh'
+
+ ### dcs setting ###
+ # servers
+ nodes = dbcfgs['node_list'].split(',')
+ dcs_cnt = dbcfgs['dcs_cnt_per_node']
+ dcs_servers = ''
+ for node in nodes:
+ dcs_servers += '%s %s\n' % (node, dcs_cnt)
+
+ write_file(DCS_SRV_FILE, dcs_servers)
+
+ ### modify dcs config files ###
+ # modify master
+ dcs_master = nodes[0]
+ append_file(DCS_MASTER_FILE, dcs_master)
+
+ # modify sqenvcom.sh
+ append_file(SQENV_FILE, DCS_INSTALL_ENV)
+ append_file(SQENV_FILE, REST_INSTALL_ENV)
+
+ # modify dcs-env.sh
+ mod_file(DCS_ENV_FILE, {'.*DCS_MANAGES_ZK=.*':'export DCS_MANAGES_ZK=false'})
+
+ # modify trafci
+ mod_file(TRAFCI_FILE, {'HNAME=.*':'HNAME=%s:23400' % dcs_master})
+
+ # modify dcs-site.xml
+ net_interface = cmd_output('netstat -rn | grep "^0.0.0.0" | awk \'{print $8}\'').strip()
+ hb = ParseXML(HBASE_XML_FILE)
+ zk_hosts = hb.get_property('hbase.zookeeper.quorum')
+ zk_port = hb.get_property('hbase.zookeeper.property.clientPort')
+
+ p = ParseXML(DCS_SITE_FILE)
+ p.add_property('dcs.zookeeper.property.clientPort', zk_port)
+ p.add_property('dcs.zookeeper.quorum', zk_hosts)
+ p.add_property('dcs.dns.interface', net_interface)
+
+ if dbcfgs['dcs_ha'] == 'Y':
+ dcs_floating_ip = dbcfgs['dcs_floating_ip']
+ dcs_backup_nodes = dbcfgs['dcs_backup_nodes']
+ p.add_property('dcs.master.floating.ip', 'true')
+ p.add_property('dcs.master.floating.ip.external.interface', net_interface)
+ p.add_property('dcs.master.floating.ip.external.ip.address', dcs_floating_ip)
+ p.rm_property('dcs.dns.interface')
+
+ # modify backup_master
+ write_file(DCS_BKMASTER_FILE, dcs_backup_nodes)
+
+ p.write_xml()
+
+ ### rest setting ###
+ p = ParseXML(REST_SITE_FILE)
+ p.add_property('rest.zookeeper.property.clientPort', zk_port)
+ p.add_property('rest.zookeeper.quorum', zk_hosts)
+ p.write_xml()
+
+ ### run sqcertgen ###
+ run_cmd('sqcertgen')
+
+# main
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+run()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/904b53df/install/python-installer/discovery.py
----------------------------------------------------------------------
diff --git a/install/python-installer/discovery.py b/install/python-installer/discovery.py
new file mode 100755
index 0000000..9f2d66e
--- /dev/null
+++ b/install/python-installer/discovery.py
@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+import os
+import time
+import json
+import getpass
+from optparse import OptionParser
+from collections import defaultdict
+try:
+ from prettytable import PrettyTable
+except ImportError:
+ print 'Python module prettytable is not found. Install python-prettytable first.'
+ exit(1)
+from common import err_m, err, ParseInI, expNumRe, format_output, DBCFG_FILE
+import wrapper
+
+
+def get_options():
+ usage = 'usage: %prog [options]\n'
+ usage += ' Trafodion install main script.'
+ parser = OptionParser(usage=usage)
+ parser.add_option("-c", "--config-file", dest="cfgfile", metavar="FILE",
+ help="Json format file. If provided, all install prompts \
+ will be taken from this file and not prompted for.")
+ parser.add_option("-u", "--remote-user", dest="user", metavar="USER",
+ help="Specify ssh login user for remote server, \
+ if not provided, use current login user as default.")
+ parser.add_option("--enable-pass", action="store_true", dest="pwd", default=True,
+ help="Not Prompt SSH login password for remote hosts.")
+
+ (options, args) = parser.parse_args()
+ return options
+
+# row format
+def output_row(results):
+ items = []
+ for result in results:
+ host, content = result.items()[0]
+ cfg_dict = json.loads(content)
+
+ cfg_tuples = sorted(cfg_dict.items())
+ title = ['Host']
+ item = [host]
+ for key, value in cfg_tuples:
+ title.append(key)
+ item.append(value)
+ items.append(item)
+
+ pt = PrettyTable(title)
+ for item in items: pt.add_row(item)
+
+ return str(pt)
+
+# column format
+def output_column(results):
+ items = []
+ for result in results:
+ host, content = result.items()[0]
+ cfg_dict = json.loads(content)
+
+ item = []
+ title = []
+ cfg_tuples = sorted(cfg_dict.items())
+ for key, value in cfg_tuples:
+ title.append(key)
+ item.append(value)
+ items.append([host, item])
+
+ pt = PrettyTable()
+ pt.add_column('Host', title)
+ for item in items:
+ pt.add_column(item[0], item[1])
+
+ return str(pt)
+
+def main():
+ options = get_options()
+
+ cfgs = defaultdict(str)
+
+ if options.cfgfile:
+ if not os.path.exists(options.cfgfile):
+ err_m('Cannot find config file \'%s\'' % options.cfgfile)
+ config_file = options.cfgfile
+ else:
+ config_file = DBCFG_FILE
+
+ if options.pwd:
+ pwd = getpass.getpass('Input remote host SSH Password: ')
+ else:
+ pwd = ''
+
+ if os.path.exists(config_file):
+ cfgs = ParseInI(config_file).load()
+ else:
+ node_lists = expNumRe(raw_input('Enter list of Nodes separated by comma, support numeric RE, i.e. n[01-12]: '))
+
+ # check if node list is expanded successfully
+ if len([1 for node in node_lists if '[' in node]):
+ err('Failed to expand node list, please check your input.')
+
+ cfgs['node_list'] = ','.join(node_lists)
+
+
+ results = wrapper.run(cfgs, options, mode='discover', pwd=pwd)
+
+ format_output('Discover results')
+
+ if len(results) > 4:
+ output = output_row(results)
+ else:
+ output = output_column(results)
+
+ print output
+ with open('discover_result', 'w') as f:
+ f.write('Discover Date: %s\n' % time.strftime('%Y-%m-%d %H:%M'))
+ f.write(output)
+
+if __name__ == "__main__":
+ try:
+ main()
+ except (KeyboardInterrupt, EOFError):
+ print '\nAborted...'
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/904b53df/install/python-installer/hadoop_mods.py
----------------------------------------------------------------------
diff --git a/install/python-installer/hadoop_mods.py b/install/python-installer/hadoop_mods.py
new file mode 100755
index 0000000..168a81f
--- /dev/null
+++ b/install/python-installer/hadoop_mods.py
@@ -0,0 +1,203 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on local node ###
+
+import time
+import sys
+import json
+from common import ParseHttp, ParseJson, MODCFG_FILE, err
+
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+
+dbcfgs = json.loads(dbcfgs_json)
+modcfgs = ParseJson(MODCFG_FILE).load()
+
+MOD_CFGS = modcfgs['MOD_CFGS']
+HBASE_MASTER_CONFIG = modcfgs['HBASE_MASTER_CONFIG']
+HBASE_RS_CONFIG = modcfgs['HBASE_RS_CONFIG']
+HDFS_CONFIG = modcfgs['HDFS_CONFIG']
+ZK_CONFIG = modcfgs['ZK_CONFIG']
+
+CLUSTER_URL_PTR = '%s/api/v1/clusters/%s'
+RESTART_URL_PTR = CLUSTER_URL_PTR + '/commands/restart'
+RESTART_SRV_URL_PTR = CLUSTER_URL_PTR + '/services/%s/commands/restart'
+SRVCFG_URL_PTR = CLUSTER_URL_PTR + '/services/%s/config'
+RSGRP_BASEURL_PTR = '%s/api/v6/clusters/%s/services/%s/roleConfigGroups'
+DEPLOY_CFG_URL_PTR = '%s/api/v6/clusters/%s/commands/deployClientConfig'
+CMD_STAT_URL_PTR = '%s/api/v1/commands/%s'
+
+class CDHMod(object):
+ """ Modify CDH configs for trafodion and restart CDH services """
+ def __init__(self, user, passwd, url, cluster_name):
+ self.url = url
+ self.cluster_name = cluster_name
+ self.p = ParseHttp(user, passwd)
+
+ def __retry_check(self, cid, maxcnt, interval, msg):
+ stat_url = CMD_STAT_URL_PTR % (self.url, cid)
+ stat = self.p.get(stat_url)
+ retry_cnt = 0
+ while not (stat['success'] is True and stat['active'] is False):
+ retry_cnt += 1
+ flush_str = '.' * retry_cnt
+ print '\rCheck CDH services %s status (timeout: %dmin) %s' % (msg, maxcnt*interval/60, flush_str),
+ sys.stdout.flush()
+ time.sleep(interval)
+ stat = self.p.get(stat_url)
+ if retry_cnt == maxcnt: return False
+ return True
+
+ def mod(self):
+ hdfs_service = dbcfgs['hdfs_service_name']
+ hbase_service = dbcfgs['hbase_service_name']
+ zk_service = dbcfgs['zookeeper_service_name']
+ services = {hdfs_service:HDFS_CONFIG, hbase_service:HBASE_MASTER_CONFIG, zk_service:ZK_CONFIG}
+
+ for srv, cfg in services.iteritems():
+ srvcfg_url = SRVCFG_URL_PTR % (self.url, self.cluster_name, srv)
+ self.p.put(srvcfg_url, cfg)
+
+ # set configs in each regionserver group
+ rsgrp_baseurl = RSGRP_BASEURL_PTR % (self.url, self.cluster_name, hbase_service)
+ rscfg = self.p.get(rsgrp_baseurl)
+ rsgrp_urls = ['%s/%s/config' % (rsgrp_baseurl, r['name']) for r in rscfg['items'] if r['roleType'] == 'REGIONSERVER']
+
+ for rsgrp_url in rsgrp_urls:
+ self.p.put(rsgrp_url, HBASE_RS_CONFIG)
+
+ def restart(self):
+ restart_url = RESTART_URL_PTR % (self.url, self.cluster_name)
+ deploy_cfg_url = DEPLOY_CFG_URL_PTR % (self.url, self.cluster_name)
+
+ print 'Restarting CDH services ...'
+ rc1 = self.p.post(restart_url)
+ if self.__retry_check(rc1['id'], 40, 15, 'restart'):
+ print 'Restart CDH successfully!'
+ else:
+ err('Failed to restart CDH, max retry count reached')
+
+ rc2 = self.p.post(deploy_cfg_url)
+ if self.__retry_check(rc2['id'], 30, 10, 'deploy'):
+ print 'Deploy client config successfully!'
+ else:
+ err('Failed to deploy CDH client config, max retry count reached')
+
+
+class HDPMod(object):
+ """ Modify HDP configs for trafodion and restart HDP services """
+ def __init__(self, user, passwd, url, cluster_name):
+ self.url = url
+ self.cluster_name = cluster_name
+ self.p = ParseHttp(user, passwd, json_type=False)
+
+ def mod(self):
+ cluster_url = CLUSTER_URL_PTR % (self.url, self.cluster_name)
+ desired_cfg_url = cluster_url + '?fields=Clusters/desired_configs'
+ cfg_url = cluster_url + '/configurations?type={0}&tag={1}'
+ desired_cfg = self.p.get(desired_cfg_url)
+
+ for config_type in MOD_CFGS.keys():
+ desired_tag = desired_cfg['Clusters']['desired_configs'][config_type]['tag']
+ current_cfg = self.p.get(cfg_url.format(config_type, desired_tag))
+ tag = 'version' + str(int(time.time() * 1000000))
+ new_properties = current_cfg['items'][0]['properties']
+ new_properties.update(MOD_CFGS[config_type])
+ config = {
+ 'Clusters': {
+ 'desired_config': {
+ 'type': config_type,
+ 'tag': tag,
+ 'properties': new_properties
+ }
+ }
+ }
+ self.p.put(cluster_url, config)
+
+
+ def restart(self):
+ srv_baseurl = CLUSTER_URL_PTR % (self.url, self.cluster_name) + '/services/'
+ srvs = ['HBASE', 'ZOOKEEPER', 'HDFS']
+
+ # Stop
+ print 'Restarting HDP services ...'
+ for srv in srvs:
+ srv_url = srv_baseurl + srv
+ config = {'RequestInfo': {'context' :'Stop %s services' % srv}, 'ServiceInfo': {'state' : 'INSTALLED'}}
+ rc = self.p.put(srv_url, config)
+
+ # check stop status
+ if rc:
+ stat = self.p.get(srv_url)
+
+ retry_cnt, maxcnt, interval = 0, 30, 5
+ while stat['ServiceInfo']['state'] != 'INSTALLED':
+ retry_cnt += 1
+ flush_str = '.' * retry_cnt
+ print '\rCheck HDP service %s stop status (timeout: %dmin) %s' % (srv, maxcnt*interval/60, flush_str),
+ sys.stdout.flush()
+ time.sleep(interval)
+ stat = self.p.get(srv_url)
+ if retry_cnt == maxcnt: err('Failed to stop HDP service %s, timeout' % srv)
+ # wrap line
+ print
+ else:
+ print 'HDP service %s had already been stopped' % srv
+
+ time.sleep(5)
+ # Start
+ config = {'RequestInfo': {'context' :'Start All services'}, 'ServiceInfo': {'state' : 'STARTED'}}
+ rc = self.p.put(srv_baseurl, config)
+
+ # check start status
+ if rc:
+ result_url = rc['href']
+ stat = self.p.get(result_url)
+ retry_cnt, maxcnt, interval = 0, 120, 5
+ while stat['Requests']['request_status'] != 'COMPLETED':
+ retry_cnt += 1
+ flush_str = '.' * retry_cnt
+ print '\rCheck HDP services start status (timeout: %dmin) %s' % (maxcnt*interval/60, flush_str),
+ sys.stdout.flush()
+ time.sleep(interval)
+ stat = self.p.get(result_url)
+ if retry_cnt == maxcnt: err('Failed to start all HDP services')
+ print 'HDP services started successfully!'
+ else:
+ print 'HDP services had already been started'
+
+def run():
+ if 'CDH' in dbcfgs['distro']:
+ cdh = CDHMod(dbcfgs['mgr_user'], dbcfgs['mgr_pwd'], dbcfgs['mgr_url'], dbcfgs['cluster_name'])
+ cdh.mod()
+ cdh.restart()
+ elif 'HDP' in dbcfgs['distro']:
+ hdp = HDPMod(dbcfgs['mgr_user'], dbcfgs['mgr_pwd'], dbcfgs['mgr_url'], dbcfgs['cluster_name'])
+ hdp.mod()
+ hdp.restart()
+
+# main
+run()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/904b53df/install/python-installer/hdfs_cmds.py
----------------------------------------------------------------------
diff --git a/install/python-installer/hdfs_cmds.py b/install/python-installer/hdfs_cmds.py
new file mode 100755
index 0000000..dc08b7e
--- /dev/null
+++ b/install/python-installer/hdfs_cmds.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+### this script should be run on first node with trafodion user ###
+
+import os
+import sys
+import json
+from common import err, run_cmd, cmd_output, run_cmd_as_user
+
+def run():
+ hdfs_bin = '/usr/bin/hdfs'
+
+ dbcfgs = json.loads(dbcfgs_json)
+ DISTRO = dbcfgs['distro']
+
+ if 'CDH' in DISTRO:
+ parcel_lib = '/opt/cloudera/parcels/CDH/lib/hbase/lib'
+ if os.path.exists(parcel_lib): hdfs_bin = '/opt/cloudera/parcels/CDH/bin/hdfs'
+ elif 'APACHE' in DISTRO:
+ hdfs_bin = dbcfgs['hadoop_home'] + '/bin/hdfs'
+
+ traf_loc = '/user/trafodion'
+ traf_user = dbcfgs['traf_user']
+ hdfs_user = dbcfgs['hdfs_user']
+ hbase_user = dbcfgs['hbase_user']
+
+ run_cmd_as_user(hdfs_user, '%s dfsadmin -safemode wait' % hdfs_bin)
+ run_cmd_as_user(hdfs_user, '%s dfs -mkdir -p %s/{trafodion_backups,bulkload,lobs} /hbase/archive /hbase-staging' % (hdfs_bin, traf_loc))
+ run_cmd_as_user(hdfs_user, '%s dfs -chown -R %s:%s /hbase/archive /hbase-staging' % (hdfs_bin, hbase_user, hbase_user))
+ run_cmd_as_user(hdfs_user, '%s dfs -chown -R %s:%s %s/{trafodion_backups,bulkload,lobs}' % (hdfs_bin, traf_user, traf_user, traf_loc))
+ run_cmd_as_user(hdfs_user, '%s dfs -setfacl -R -m user:%s:rwx /hbase/archive' % (hdfs_bin, traf_user))
+ run_cmd_as_user(hdfs_user, '%s dfs -setfacl -R -m default:user:%s:rwx /hbase/archive' % (hdfs_bin, traf_user))
+ run_cmd_as_user(hdfs_user, '%s dfs -setfacl -R -m mask::rwx /hbase/archive' % hdfs_bin)
+
+ # Grant all privileges to the Trafodion principal in HBase
+ if dbcfgs['secure_hadoop'] == 'Y':
+ run_cmd('grant "%s", "RWXC" | sudo -u %s hbase shell > /tmp/hbase_shell.out' % (traf_user, hbase_user))
+ has_err = cmd_output('grep -c ERROR /tmp/hbase_shell.out')
+ if int(has_err):
+ err('Failed to grant HBase privileges to %s' % traf_user)
+ run_cmd('rm /tmp/hbase_shell.out')
+# main
+try:
+ dbcfgs_json = sys.argv[1]
+except IndexError:
+ err('No db config found')
+run()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/904b53df/install/python-installer/mod_cfgs.json
----------------------------------------------------------------------
diff --git a/install/python-installer/mod_cfgs.json b/install/python-installer/mod_cfgs.json
new file mode 100644
index 0000000..822b470
--- /dev/null
+++ b/install/python-installer/mod_cfgs.json
@@ -0,0 +1,73 @@
+{
+"MOD_CFGS": {
+ "hbase-site": {
+ "hbase.master.distributed.log.splitting": "false",
+ "hbase.snapshot.master.timeoutMillis": "600000",
+ "hbase.coprocessor.region.classes": "org.apache.hadoop.hbase.coprocessor.transactional.TrxRegionObserver,org.apache.hadoop.hbase.coprocessor.transactional.TrxRegionEndpoint,org.apache.hadoop.hbase.coprocessor.AggregateImplementation",
+ "hbase.hregion.impl": "org.apache.hadoop.hbase.regionserver.transactional.TransactionalRegion",
+ "hbase.regionserver.region.split.policy": "org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy",
+ "hbase.snapshot.enabled": "true",
+ "hbase.bulkload.staging.dir": "/hbase-staging",
+ "hbase.regionserver.region.transactional.tlog": "true",
+ "hbase.snapshot.region.timeout": "600000",
+ "hbase.client.scanner.timeout.period": "600000"
+ },
+ "hdfs-site": { "dfs.namenode.acls.enabled": "true" },
+ "zoo.cfg": { "maxClientCnxns": "0" }
+},
+
+"HBASE_MASTER_CONFIG": {
+"roleTypeConfigs" : [ {
+ "roleType" : "MASTER",
+ "items" : [ {
+ "name" : "hbase_master_config_safety_valve",
+ "value" : "<property>\r\n <name>hbase.master.distributed.log.splitting</name>\r\n <value>false</value>\r\n</property>\r\n <property>\r\n <name>hbase.snapshot.master.timeoutMillis</name>\r\n <value>600000</value>\r\n</property>\r\n"
+ } ]
+ } ]
+},
+
+"HBASE_RS_CONFIG": {
+"items" : [ {
+ "name" : "hbase_coprocessor_region_classes",
+ "value" : "org.apache.hadoop.hbase.coprocessor.transactional.TrxRegionObserver,org.apache.hadoop.hbase.coprocessor.transactional.TrxRegionEndpoint,org.apache.hadoop.hbase.coprocessor.AggregateImplementation"
+ }, {
+ "name" : "hbase_regionserver_lease_period",
+ "value" : "600000"
+ }, {
+ "name" : "hbase_regionserver_config_safety_valve",
+ "value" : "<property>\r\n <name>hbase.hregion.impl</name>\r\n <value>org.apache.hadoop.hbase.regionserver.transactional.TransactionalRegion</value>\r\n</property>\r\n <property>\r\n <name>hbase.regionserver.region.split.policy</name>\r\n <value>org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy</value>\r\n</property>\r\n <property>\r\n <name>hbase.snapshot.enabled</name>\r\n <value>true</value>\r\n</property>\r\n <property>\r\n <name>hbase.bulkload.staging.dir</name>\r\n <value>/hbase-staging</value>\r\n</property>\r\n <property>\r\n <name>hbase.regionserver.region.transactional.tlog</name>\r\n <value>true</value>\r\n</property>\r\n <property>\r\n <name>hbase.snapshot.region.timeout</name>\r\n <value>600000</value>\r\n</property>\r\n "
+ } ]
+},
+
+"HDFS_CONFIG": {
+"roleTypeConfigs" : [ {
+ "roleType" : "NAMENODE",
+ "items": [ {
+ "name" : "namenode_java_heapsize",
+ "value" : "1073741824"
+ } ]
+ }, {
+ "roleType" : "SECONDARYNAMENODE",
+ "items":[ {
+ "name" : "secondary_namenode_java_heapsize",
+ "value" : "1073741824"
+ } ]
+ } ],
+ "items": [ {
+ "name":"dfs_namenode_acls_enabled",
+ "value":"true"
+ } ]
+},
+
+"ZK_CONFIG":{
+"roleTypeConfigs" :
+ [ {
+ "roleType" : "SERVER",
+ "items":
+ [ {
+ "name" : "maxClientCnxns",
+ "value" : "0"
+ } ]
+ } ]
+}
+}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/904b53df/install/python-installer/prompt.json
----------------------------------------------------------------------
diff --git a/install/python-installer/prompt.json b/install/python-installer/prompt.json
new file mode 100644
index 0000000..268eca4
--- /dev/null
+++ b/install/python-installer/prompt.json
@@ -0,0 +1,227 @@
+{
+ "traf_start":
+ {
+ "prompt":"Start instance after installation",
+ "default":"Y",
+ "isYN":true
+ },
+ "dcs_ha":
+ {
+ "prompt":"Enable DCS High Avalability",
+ "default":"N",
+ "isYN":true
+ },
+ "dcs_floating_ip":
+ {
+ "prompt":"Enter Floating IP address for DCS HA",
+ "isIP":true
+ },
+ "dcs_interface":
+ {
+ "prompt":"Enter interface for Floating IP address",
+ "default":"eth0"
+ },
+ "dcs_backup_nodes":
+ {
+ "prompt":"Enter DCS Backup Master Nodes for DCS HA (comma separated)"
+ },
+ "kdc_server":
+ {
+ "prompt":"Enter KDC server address"
+ },
+ "admin_principal":
+ {
+ "prompt":"Enter admin principal (include realm)"
+ },
+ "kdcadmin_pwd":
+ {
+ "prompt":"Enter password for admin principal",
+ "ispasswd":true
+ },
+ "hdfs_keytab":
+ {
+ "prompt":"Enter fully qualified name for HDFS keytab"
+ },
+ "hbase_keytab":
+ {
+ "prompt":"Enter fully qualified name for HBase keytab"
+ },
+ "max_lifetime":
+ {
+ "prompt":"Enter max lifetime for Trafodion principal (valid format required)",
+ "default":"24hours"
+ },
+ "max_renew_lifetime":
+ {
+ "prompt":"Enter renew lifetime for Trafodion principal (valid format required)",
+ "default":"7days"
+ },
+ "traf_keytab":
+ {
+ "prompt":"Enter Trafodion keytab name"
+ },
+ "ldap_security":
+ {
+ "prompt":"Enable LDAP security",
+ "default":"N",
+ "isYN":true
+ },
+ "ldap_hosts":
+ {
+ "prompt":"Enter list of LDAP Hostnames (comma separated)"
+ },
+ "ldap_port":
+ {
+ "prompt":"Enter LDAP Port number (Example: 389 for no encryption or TLS, 636 for SSL)",
+ "default":"389",
+ "isdigit":true
+ },
+ "ldap_identifiers":
+ {
+ "prompt":"Enter all LDAP unique identifiers (blank separated)"
+ },
+ "ldap_encrypt":
+ {
+ "prompt":"Enter LDAP Encryption Level (0: Encryption not used, 1: SSL, 2: TLS)",
+ "default":"0",
+ "isdigit":true
+ },
+ "ldap_certpath":
+ {
+ "prompt":"Enter full path to TLS certificate file (*.pem)",
+ "isexist":true
+ },
+ "ldap_userinfo":
+ {
+ "prompt":"If Requred search user name/password",
+ "default":"N",
+ "isYN":true
+ },
+ "ldap_user":
+ {
+ "prompt":"Enter Search user name (if required)",
+ "default":" "
+ },
+ "ldap_pwd":
+ {
+ "prompt":"Enter Search password (if required)",
+ "default":" "
+ },
+ "scratch_locs":
+ {
+ "prompt":"Enter trafodion scratch file folder location(should be a large disk), if more than one folder, use comma seperated",
+ "default":"$MY_SQROOT/tmp"
+ },
+ "local_repo_dir":
+ {
+ "prompt":"Enter local repository folder location to install trafodion RPM dependencies",
+ "isexist":true
+ },
+ "java_home":
+ {
+ "prompt":"Specify location of Java(JDK) on trafodion nodes",
+ "isremote_exist":true,
+ "default":"/usr/lib/jvm/java-1.7.0-openjdk.x86_64"
+ },
+ "dcs_cnt_per_node":
+ {
+ "prompt":"Enter number of DCS client connections per node",
+ "default":"4",
+ "isdigit":true
+ },
+ "first_rsnode":
+ {
+ "prompt":"Enter the hostname of first Apache HBase RegionServer node"
+ },
+ "hdfs_user":
+ {
+ "prompt":"Enter hadoop user name",
+ "default":"hdfs",
+ "isuser":true
+ },
+ "hbase_user":
+ {
+ "prompt":"Enter hbase user name",
+ "default":"hbase",
+ "isuser":true
+ },
+ "hadoop_home":
+ {
+ "prompt":"Enter Apache Hadoop directory location",
+ "isremote_exist":true
+ },
+ "hbase_home":
+ {
+ "prompt":"Enter Apache HBase directory location",
+ "isremote_exist":true
+ },
+ "hive_home":
+ {
+ "prompt":"Enter Apache Hive directory location if exists",
+ "default":"NO_HIVE"
+ },
+ "mgr_url":
+ {
+ "prompt":"Enter HDP/CDH web manager URL:port, (full URL, if no http/https prefix, default prefix is http://)"
+ },
+ "mgr_user":
+ {
+ "prompt":"Enter HDP/CDH web manager user name",
+ "default":"admin",
+ "isuser":true
+ },
+ "mgr_pwd":
+ {
+ "prompt":"Enter HDP/CDH web manager user password",
+ "ispasswd":true
+ },
+ "traf_user":
+ {
+ "prompt":"Enter trafodion user name",
+ "default":"trafodion",
+ "isuser":true
+ },
+ "traf_pwd":
+ {
+ "prompt":"Enter trafodion user password",
+ "ispasswd":true
+ },
+ "traf_package":
+ {
+ "prompt":"Enter full path to Trafodion tar file",
+ "isexist":true
+ },
+ "db_root_user":
+ {
+ "prompt":"Enter LDAP user name to be assigned DB root privileges (DB__ROOT)",
+ "default":"trafodion",
+ "isuser":true
+ },
+ "db_admin_user":
+ {
+ "prompt":"Enter LDAP user name to be assigned DB Admin privileges",
+ "default":"admin",
+ "isuser":true
+ },
+ "db_admin_pwd":
+ {
+ "prompt":"Enter LDAP user password to be assigned DB Admin privileges",
+ "default":"traf123"
+ },
+ "node_list":
+ {
+ "prompt":"Enter list of Nodes separated by comma, support simple numeric RE,\n i.e. \"n[01-12],n[21-25]\",\"n0[1-5].com\""
+ },
+ "cluster_no":
+ {
+ "prompt":"Select the above cluster number for installing Trafodion",
+ "default":"1",
+ "isdigit":true
+ },
+ "use_data_node":
+ {
+ "prompt":"Install Trafodion nodes on all DataNodes",
+ "default":"Y",
+ "isYN":true
+ }
+}