You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by od...@apache.org on 2016/03/31 02:24:26 UTC
[33/38] incubator-hawq git commit: HAWQ-560. Set
'output.replace-datanode-on-failure' depend on datanodes number to
hdfs-client.xml
HAWQ-560. Set 'output.replace-datanode-on-failure' depend on datanodes number to hdfs-client.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2becc74d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2becc74d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2becc74d
Branch: refs/heads/HAWQ-546
Commit: 2becc74dc6a4612732c253c89a14aadcbe86a8a2
Parents: 312de8f
Author: rlei <rl...@pivotal.io>
Authored: Sun Mar 27 16:47:16 2016 +0800
Committer: Oleksandr Diachenko <od...@pivotal.io>
Committed: Wed Mar 30 17:23:33 2016 -0700
----------------------------------------------------------------------
tools/bin/hawq_ctl | 60 +++++++++++++++++++++++++++++--------
tools/bin/hawqconfig | 60 +++----------------------------------
tools/bin/hawqpylib/hawqlib.py | 19 ++++++++++--
3 files changed, 68 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2becc74d/tools/bin/hawq_ctl
----------------------------------------------------------------------
diff --git a/tools/bin/hawq_ctl b/tools/bin/hawq_ctl
index ce9fa7d..c567b09 100755
--- a/tools/bin/hawq_ctl
+++ b/tools/bin/hawq_ctl
@@ -182,16 +182,50 @@ class HawqInit:
logger.warn("Set standby host name failed")
return result
- def set_number_by_datanodes_num(self):
- reuslt = 0
- if self.hosts_count_number < 4:
- # Set output.replace-datanode-on-failure to false if datanode number less than 4.
- logger.info("Set output.replace-datanode-on-failure to false as %s datanode detected" % self.hosts_count_number)
- logger.info("If you actually have more than 4 datanodes, please set it to 'true'")
- cmd = "hawq config -c 'output.replace-datanode-on-failure' -v 'false' --skipvalidation -q > /dev/null"
- result = local_ssh(cmd, logger)
- if result != 0:
- logger.warn("Set output.replace-datanode-on-failure failed")
+ def sync_hdfs_client(self):
+ sync_host_str = ""
+ for node in self.host_list:
+ sync_host_str += " -h %s" % node
+
+ if 'hawq_standby_address_host' in self.hawq_dict and self.standby_host_name.lower() not in ('', 'none'):
+ sync_host_str += " -h %s" % self.standby_host_name
+
+ result = local_ssh("hawq scp %s %s/etc/hdfs-client.xml =:%s/etc/" % (sync_host_str, self.GPHOME, self.GPHOME))
+ if result != 0:
+ logger.error("Sync hdfs-client.xml failed.")
+ sys.exit(1)
+
+ def set_replace_datanode_on_failure(self):
+ xml_file = "%s/etc/hdfs-client.xml" % self.GPHOME
+ property_name = 'output.replace-datanode-on-failure'
+ pexist, pname, pvalue = check_property_exist_xml(xml_file, property_name)
+ datanodes_threshold = 4
+
+ if self.hosts_count_number < datanodes_threshold:
+ property_value = 'false'
+ else:
+ property_value = 'true'
+
+ if pvalue != property_value:
+ logger.info("Set output.replace-datanode-on-failure to %s" % property_value)
+ if pexist:
+ logger.debug("Update output.replace-datanode-on-failure to %s" % property_value)
+ update_xml_property(xml_file, property_name, property_value)
+ else:
+ logger.debug("Add output.replace-datanode-on-failure as %s" % property_value)
+ append_xml_property(xml_file, property_name, property_value)
+
+ self.sync_hdfs_client()
+
+ # Check hdfs-client.xml again to validate changes
+ pexist, pname, pvalue = check_property_exist_xml(xml_file, property_name)
+ if pvalue == property_value:
+ result = 0
+ else:
+ result = 1
+ else:
+ result = 0
+
return result
def set_total_vsegment_num(self):
@@ -270,7 +304,7 @@ class HawqInit:
logger.error("Failed to connect to database, this script can only be run when the database is up")
cmd = "%s; hawq stop master -a -M fast" % source_hawq_env
check_return_code(local_ssh(cmd, logger), logger, "Stop hawq master failed, exit")
- remove_property_xml("hawq_standby_address_host", "%s/etc/hawq-site.xml" % self.GPHOME)
+ remove_property_xml("hawq_standby_address_host", "%s/etc/hawq-site.xml" % self.GPHOME, self.quiet)
host_list = parse_hosts_file(self.GPHOME)
sync_hawq_site(self.GPHOME, host_list)
gpsyncmaster_pid = gp.getSyncmasterPID(running_standby_host, self.master_data_directory)
@@ -351,7 +385,7 @@ class HawqInit:
check_return_code(self.set_total_vsegment_num())
logger.info("Set hawq_rm_nvseg_perquery_perseg_limit as: %s" % self.vseg_num_per_node)
check_return_code(self.set_vsegment_num_per_node())
- check_return_code(self.set_number_by_datanodes_num())
+ check_return_code(self.set_replace_datanode_on_failure())
master_cmd = self._get_master_init_cmd()
logger.info("Start to init master node: '%s'" % self.master_host_name)
@@ -396,7 +430,7 @@ class HawqInit:
check_return_code(self.set_total_vsegment_num())
logger.info("Set hawq_rm_nvseg_perquery_perseg_limit as: %s" % self.vseg_num_per_node)
check_return_code(self.set_vsegment_num_per_node())
- check_return_code(self.set_number_by_datanodes_num())
+ check_return_code(self.set_replace_datanode_on_failure())
logger.info("Start to init master")
cmd = self._get_master_init_cmd()
check_return_code(local_ssh(cmd, logger, warning = True), logger, \
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2becc74d/tools/bin/hawqconfig
----------------------------------------------------------------------
diff --git a/tools/bin/hawqconfig b/tools/bin/hawqconfig
index fda8782..6618bc3 100755
--- a/tools/bin/hawqconfig
+++ b/tools/bin/hawqconfig
@@ -18,7 +18,7 @@
try:
import os, sys, re
from optparse import Option, OptionParser
- from hawqpylib.hawqlib import HawqXMLParser, parse_hosts_file, remove_property_xml, local_ssh
+ from hawqpylib.hawqlib import HawqXMLParser, parse_hosts_file, remove_property_xml, update_xml_property, append_xml_property, local_ssh
from gppylib.commands.unix import getLocalHostname, getUserName
from gppylib.db import dbconn
from pg import DatabaseError
@@ -166,64 +166,12 @@ def show_property(hawq_site, property_name):
return None
-def update_line(xmlfile, property_name, property_value):
- with open(xmlfile) as f:
- f_tmp = open(xmlfile + '.backup', 'w')
- while 1:
- line = f.readline()
-
- if not line:
- break
-
- m = re.match('\s*<name>%s' % property_name, line)
- if m:
- while 1:
- next_line = f.readline()
- m2 = re.match('\s*<value>', next_line)
- if m2:
- f_tmp.write(line)
- p = re.compile('\s*<value>(.*)</value>')
- p_value = p.match(next_line).group(1)
- next_line_new = re.sub(p_value, property_value, next_line)
- f_tmp.write(next_line_new)
- break
- else:
- f_tmp.write(line)
- f_tmp.close()
- return None
-
-
-def add_property(xmlfile, property_name, property_value):
- with open(xmlfile) as f:
- f_tmp = open(xmlfile + '.backup', 'w')
- while 1:
- line = f.readline()
-
- if not line:
- break
-
- m = re.match('\s*</configuration>', line)
- if m:
- f_tmp.write(' <property>\n')
- f_tmp.write(' <name>%s</name>\n' % property_name)
- f_tmp.write(' <value>%s</value>\n' % property_value)
- f_tmp.write(' </property>\n')
- f_tmp.write('</configuration>\n')
- else:
- f_tmp.write(line)
- f_tmp.close()
- return None
-
-
def update_hawq_site(org_config_file, hawq_site, property_name, property_value):
- os.system('cp %s %s' % (org_config_file, org_config_file + '.org'))
result = check_property(hawq_site, property_name, property_value)
if result:
- update_line(org_config_file, property_name, property_value)
- os.system('mv %s %s' % (org_config_file + '.backup', org_config_file))
+ update_xml_property(org_config_file, property_name, property_value)
else:
- add_property(org_config_file, property_name, property_value)
- os.system('mv %s %s' % (org_config_file + '.backup', org_config_file))
+ append_xml_property(org_config_file, property_name, property_value)
def sync_hawq_site(config_dir, host_list):
@@ -283,7 +231,7 @@ if __name__ == '__main__':
if options.remove in except_list:
print "Remove %s is not allowed" % options.remove
sys.exit(1)
- remove_property_xml(options.remove, org_config_file)
+ remove_property_xml(options.remove, org_config_file, options.quiet)
sync_hawq_site(GPHOME, host_list)
else:
print "Please input correct options"
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2becc74d/tools/bin/hawqpylib/hawqlib.py
----------------------------------------------------------------------
diff --git a/tools/bin/hawqpylib/hawqlib.py b/tools/bin/hawqpylib/hawqlib.py
index cc9e83c..85354b4 100755
--- a/tools/bin/hawqpylib/hawqlib.py
+++ b/tools/bin/hawqpylib/hawqlib.py
@@ -115,6 +115,20 @@ class threads_with_return(object):
return self.return_values
+def check_property_exist_xml(xml_file, property_name):
+ property_exist = False
+ property_value = ''
+ with open(xml_file) as f:
+ xmldoc = minidom.parse(f)
+ for node in xmldoc.getElementsByTagName('property'):
+ name, value = (node.getElementsByTagName('name')[0].childNodes[0].data,
+ node.getElementsByTagName('value')[0].childNodes[0].data)
+ if name == property_name:
+ property_exist = True
+ property_value = value
+ return property_exist, property_name, property_value
+
+
class HawqXMLParser:
def __init__(self, GPHOME):
self.GPHOME = GPHOME
@@ -416,14 +430,15 @@ def append_xml_property(xmlfile, property_name, property_value):
shutil.move(xmlfile_swap, xmlfile)
-def remove_property_xml(property_name, org_config_file):
+def remove_property_xml(property_name, org_config_file, quiet = False):
tree = ElementTree()
tree.parse(org_config_file)
root = tree.getroot()
for child in root:
for subet in child:
if subet.text == property_name:
- print "Remove property %s." % subet.text
+ if not quiet:
+ print "Remove property %s." % subet.text
root.remove(child)
tree.write(org_config_file, encoding="utf-8")