You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by rl...@apache.org on 2016/03/28 05:07:34 UTC
[2/2] incubator-hawq git commit: HAWQ-560. Set
'output.replace-datanode-on-failure' depend on datanodes number
HAWQ-560. Set 'output.replace-datanode-on-failure' depend on datanodes number
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/a4969d26
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/a4969d26
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/a4969d26
Branch: refs/heads/master
Commit: a4969d26c70bc4b675763ebb7287674bfb7fc408
Parents: 3db0b6f
Author: rlei <rl...@pivotal.io>
Authored: Fri Mar 18 18:01:46 2016 +0800
Committer: rlei <rl...@pivotal.io>
Committed: Mon Mar 28 11:07:03 2016 +0800
----------------------------------------------------------------------
tools/bin/hawq_ctl | 24 +++++++++++++--
tools/bin/hawqpylib/hawqlib.py | 61 +++++++++++++++++++++++++++++++++++++
2 files changed, 83 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a4969d26/tools/bin/hawq_ctl
----------------------------------------------------------------------
diff --git a/tools/bin/hawq_ctl b/tools/bin/hawq_ctl
index 0d2aa09..ce9fa7d 100755
--- a/tools/bin/hawq_ctl
+++ b/tools/bin/hawq_ctl
@@ -182,6 +182,18 @@ class HawqInit:
logger.warn("Set standby host name failed")
return result
+ def set_number_by_datanodes_num(self):
+ reuslt = 0
+ if self.hosts_count_number < 4:
+ # Set output.replace-datanode-on-failure to false if datanode number less than 4.
+ logger.info("Set output.replace-datanode-on-failure to false as %s datanode detected" % self.hosts_count_number)
+ logger.info("If you actually have more than 4 datanodes, please set it to 'true'")
+ cmd = "hawq config -c 'output.replace-datanode-on-failure' -v 'false' --skipvalidation -q > /dev/null"
+ result = local_ssh(cmd, logger)
+ if result != 0:
+ logger.warn("Set output.replace-datanode-on-failure failed")
+ return result
+
def set_total_vsegment_num(self):
cmd = "%s; hawq config -c default_hash_table_bucket_number -v %s --skipvalidation -q > /dev/null" % \
(source_hawq_env, self.total_vseg_num)
@@ -337,7 +349,9 @@ class HawqInit:
logger.info("%s segment hosts defined" % self.hosts_count_number)
logger.info("Set default_hash_table_bucket_number as: %s" % self.total_vseg_num)
check_return_code(self.set_total_vsegment_num())
+ logger.info("Set hawq_rm_nvseg_perquery_perseg_limit as: %s" % self.vseg_num_per_node)
check_return_code(self.set_vsegment_num_per_node())
+ check_return_code(self.set_number_by_datanodes_num())
master_cmd = self._get_master_init_cmd()
logger.info("Start to init master node: '%s'" % self.master_host_name)
@@ -380,7 +394,9 @@ class HawqInit:
logger.info("%s segment hosts defined" % self.hosts_count_number)
logger.info("Set default_hash_table_bucket_number as: %s" % self.total_vseg_num)
check_return_code(self.set_total_vsegment_num())
+ logger.info("Set hawq_rm_nvseg_perquery_perseg_limit as: %s" % self.vseg_num_per_node)
check_return_code(self.set_vsegment_num_per_node())
+ check_return_code(self.set_number_by_datanodes_num())
logger.info("Start to init master")
cmd = self._get_master_init_cmd()
check_return_code(local_ssh(cmd, logger, warning = True), logger, \
@@ -931,7 +947,8 @@ def get_args():
quiet_stdout_logging()
logger, log_filename = setup_hawq_tool_logging('hawq_%s' % opts.hawq_command,getLocalHostname(),getUserName(), opts.log_dir)
logger.info("Prepare to do 'hawq %s'" % opts.hawq_command)
- logger.info("You can check log in %s" % log_filename)
+ logger.info("You can find log in:")
+ logger.info(log_filename)
else:
print COMMON_HELP
sys.exit(1)
@@ -953,7 +970,8 @@ def get_args():
if not opts.GPHOME:
logger.error("Didn't get GPHOME value, exit")
sys.exit()
- logger.debug("GPHOME is %s" % opts.GPHOME)
+ logger.info("GPHOME is set to:")
+ logger.info(opts.GPHOME)
global source_hawq_env
source_hawq_env = "source %s/greenplum_path.sh" % opts.GPHOME
@@ -963,6 +981,8 @@ def get_args():
logger.error("'root' user is not allowed")
sys.exit()
logger.debug("Current user is '%s'" % opts.user)
+ logger.debug("Parsing config file:")
+ logger.debug("%s/etc/hawq-site.xml" % opts.GPHOME)
hawqsite = HawqXMLParser(opts.GPHOME)
hawqsite.get_all_values()
hawq_dict = hawqsite.hawq_dict
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a4969d26/tools/bin/hawqpylib/hawqlib.py
----------------------------------------------------------------------
diff --git a/tools/bin/hawqpylib/hawqlib.py b/tools/bin/hawqpylib/hawqlib.py
index 11a9dfa..cc9e83c 100755
--- a/tools/bin/hawqpylib/hawqlib.py
+++ b/tools/bin/hawqpylib/hawqlib.py
@@ -22,7 +22,9 @@ import threading
import Queue
from xml.dom import minidom
from xml.etree.ElementTree import ElementTree
+import shutil
from gppylib.db import dbconn
+import re
class HawqCommands(object):
@@ -355,6 +357,65 @@ def parse_hosts_file(GPHOME):
return host_list
+def update_xml_property(xmlfile, property_name, property_value):
+ file_path, filename = os.path.split(xmlfile)
+ xmlfile_backup = os.path.join(file_path, '.bak.' + filename)
+ xmlfile_swap = os.path.join(file_path, '.swp.' + filename)
+ shutil.copyfile(xmlfile, xmlfile_backup)
+ with open(xmlfile_backup) as f:
+ f_tmp = open(xmlfile_swap, 'w')
+ while 1:
+ line = f.readline()
+
+ if not line:
+ break
+
+ m = re.match('\s*<name>%s' % property_name, line)
+ if m:
+ while 1:
+ next_line = f.readline()
+ m2 = re.match('\s*<value>', next_line)
+ if m2:
+ f_tmp.write(line)
+ p = re.compile('\s*<value>(.*)</value>')
+ p_value = p.match(next_line).group(1)
+ next_line_new = re.sub(p_value, property_value, next_line)
+ f_tmp.write(next_line_new)
+ break
+ else:
+ f_tmp.write(line)
+ f_tmp.close()
+
+ shutil.move(xmlfile_swap, xmlfile)
+
+
+def append_xml_property(xmlfile, property_name, property_value):
+ file_path, filename = os.path.split(xmlfile)
+ xmlfile_backup = os.path.join(file_path, '.bak.' + filename)
+ xmlfile_swap = os.path.join(file_path, '.swp.' + filename)
+ shutil.copyfile(xmlfile, xmlfile_backup)
+ with open(xmlfile_backup) as f:
+ f_tmp = open(xmlfile_swap, 'w')
+ while 1:
+ line = f.readline()
+
+ if not line:
+ break
+
+ m = re.match('\s*</configuration>', line)
+ if m:
+ f_tmp.write(' <property>\n')
+ f_tmp.write(' <name>%s</name>\n' % property_name)
+ f_tmp.write(' <value>%s</value>\n' % property_value)
+ f_tmp.write(' </property>\n')
+ f_tmp.write('</configuration>\n')
+ else:
+ f_tmp.write(line)
+ f_tmp.close()
+
+ shutil.move(xmlfile_swap, xmlfile)
+
+
def remove_property_xml(property_name, org_config_file):
tree = ElementTree()
tree.parse(org_config_file)