You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by bh...@apache.org on 2016/03/29 15:59:30 UTC
ambari git commit: AMBARI-15595: HAWQ - Add option to exchange keys
during start or not (bhuvnesh2703)
Repository: ambari
Updated Branches:
refs/heads/trunk abca3050c -> 6d668d295
AMBARI-15595: HAWQ - Add option to exchange keys during start or not (bhuvnesh2703)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6d668d29
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6d668d29
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6d668d29
Branch: refs/heads/trunk
Commit: 6d668d29525e722b9408bef29594125e0ebd8793
Parents: abca305
Author: Bhuvnesh Chaudhary <bc...@pivotal.io>
Authored: Tue Mar 29 07:00:31 2016 -0700
Committer: Bhuvnesh Chaudhary <bc...@pivotal.io>
Committed: Tue Mar 29 07:00:31 2016 -0700
----------------------------------------------------------------------
.../HAWQ/2.0.0/configuration/hawq-env.xml | 16 +++++++-
.../HAWQ/2.0.0/package/scripts/common.py | 20 ++++++++++
.../HAWQ/2.0.0/package/scripts/hawqmaster.py | 10 ++++-
.../HAWQ/2.0.0/package/scripts/master_helper.py | 41 +++++++-------------
.../python/stacks/2.3/HAWQ/test_hawqmaster.py | 18 +--------
.../python/stacks/2.3/HAWQ/test_hawqstandby.py | 7 ----
.../python/stacks/2.3/configs/hawq_default.json | 3 +-
ambari-web/app/data/HDP2.3/site_properties.js | 9 ++++-
8 files changed, 69 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6d668d29/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-env.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-env.xml
index fc5385f..d2da5b2 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-env.xml
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-env.xml
@@ -22,7 +22,7 @@
<display-name>HAWQ System User Password</display-name>
<value></value>
<description>
- Sets the HAWQ system user password. This password is used for ssh key exchange when initializing a cluster or provisioning a new host.
+ Sets the HAWQ system user (gpadmin) password. This password is used for ssh key exchange when initializing a cluster or provisioning a new host.
Set when the HAWQ system user is created.
When changing passwords after initial creation, first update the password directly on the hosts, then enter a new hawq_password value.
</description>
@@ -31,4 +31,16 @@
</value-attributes>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>hawq_ssh_exkeys</name>
+ <display-name>Exchange SSH Keys</display-name>
+ <value>true</value>
+ <description>
+ Enable/Disable exchanging ssh keys for HAWQ system user (gpadmin) across all HAWQ nodes during start of HAWQ Master. HAWQ system user
+ must be able to login to HAWQ nodes using passwordless ssh, else HAWQ operations will fail. SSH keys must be exchanged during installation of
+ HAWQ service or when new HAWQ hosts are added. By default, ssh keys are exchanged every time with HAWQ Master start/restart.
+ For larger clusters, this may be set to false temporarily to allow faster start time. Valid values are true/false.
+ </description>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/6d668d29/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
index d634fb2..c21dc88 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
@@ -21,6 +21,7 @@ import time
import crypt
import filecmp
from resource_management.core.resources.system import Execute, Directory, File
+from resource_management.libraries.functions.default import default
from resource_management.core.logger import Logger
from resource_management.core.system import System
from resource_management.core.exceptions import Fail
@@ -29,6 +30,7 @@ import xml.etree.ElementTree as ET
import utils
import hawq_constants
+import custom_params
def setup_user():
@@ -279,6 +281,7 @@ def start_component(component_name, port, data_dir):
"""
If data directory exists start the component, else initialize the component
"""
+ __check_dfs_truncate_enforced()
if os.path.exists(os.path.join(data_dir, hawq_constants.postmaster_opts_filename)):
return utils.exec_hawq_operation(hawq_constants.START,
"{0} -a -v".format(component_name),
@@ -292,3 +295,20 @@ def stop_component(component_name, port, mode):
utils.exec_hawq_operation(hawq_constants.STOP,
"{0} -M {1} -a -v".format(component_name, mode),
only_if=utils.chk_hawq_process_status_cmd(port, component_name))
+
+def __check_dfs_truncate_enforced():
+ """
+ If enforce_hdfs_truncate is set to True:
+ throw an ERROR, HAWQ components start should fail
+ Else:
+ throw a WARNING,
+ """
+ DFS_ALLOW_TRUNCATE_WARNING_MSG = "It is recommended to set dfs.allow.truncate as true in hdfs-site.xml configuration file, currently it is set to false. Please review HAWQ installation guide for more information."
+
+ # Check if dfs.allow.truncate exists in hdfs-site.xml and throw appropriate exception if not set to True
+ dfs_allow_truncate = default("/configurations/hdfs-site/dfs.allow.truncate", None)
+ if dfs_allow_truncate is None or str(dfs_allow_truncate).lower() != 'true':
+ if custom_params.enforce_hdfs_truncate:
+ raise Fail("**ERROR**: {0}".format(DFS_ALLOW_TRUNCATE_WARNING_MSG))
+ else:
+ Logger.error("**WARNING**: {0}".format(DFS_ALLOW_TRUNCATE_WARNING_MSG))
http://git-wip-us.apache.org/repos/asf/ambari/blob/6d668d29/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py
index 7c3a832..4edf64f 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py
@@ -20,6 +20,8 @@ from resource_management import Script
from resource_management.core.resources.system import Execute
from resource_management.core.logger import Logger
from resource_management.libraries.functions.check_process_status import check_process_status
+from resource_management.libraries.functions.default import default
+from resource_management.core.source import InlineTemplate
try:
from resource_management.libraries.functions import stack_select as hadoop_select
except ImportError:
@@ -50,7 +52,11 @@ class HawqMaster(Script):
import params
self.configure(env)
common.validate_configuration()
- master_helper.setup_passwordless_ssh()
+ exchange_ssh_keys = default('/configurations/hawq-env/hawq_ssh_exkeys', None)
+ if exchange_ssh_keys is None or str(exchange_ssh_keys).lower() == 'false':
+ Logger.info("Skipping ssh key exchange with HAWQ hosts as hawq_ssh_exkeys is either set to false or is not available in hawq-env.xml")
+ else:
+ master_helper.setup_passwordless_ssh()
common.start_component(hawq_constants.MASTER, params.hawq_master_address_port, params.hawq_master_dir)
def stop(self, env):
@@ -73,7 +79,9 @@ class HawqMaster(Script):
exec_psql_cmd(cmd, params.hawqmaster_host, params.hawq_master_address_port)
def run_hawq_check(self, env):
+ import params
Logger.info("Executing HAWQ Check ...")
+ params.File(hawq_constants.hawq_hosts_file, content=InlineTemplate("{% for host in hawq_all_hosts %}{{host}}\n{% endfor %}"))
Execute("source {0} && hawq check -f {1} --hadoop {2} --config {3}".format(hawq_constants.hawq_greenplum_path_file, hawq_constants.hawq_hosts_file, hadoop_select.get_hadoop_dir('home'), hawq_constants.hawq_check_file),
user=hawq_constants.hawq_user,
timeout=hawq_constants.default_exec_timeout)
http://git-wip-us.apache.org/repos/asf/ambari/blob/6d668d29/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py
index 9d8b32d..87d3c8a 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py
@@ -22,6 +22,7 @@ from resource_management.core.source import InlineTemplate
from resource_management.core.logger import Logger
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.default import default
+from resource_management.core.exceptions import Fail
import utils
import common
@@ -39,17 +40,27 @@ def __setup_master_specific_conf_files():
params.File(hawq_constants.hawq_slaves_file,
content=InlineTemplate("{% for host in hawqsegment_hosts %}{{host}}\n{% endfor %}"))
- params.File(hawq_constants.hawq_hosts_file,
- content=InlineTemplate("{% for host in hawq_all_hosts %}{{host}}\n{% endfor %}"))
-
def setup_passwordless_ssh():
"""
Exchanges ssh keys to setup passwordless ssh for the hawq_user between the HAWQ Master and the HAWQ Segment nodes
"""
import params
- utils.exec_hawq_operation("ssh-exkeys", format('-f {hawq_hosts_file} -p {hawq_password!p}', hawq_hosts_file=hawq_constants.hawq_hosts_file, hawq_password=params.hawq_password))
+ failed_hosts = []
+ for host in params.hawq_all_hosts:
+ try:
+ utils.exec_hawq_operation("ssh-exkeys", format('-h {hawq_host} -p {hawq_password!p}', hawq_host=host, hawq_password=params.hawq_password))
+ except:
+ failed_hosts.append(host)
+
+ failed_hosts_cnt = len(failed_hosts)
+ if failed_hosts_cnt > 0:
+ DEBUG_HELP_MSG = "Please verify the logs below to debug the cause of failure."
+ if failed_hosts_cnt == len(params.hawq_all_hosts):
+ raise Fail("Setting up passwordless ssh failed for all the HAWQ hosts. {0}".format(DEBUG_HELP_MSG))
+ else:
+ Logger.error("**WARNING**: Setting up passwordless ssh failed with the hosts below, proceeding with HAWQ Master start:\n{0}\n\n{1}".format("\n".join(failed_hosts), DEBUG_HELP_MSG))
def configure_master():
"""
@@ -61,7 +72,6 @@ def configure_master():
__setup_master_specific_conf_files()
common.create_master_dir(params.hawq_master_dir)
common.create_temp_dirs(params.hawq_master_temp_dirs)
- __check_dfs_truncate_enforced()
def __setup_hdfs_dirs():
"""
@@ -80,24 +90,3 @@ def __setup_hdfs_dirs():
recursive_chown = True,
mode=0755)
params.HdfsResource(None, action="execute")
-
-def __check_dfs_truncate_enforced():
- """
- If enforce_hdfs_truncate is set to True:
- throw an ERROR, HAWQMASTER or HAWQSTANDBY start should fail
- Else:
- throw a WARNING,
- """
- import custom_params
-
- DFS_ALLOW_TRUNCATE_EXCEPTION_MESSAGE = "dfs.allow.truncate property in hdfs-site.xml configuration file should be set to True. Please review HAWQ installation guide for more information."
-
- # Check if dfs.allow.truncate exists in hdfs-site.xml and throw appropriate exception if not set to True
- dfs_allow_truncate = default('/configurations/hdfs-site/dfs.allow.truncate', None)
-
- if dfs_allow_truncate is None or str(dfs_allow_truncate).lower() != 'true':
- if custom_params.enforce_hdfs_truncate:
- Logger.error("**ERROR**: {0}".format(DFS_ALLOW_TRUNCATE_EXCEPTION_MESSAGE))
- sys.exit(1)
- else:
- Logger.warning("**WARNING**: {0}".format(DFS_ALLOW_TRUNCATE_EXCEPTION_MESSAGE))
http://git-wip-us.apache.org/repos/asf/ambari/blob/6d668d29/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqmaster.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqmaster.py b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqmaster.py
index 4dc3cc8..20624ec 100644
--- a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqmaster.py
+++ b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqmaster.py
@@ -87,13 +87,6 @@ class TestHawqMaster(RMFTestCase):
mode = 0644
)
- self.assertResourceCalled('File', '/usr/local/hawq/etc/hawq_hosts',
- content = InlineTemplate('c6401.ambari.apache.org\nc6402.ambari.apache.org\nc6403.ambari.apache.org\n\n'),
- group = self.GPADMIN,
- owner = self.GPADMIN,
- mode = 0644
- )
-
self.assertResourceCalled('Directory', '/data/hawq/master',
group = self.GPADMIN,
owner = self.GPADMIN,
@@ -113,8 +106,7 @@ class TestHawqMaster(RMFTestCase):
@patch ('hawqmaster.common.__set_osparams')
- @patch ('hawqmaster.master_helper.__check_dfs_truncate_enforced')
- def test_configure_default(self, set_dfs_truncate_mock, set_osparams_mock):
+ def test_configure_default(self, set_osparams_mock):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqmaster.py',
classname = 'HawqMaster',
@@ -155,14 +147,6 @@ class TestHawqMaster(RMFTestCase):
self.__asserts_for_configure()
- self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && hawq ssh-exkeys -f /usr/local/hawq/etc/hawq_hosts -p gpadmin',
- logoutput = True,
- not_if = None,
- only_if = None,
- user = self.GPADMIN,
- timeout = 900
- )
-
self.assertResourceCalled('HdfsResource', '/hawq_default',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
default_fs = u'hdfs://c6401.ambari.apache.org:8020',
http://git-wip-us.apache.org/repos/asf/ambari/blob/6d668d29/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqstandby.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqstandby.py b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqstandby.py
index e98b253..be5989d 100644
--- a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqstandby.py
+++ b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqstandby.py
@@ -85,13 +85,6 @@ class TestHawqStandby(RMFTestCase):
mode = 0644
)
- self.assertResourceCalled('File', '/usr/local/hawq/etc/hawq_hosts',
- content = InlineTemplate('c6401.ambari.apache.org\nc6402.ambari.apache.org\nc6403.ambari.apache.org\n\n'),
- group = self.GPADMIN,
- owner = self.GPADMIN,
- mode = 0644
- )
-
self.assertResourceCalled('Directory', '/data/hawq/master',
group = self.GPADMIN,
owner = self.GPADMIN,
http://git-wip-us.apache.org/repos/asf/ambari/blob/6d668d29/ambari-server/src/test/python/stacks/2.3/configs/hawq_default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/hawq_default.json b/ambari-server/src/test/python/stacks/2.3/configs/hawq_default.json
index 99d43cb..36513c5 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/hawq_default.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/hawq_default.json
@@ -439,7 +439,8 @@
},
"ranger-hbase-plugin-properties": {},
"hawq-env": {
- "hawq_password": "gpadmin"
+ "hawq_password": "gpadmin",
+ "hawq_ssh_exkeys": "false"
},
"core-site": {
"ipc.server.listen.queue.size": "3300",
http://git-wip-us.apache.org/repos/asf/ambari/blob/6d668d29/ambari-web/app/data/HDP2.3/site_properties.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/data/HDP2.3/site_properties.js b/ambari-web/app/data/HDP2.3/site_properties.js
index e706d40..e04d060 100644
--- a/ambari-web/app/data/HDP2.3/site_properties.js
+++ b/ambari-web/app/data/HDP2.3/site_properties.js
@@ -340,13 +340,20 @@ hdp23properties.push({
"index": 9
},
{
- "name": "hawq_password",
+ "name": "hawq_ssh_exkeys",
"filename": "hawq-env.xml",
"category": "General",
"serviceName": "HAWQ",
"index": 10
},
{
+ "name": "hawq_password",
+ "filename": "hawq-env.xml",
+ "category": "General",
+ "serviceName": "HAWQ",
+ "index": 11
+ },
+ {
"name": "content",
"serviceName": "HAWQ",
"filename": "hawq-check-env.xml",