You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2016/11/20 23:29:21 UTC
[2/2] ambari git commit: AMBARI-18943. If multiple HSI components are
deployed then HSI start should not restart llap (smohanty)
AMBARI-18943. If multiple HSI components are deployed then HSI start should not restart llap (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/795dd1f8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/795dd1f8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/795dd1f8
Branch: refs/heads/branch-2.5
Commit: 795dd1f814b8bf342a53c5a6c15ff60d568c2980
Parents: 351c20c
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sun Nov 20 15:26:38 2016 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sun Nov 20 15:26:52 2016 -0800
----------------------------------------------------------------------
.../package/scripts/hive_server_interactive.py | 22 +-
.../0.12.0.2.0/package/scripts/params_linux.py | 1 +
.../stacks/2.5/HIVE/test_hive_server_int.py | 112 ++
.../test/python/stacks/2.5/configs/hsi_ha.json | 1254 ++++++++++++++++++
4 files changed, 1387 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/795dd1f8/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
index 996b4da..2c6d4b0 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
@@ -253,9 +253,24 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
def _llap_start(self, env, cleanup=False):
import params
env.set_params(params)
+
+ LLAP_APP_NAME = 'llap0'
+
+ if params.hive_server_interactive_ha:
+ """
+ Check llap app state
+ """
+ Logger.info("HSI HA is enabled. Checking if LLAP is already running ...")
+ status = self.check_llap_app_status(LLAP_APP_NAME, 2, params.hive_server_interactive_ha)
+ if status:
+ Logger.info("LLAP app '{0}' is already running.".format(LLAP_APP_NAME))
+ return True
+ else:
+ Logger.info("LLAP app '{0}' is not running. llap will be started.".format(LLAP_APP_NAME))
+ pass
+
Logger.info("Starting LLAP")
LLAP_PACKAGE_CREATION_PATH = Script.get_tmp_dir()
- LLAP_APP_NAME = 'llap0'
unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')
@@ -447,7 +462,7 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
Parameters: llap_app_name : deployed llap app name.
num_retries : Number of retries to check the LLAP app status.
"""
- def check_llap_app_status(self, llap_app_name, num_retries):
+ def check_llap_app_status(self, llap_app_name, num_retries, return_immediately_if_stopped=False):
# counters based on various states.
curr_time = time.time()
@@ -466,6 +481,9 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
Logger.error("Malformed JSON data received for LLAP app. Exiting ....")
return False
+ if return_immediately_if_stopped and (llap_app_info['state'].upper() in ('APP_NOT_FOUND', 'COMPLETE')):
+ return False
+
if llap_app_info['state'].upper() == 'RUNNING_ALL':
Logger.info(
"LLAP app '{0}' in '{1}' state.".format(llap_app_name, llap_app_info['state']))
http://git-wip-us.apache.org/repos/asf/ambari/blob/795dd1f8/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index e214547..bba0f3b 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -312,6 +312,7 @@ hive_server_host = hive_server_hosts[0] if len(hive_server_hosts) > 0 else None
hive_server_interactive_hosts = default('/clusterHostInfo/hive_server_interactive_hosts', [])
hive_server_interactive_host = hive_server_interactive_hosts[0] if len(hive_server_interactive_hosts) > 0 else None
+hive_server_interactive_ha = True if len(hive_server_interactive_hosts) > 1 else False
# End, Common Hosts and Ports
hive_transport_mode = config['configurations']['hive-site']['hive.server2.transport.mode']
http://git-wip-us.apache.org/repos/asf/ambari/blob/795dd1f8/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
index 7477457..3221d5d 100644
--- a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
+++ b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
@@ -131,6 +131,118 @@ class TestHiveServerInteractive(RMFTestCase):
self.assertNoMoreResources()
"""
+ Tests HSI start with llap package creation output having single line.
+ Sample output : "Prepared llap-slider-05Apr2016/run.sh for running LLAP"
+ HSI HA is enabled and llap is stopped, so it will be restartd
+ """
+ #@patch("Script.get_tmp_dir()")
+ @patch("os.path.isfile")
+ @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
+ @patch("socket.socket")
+ @patch("time.sleep")
+ def test_start_hsi_ha_with_llap_single_line_output(self, sleep_mock, socket_mock, copy_to_hfds_mock, is_file_mock): #, get_tmp_dir_mock):
+ self.maxDiff = None
+ copy_to_hfds_mock.return_value = False
+ s = socket_mock.return_value
+ is_file_mock.return_value = True
+ #get_tmp_dir_mock.return_value = "/var/lib/ambari-agent/tmp"
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server_interactive.py",
+ classname="HiveServerInteractive",
+ command="start",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.5/configs/hsi_ha.json",
+ stack_version=self.STACK_VERSION,
+ target=RMFTestCase.TARGET_COMMON_SERVICES,
+ checked_call_mocks=[(0, """{
+ \"state\" : \"COMPLETED\"
+ }""", ""),
+ (0, "Prepared llap-slider-05Apr2016/run.sh for running LLAP", ""),
+ (0, """{
+ \"state\" : \"RUNNING_ALL\"
+ }""", ""),
+ (0, """{
+ \"state\" : \"RUNNING_ALL\"
+ }""", ""),
+ (0, "OK.", "")],
+ )
+
+ self.assert_configure_default()
+
+ self.assertResourceCalled('Execute',
+ '/home/hive/llap-slider-05Apr2016/run.sh',
+ logoutput= True, user='hive'
+ )
+ self.assertResourceCalled('Execute',
+ 'hive --config /usr/hdp/current/hive-server2-hive2/conf/conf.server --service metatool -updateLocation hdfs://c6401.ambari.apache.org:8020 OK.',
+ environment={'PATH': '/usr/hdp/current/hadoop-client/bin'},
+ user='hive'
+ )
+ self.assertResourceCalled('Execute',
+ '/tmp/start_hiveserver2_interactive_script /var/run/hive/hive-server2-interactive.out /var/log/hive/hive-server2-interactive.err /var/run/hive/hive-interactive.pid /usr/hdp/current/hive-server2-hive2/conf/conf.server /var/log/hive',
+ environment={'HADOOP_HOME': '/usr/hdp/current/hadoop-client',
+ 'HIVE_BIN': 'hive2',
+ 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ not_if="ls /var/run/hive/hive-interactive.pid >/dev/null 2>&1 && ps -p 123 >/dev/null 2>&1",
+ user='hive',
+ path=['/bin:/usr/hdp/current/hive-server2-hive2/bin:/usr/hdp/current/hadoop-client/bin'],
+ )
+ self.assertResourceCalled('Execute',
+ '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/hdp/current/hive-server2-hive2/lib/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\' hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
+ path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
+ tries=5,
+ try_sleep=10
+ )
+ self.assertNoMoreResources()
+
+ """
+ Tests HSI start with llap running and HA deployment. As llap is running it will not be started."
+ """
+ #@patch("Script.get_tmp_dir()")
+ @patch("os.path.isfile")
+ @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
+ @patch("socket.socket")
+ @patch("time.sleep")
+ def test_start_with_ha_and_llap_running(self, sleep_mock, socket_mock, copy_to_hfds_mock, is_file_mock): #, get_tmp_dir_mock):
+ self.maxDiff = None
+ copy_to_hfds_mock.return_value = False
+ s = socket_mock.return_value
+ is_file_mock.return_value = True
+ #get_tmp_dir_mock.return_value = "/var/lib/ambari-agent/tmp"
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server_interactive.py",
+ classname="HiveServerInteractive",
+ command="start",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.5/configs/hsi_ha.json",
+ stack_version=self.STACK_VERSION,
+ target=RMFTestCase.TARGET_COMMON_SERVICES,
+ checked_call_mocks=[(0, """{
+ \"state\" : \"RUNNING_ALL\"
+ }""", "")],
+ )
+
+ self.assert_configure_default()
+
+ self.assertResourceCalled('Execute',
+ 'hive --config /usr/hdp/current/hive-server2-hive2/conf/conf.server --service metatool -updateLocation hdfs://c6401.ambari.apache.org:8020 OK.',
+ environment={'PATH': '/usr/hdp/current/hadoop-client/bin'},
+ user='hive'
+ )
+ self.assertResourceCalled('Execute',
+ '/tmp/start_hiveserver2_interactive_script /var/run/hive/hive-server2-interactive.out /var/log/hive/hive-server2-interactive.err /var/run/hive/hive-interactive.pid /usr/hdp/current/hive-server2-hive2/conf/conf.server /var/log/hive',
+ environment={'HADOOP_HOME': '/usr/hdp/current/hadoop-client',
+ 'HIVE_BIN': 'hive2',
+ 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ not_if="ls /var/run/hive/hive-interactive.pid >/dev/null 2>&1 && ps -p 123 >/dev/null 2>&1",
+ user='hive',
+ path=['/bin:/usr/hdp/current/hive-server2-hive2/bin:/usr/hdp/current/hadoop-client/bin'],
+ )
+ self.assertResourceCalled('Execute',
+ '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/hdp/current/hive-server2-hive2/lib/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\' hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
+ path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
+ tries=5,
+ try_sleep=10
+ )
+ self.assertNoMoreResources()
+
+ """
Tests HSI start with llap package creation output having multiple lines.
Sample output : "UNWANTED STRING \n Prepared llap-slider-05Apr2016/run.sh for running LLAP \n UNWANTED STRING \n"
"""