You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2016/06/02 23:32:37 UTC
ambari git commit: AMBARI-17017. Fixing open issues for 'Hive Server
Interactive' in Kerberized environment. AMBARI-17020. Use 'llapstatus' comand
after starting llap to check its status before starting HiveServer2.
Repository: ambari
Updated Branches:
refs/heads/trunk 1d3523a5f -> 396adbedf
AMBARI-17017. Fixing open issues for 'Hive Server Interactive' in Kerberized environment. AMBARI-17020. Use 'llapstatus' comand after starting llap to check its status before starting HiveServer2.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/396adbed
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/396adbed
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/396adbed
Branch: refs/heads/trunk
Commit: 396adbedf0915ea4f12c0063f10bc5b46582b22d
Parents: 1d3523a
Author: Swapan Shridhar <ss...@hortonworks.com>
Authored: Tue May 31 12:06:01 2016 -0700
Committer: Swapan Shridhar <ss...@hortonworks.com>
Committed: Thu Jun 2 16:32:11 2016 -0700
----------------------------------------------------------------------
.../HIVE/0.12.0.2.0/kerberos.json | 5 ---
.../package/scripts/hive_interactive.py | 2 -
.../package/scripts/hive_server_interactive.py | 46 ++++++--------------
.../package/scripts/hive_service_interactive.py | 4 --
.../stacks/HDP/2.5/services/YARN/kerberos.json | 35 +++++++++++++++
5 files changed, 48 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/396adbed/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json
index a9e3840..872bfac 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json
@@ -18,11 +18,6 @@
}
},
{
- "hive-interactive-site": {
- "hive.llap.daemon.work.dirs": "/hadoop/llap/local"
- }
- },
- {
"webhcat-site": {
"templeton.kerberos.secret": "secret",
"templeton.hive.properties": "hive.metastore.local=false,hive.metastore.uris=${clusterHostInfo/hive_metastore_host|each(thrift://%s:9083, \\\\,, \\s*\\,\\s*)},hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}"
http://git-wip-us.apache.org/repos/asf/ambari/blob/396adbed/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
index 14ba0f9..b6b963b 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
@@ -101,8 +101,6 @@ def hive_interactive(name=None):
'''
remove_atlas_hook_if_exists(merged_hive_interactive_site)
- # Anything TODO for attributes
-
# Merge tez-interactive with tez-site
XmlConfig("tez-site.xml",
conf_dir = params.tez_interactive_config_dir,
http://git-wip-us.apache.org/repos/asf/ambari/blob/396adbed/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
index 77e1cd4..7de3d49 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
@@ -112,7 +112,6 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
# Check status and based on that decide on [re]starting.
# Start LLAP before Hive Server Interactive start.
- # TODO, why does LLAP have to be started before Hive Server Interactive???
status = self._llap_start(env)
if not status:
raise Fail("Skipping START of Hive Server Interactive since LLAP app couldn't be STARTED.")
@@ -129,10 +128,7 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
if params.security_enabled:
self.do_kinit()
- # TODO, why must Hive Server Interactive be stopped before LLAP???
-
# Stop Hive Interactive Server first
- # TODO : Upgrade check comes here.
hive_service_interactive('hiveserver2', action='stop')
self._llap_stop(env)
@@ -141,7 +137,8 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
import status_params
env.set_params(status_params)
- # TODO : LLAP app status check
+ # We are not doing 'llap' status check done here as part of status check for 'HSI', as 'llap' status
+ # check is a heavy weight operation.
pid_file = format("{hive_pid_dir}/{hive_interactive_pid}")
# Recursively check all existing gmetad pid files
@@ -200,11 +197,6 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
Logger.info("Starting LLAP")
LLAP_APP_NAME = 'llap0'
- # TODO, start only if not already running.
- # TODO : Currently hardcoded the params. Need to read the suggested values from hive2/hive-site.xml.
- # TODO, ensure that script works as hive from cmd when not cd'ed in /home/hive
- # Needs permission to write to hive home dir.
-
unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')
cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances {params.num_llap_nodes}"
@@ -219,10 +211,6 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
# Append args.
cmd+= format(" --args \" {llap_app_java_opts}\"")
- # TODO: Remove adding "-XX:MaxDirectMemorySize" when Driver starts appending itself.
- if params.hive_llap_io_mem_size > params.llap_heap_size:
- max_dir_mem_size = long(params.hive_llap_io_mem_size) + 256
- cmd = cmd[0:len(cmd)-1] + format(" -XX:MaxDirectMemorySize={max_dir_mem_size}m\"")
run_file_path = None
try:
@@ -250,23 +238,17 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
Logger.info(format("Run file path: {run_file_path}"))
if os.path.isfile(run_file_path):
Execute(run_file_path, user=params.hive_user)
-
- # TODO : Sleep below is not a good idea. We need to check the status of LLAP app to figure out it got
- # launched properly and is in running state. Then go ahead with Hive Interactive Server start.
- Logger.info("Sleeping for 30 secs")
- time.sleep(30)
Logger.info("Submitted LLAP app name : {0}".format(LLAP_APP_NAME))
- # TODO: Uncomment this when 'llapstatus' commands returns correct output.
- '''
+ # We need to check the status of LLAP app to figure out it got
+ # launched properly and is in running state. Then go ahead with Hive Interactive Server start.
status = self.check_llap_app_status(LLAP_APP_NAME, params.num_retries_for_checking_llap_status)
if status:
Logger.info("LLAP app '{0}' deployed successfully.".format(LLAP_APP_NAME))
return True
else:
+ Logger.error("LLAP app '{0}' deployment unsuccessful.".format(LLAP_APP_NAME))
return False
- '''
- return True
else:
raise Fail(format("Did not find run file {run_file_path}"))
except:
@@ -336,7 +318,6 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
num_retries = 2
if num_retries > 20:
num_retries = 20
-
@retry(times=num_retries, sleep_time=15, err_class=Fail)
def do_retries():
live_instances = 0
@@ -344,7 +325,6 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
percent_desired_instances_to_be_up = 80 # Used in 'RUNNING_PARTIAL' state.
llap_app_info = self._get_llap_app_status_info(llap_app_name)
-
if llap_app_info is None or 'state' not in llap_app_info:
Logger.error("Malformed JSON data received for LLAP app. Exiting ....")
return False
@@ -372,14 +352,14 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
if live_instances > 0:
percentInstancesUp = float(live_instances) / desired_instances * 100
if percentInstancesUp >= percent_desired_instances_to_be_up:
- Logger.info("Slider app '{0}' in '{1}' state. Live Instances : '{2}' >= {3}% of Desired Instances : " \
- "'{4}'".format(llap_app_name, llap_app_info['state'],
+ Logger.info("LLAP app '{0}' in '{1}' state. Live Instances : '{2}' >= {3}% of Desired Instances : " \
+ "'{4}'.".format(llap_app_name, llap_app_info['state'],
llap_app_info['liveInstances'],
percent_desired_instances_to_be_up,
llap_app_info['desiredInstances']))
return True
else:
- Logger.info("Slider app '{0}' in '{1}' state. Live Instances : '{2}'. Desired Instances : " \
+ Logger.info("LLAP app '{0}' in '{1}' state. Live Instances : '{2}'. Desired Instances : " \
"'{3}' after {4} secs.".format(llap_app_name, llap_app_info['state'],
llap_app_info['liveInstances'],
llap_app_info['desiredInstances'],
@@ -387,26 +367,26 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
raise Fail("App state is RUNNING_PARTIAL. Live Instances : '{0}', Desired Instance : '{1}'".format(llap_app_info['liveInstances'],
llap_app_info['desiredInstances']))
elif llap_app_info['state'].upper() in ['APP_NOT_FOUND', 'LAUNCHING']:
- status_str = format("Slider app '{0}' current state is {1}.".format(llap_app_name, llap_app_info['state']))
+ status_str = format("LLAP app '{0}' current state is {1}.".format(llap_app_name, llap_app_info['state']))
Logger.info(status_str)
raise Fail(status_str)
else: # Covers state "COMPLETE" and any unknown that we get.
Logger.info(
- "Slider app '{0}' current state is '{1}'. Expected : 'RUNNING'".format(llap_app_name, llap_app_info['state']))
+ "LLAP app '{0}' current state is '{1}'. Expected : 'RUNNING'.".format(llap_app_name, llap_app_info['state']))
return False
try:
status = do_retries()
return status
except Exception, e:
- Logger.info("App '{0}' did not come up after a wait of {1} seconds".format(llap_app_name,
+ Logger.info("LLAP app '{0}' did not come up after a wait of {1} seconds.".format(llap_app_name,
time.time() - curr_time))
return False
-
+
def get_log_folder(self):
import params
return params.hive_log_dir
-
+
def get_user(self):
import params
return params.hive_user
http://git-wip-us.apache.org/repos/asf/ambari/blob/396adbed/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service_interactive.py
index 908c726..053b5e5 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service_interactive.py
@@ -45,8 +45,6 @@ def hive_service_interactive(name, action='start', upgrade_type=None):
pid_file = format("{hive_pid_dir}/{hive_interactive_pid}")
cmd = format("{start_hiveserver2_interactive_path} {hive_pid_dir}/hive-server2-interactive.out {hive_log_dir}/hive-server2-interactive.err {pid_file} {hive_server_interactive_conf_dir} {hive_log_dir}")
- # TODO : Kerberos work for Hive2
-
pid = get_user_call_output.get_user_call_output(format("cat {pid_file}"), user=params.hive_user, is_checked_call=False)[1]
process_id_exists_command = format("ls {pid_file} >/dev/null 2>&1 && ps -p {pid} >/dev/null 2>&1")
@@ -56,8 +54,6 @@ def hive_service_interactive(name, action='start', upgrade_type=None):
hadoop_home = params.hadoop_home
hive_interactive_bin = "hive2"
- # TODO : Upgrade checks required here.
-
Execute(daemon_cmd,
user = params.hive_user,
environment = { 'HADOOP_HOME': hadoop_home, 'JAVA_HOME': params.java64_home, 'HIVE_BIN': hive_interactive_bin },
http://git-wip-us.apache.org/repos/asf/ambari/blob/396adbed/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json
index e22b595..e8a2887 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json
@@ -77,6 +77,41 @@
}
},
{
+ "name": "/HIVE/HIVE_SERVER/hive_server_hive",
+ "principal": {
+ "configuration": "hive-interactive-site/hive.llap.daemon.service.principal"
+ },
+ "keytab": {
+ "configuration": "hive-interactive-site/hive.llap.daemon.keytab.file"
+ },
+ "when" : {
+ "contains" : ["services", "HIVE"]
+ }
+ },
+ {
+ "name": "llap_zk_hive",
+ "principal": {
+ "value": "hive@${realm}",
+ "type" : "user",
+ "configuration": "hive-interactive-site/hive.llap.zk.sm.principal"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/hive.llap.zk.sm.keytab",
+ "owner": {
+ "name": "${yarn-env/yarn_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": "r"
+ },
+ "configuration": "hive-interactive-site/hive.llap.zk.sm.keytab.file"
+ },
+ "when" : {
+ "contains" : ["services", "HIVE"]
+ }
+ },
+ {
"name": "/spnego",
"principal": {
"configuration": "yarn-site/yarn.nodemanager.webapp.spnego-principal"