You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2016/04/11 22:33:07 UTC
ambari git commit: AMBARI-15785. HiveServerInteractive. Kerberos
Support.
Repository: ambari
Updated Branches:
refs/heads/trunk 6ea7fd750 -> 9f61db3d1
AMBARI-15785. HiveServerInteractive. Kerberos Support.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9f61db3d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9f61db3d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9f61db3d
Branch: refs/heads/trunk
Commit: 9f61db3d12921dabf7572e708206b186dc8f5ffe
Parents: 6ea7fd7
Author: sshridhar <ss...@hortonworks.com>
Authored: Sun Apr 10 20:08:42 2016 -0700
Committer: swapan <sw...@apache.org>
Committed: Mon Apr 11 13:33:02 2016 -0700
----------------------------------------------------------------------
.../package/scripts/hive_server_interactive.py | 48 +++-
.../0.12.0.2.0/package/scripts/params_linux.py | 4 +-
.../configuration/hive-interactive-site.xml | 16 ++
.../stacks/HDP/2.5/services/HIVE/kerberos.json | 125 ++++++++++
.../stacks/HDP/2.5/services/YARN/kerberos.json | 243 +++++++++++++++++++
.../stacks/HDP/2.5/services/stack_advisor.py | 9 +
6 files changed, 441 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f61db3d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
index 055296b..d909f6d 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
@@ -34,11 +34,13 @@ from resource_management.core.exceptions import Fail
from resource_management.core.logger import Logger
from ambari_commons import OSCheck, OSConst
from ambari_commons.os_family_impl import OsFamilyImpl
+from pwd import getpwnam
# Local Imports
from setup_ranger_hive import setup_ranger_hive
from hive_service_interactive import hive_service_interactive
from hive_interactive import hive_interactive
+from hive_server import HiveServerDefault
class HiveServerInteractive(Script):
@@ -74,6 +76,10 @@ class HiveServerDefault(HiveServerInteractive):
env.set_params(params)
self.configure(env)
+ if params.security_enabled:
+ # Do the security setup, internally calls do_kinit()
+ self.setup_security()
+
# TODO : We need have conditional [re]start of LLAP once "status check command" for LLAP is ready.
# Check status and based on that decide on [re]starting.
@@ -87,11 +93,15 @@ class HiveServerDefault(HiveServerInteractive):
hive_service_interactive('hiveserver2', action='start', upgrade_type=upgrade_type)
else:
Logger.info("Skipping start of Hive Server Interactive due to LLAP start issue.")
+ raise Exception("Problem starting HiveServer2")
def stop(self, env, upgrade_type=None):
import params
env.set_params(params)
+ if params.security_enabled:
+ self.do_kinit()
+
# TODO, why must Hive Server Interactive be stopped before LLAP???
# Stop Hive Interactive Server first
@@ -114,7 +124,7 @@ class HiveServerDefault(HiveServerInteractive):
pass
def security_status(self, env):
- pass
+ HiveServerDefault.security_status(env)
def restart_llap(self, env):
"""
@@ -123,6 +133,10 @@ class HiveServerDefault(HiveServerInteractive):
Logger.info("Custom Command to retart LLAP")
import params
env.set_params(params)
+
+ if params.security_enabled:
+ self.do_kinit();
+
self._llap_stop(env)
self._llap_start(env)
@@ -153,6 +167,9 @@ class HiveServerDefault(HiveServerInteractive):
message += " " + error
raise Fail(message)
+ """
+ Controls the start of LLAP.
+ """
def _llap_start(self, env, cleanup=False):
import params
env.set_params(params)
@@ -162,7 +179,14 @@ class HiveServerDefault(HiveServerInteractive):
# TODO : Currently hardcoded the params. Need to read the suggested values from hive2/hive-site.xml.
# TODO, ensure that script works as hive from cmd when not cd'ed in /homve/hive
# Needs permission to write to hive home dir.
- cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances 1 -slider-am-container-mb {slider_am_container_mb} --loglevel INFO")
+
+ cmd = ''
+ if params.security_enabled:
+ cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances 1 -slider-am-container-mb "
+ "{slider_am_container_mb} --slider-keytab-dir .slider/keytabs/{params.hive_user}/ --slider-keytab "
+ "{hive_llap_keytab_file} --slider-principal {hive_headless_keytab} --loglevel INFO")
+ else:
+ cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances 1 -slider-am-container-mb {slider_am_container_mb} --loglevel INFO")
run_file_path = None
try:
@@ -184,7 +208,6 @@ class HiveServerDefault(HiveServerInteractive):
raise Fail("Did not find run.sh file in output: " + str(output))
Logger.info(format("Run file path: {run_file_path}"))
-
if os.path.isfile(run_file_path):
Execute(run_file_path, user=params.hive_user)
@@ -193,6 +216,7 @@ class HiveServerDefault(HiveServerInteractive):
Logger.info("Sleeping for 30 secs")
time.sleep(30)
Logger.info("LLAP app deployed successfully.")
+ return True
else:
raise Fail(format("Did not find run file {run_file_path}"))
except:
@@ -207,7 +231,25 @@ class HiveServerDefault(HiveServerInteractive):
# throw the original exception
raise
+ return False
+
+ """
+ Does kinit and copies keytab for Hive/LLAP to HDFS.
+ """
+ def setup_security(self):
+ import params
+
+ self.do_kinit();
+
+ # Copy params.hive_llap_keytab_file to hdfs://<host>:<port>/user/<hive_user>/.slider/keytabs/<hive_user> , required by LLAP
+ slider_keytab_install_cmd = format("slider install-keytab --keytab {params.hive_llap_keytab_file} --folder {params.hive_user} --overwrite")
+ Execute(slider_keytab_install_cmd, user=params.hive_user)
+
+ def do_kinit(self):
+ import params
+ hive_interactive_kinit_cmd = format("{kinit_path_local} -kt {hive_server2_keytab} {hive_principal}; ")
+ Execute(hive_interactive_kinit_cmd, user=params.hive_user)
if __name__ == "__main__":
HiveServerInteractive().execute()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f61db3d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index c7d57c0..f878141 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -498,7 +498,9 @@ if has_hive_interactive:
# Tez for Hive interactive related
tez_interactive_config_dir = os.path.realpath("/etc/tez_hive2/conf")
tez_interactive_user = config['configurations']['tez-env']['tez_user']
-
+ if security_enabled:
+ hive_llap_keytab_file = config['configurations']['hive-interactive-site']['hive.llap.zk.sm.keytab.file']
+ hive_headless_keytab = config['configurations']['hive-interactive-site']['hive.llap.zk.sm.principal']
# ranger host
ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f61db3d/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
index 5830a52..ba46f00 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
@@ -472,4 +472,20 @@ limitations under the License.
<value>true</value>
</property>
+ <property>
+ <name>hive.llap.daemon.work.dirs</name>
+ <value>${yarn.nodemanager.local-dirs}</value>
+ </property>
+
+ <property>
+ <name>hive.llap.zk.sm.connectionString</name>
+ <value>localhost:2181</value>
+ <depends-on>
+ <property>
+ <type>zoo.cfg</type>
+ <name>clientPort</name>
+ </property>
+ </depends-on>
+ </property>
+
</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f61db3d/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json
new file mode 100644
index 0000000..fe0c2ec
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json
@@ -0,0 +1,125 @@
+{
+ "services": [
+ {
+ "name": "HIVE",
+ "identities": [
+ {
+ "name": "/spnego"
+ },
+ {
+ "name": "/smokeuser"
+ }
+ ],
+ "configurations": [
+ {
+ "hive-site": {
+ "hive.metastore.sasl.enabled": "true",
+ "hive.server2.authentication": "KERBEROS"
+ }
+ },
+ {
+ "webhcat-site": {
+ "templeton.kerberos.secret": "secret",
+ "templeton.hive.properties": "hive.metastore.local=false,hive.metastore.uris=${clusterHostInfo/hive_metastore_host|each(thrift://%s:9083, \\\\,, \\s*\\,\\s*)},hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}"
+ }
+ },
+ {
+ "core-site": {
+ "hadoop.proxyuser.HTTP.hosts": "${clusterHostInfo/webhcat_server_host}"
+ }
+ },
+ {
+ "hive-interactive-site": {
+ "hadoop.security.authentication": "KERBEROS",
+ "hadoop.security.authorization": "true"
+ }
+ }
+ ],
+ "components": [
+ {
+ "name": "HIVE_METASTORE",
+ "identities": [
+ {
+ "name": "/HIVE/HIVE_SEVER/hive_server_hive",
+ "principal": {
+ "configuration": "hive-site/hive.metastore.kerberos.principal"
+ },
+ "keytab": {
+ "configuration": "hive-site/hive.metastore.kerberos.keytab.file"
+ }
+ }
+ ]
+ },
+ {
+ "name": "HIVE_SERVER",
+ "identities": [
+ {
+ "name": "/HDFS/NAMENODE/hdfs"
+ },
+ {
+ "name": "hive_server_hive",
+ "principal": {
+ "value": "hive/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "hive-site/hive.server2.authentication.kerberos.principal",
+ "local_username": "${hive-env/hive_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/hive.service.keytab",
+ "owner": {
+ "name": "${hive-env/hive_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": "r"
+ },
+ "configuration": "hive-site/hive.server2.authentication.kerberos.keytab"
+ }
+ },
+ {
+ "name": "/spnego",
+ "principal": {
+ "configuration": "hive-site/hive.server2.authentication.spnego.principal"
+ },
+ "keytab": {
+ "configuration": "hive-site/hive.server2.authentication.spnego.keytab"
+ }
+ }
+ ]
+ },
+ {
+ "name": "HIVE_SERVER_INTERACTIVE",
+ "identities": [
+ {
+ "name": "/HDFS/NAMENODE/hdfs"
+ },
+ {
+ "name": "/HIVE/HIVE_SERVER/hive_server_hive"
+ },
+ {
+ "name": "/HIVE/HIVE_SERVER/spnego"
+ },
+ {
+ "name": "/YARN/NODEMANAGER/llap_zk_hive"
+ }
+ ]
+ },
+ {
+ "name": "WEBHCAT_SERVER",
+ "identities": [
+ {
+ "name": "/spnego",
+ "principal": {
+ "configuration": "webhcat-site/templeton.kerberos.principal"
+ },
+ "keytab": {
+ "configuration": "webhcat-site/templeton.kerberos.keytab"
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f61db3d/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json
new file mode 100644
index 0000000..367dfd8
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json
@@ -0,0 +1,243 @@
+{
+ "services": [
+ {
+ "name": "YARN",
+ "identities": [
+ {
+ "name": "/spnego"
+ },
+ {
+ "name": "/smokeuser"
+ }
+ ],
+ "configurations": [
+ {
+ "yarn-site": {
+ "yarn.timeline-service.enabled": "false",
+ "yarn.timeline-service.http-authentication.type": "kerberos",
+ "yarn.acl.enable": "true",
+ "yarn.timeline-service.http-authentication.signature.secret": "",
+ "yarn.timeline-service.http-authentication.signature.secret.file": "",
+ "yarn.timeline-service.http-authentication.signer.secret.provider": "",
+ "yarn.timeline-service.http-authentication.signer.secret.provider.object": "",
+ "yarn.timeline-service.http-authentication.token.validity": "",
+ "yarn.timeline-service.http-authentication.cookie.domain": "",
+ "yarn.timeline-service.http-authentication.cookie.path": "",
+ "yarn.timeline-service.http-authentication.proxyusers.*.hosts": "",
+ "yarn.timeline-service.http-authentication.proxyusers.*.users": "",
+ "yarn.timeline-service.http-authentication.proxyusers.*.groups": "",
+ "yarn.timeline-service.http-authentication.kerberos.name.rules": "",
+ "yarn.resourcemanager.proxyusers.*.groups": "",
+ "yarn.resourcemanager.proxyusers.*.hosts": "",
+ "yarn.resourcemanager.proxyusers.*.users": "",
+ "yarn.resourcemanager.proxy-user-privileges.enabled": "true",
+ "yarn.nodemanager.linux-container-executor.cgroups.mount-path": ""
+ }
+ },
+ {
+ "core-site": {
+ "hadoop.proxyuser.${yarn-env/yarn_user}.groups": "*",
+ "hadoop.proxyuser.${yarn-env/yarn_user}.hosts": "${clusterHostInfo/rm_host}"
+ }
+ }
+ ],
+ "components": [
+ {
+ "name": "NODEMANAGER",
+ "identities": [
+ {
+ "name": "nodemanager_nm",
+ "principal": {
+ "value": "nm/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "yarn-site/yarn.nodemanager.principal",
+ "local_username": "${yarn-env/yarn_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/nm.service.keytab",
+ "owner": {
+ "name": "${yarn-env/yarn_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "yarn-site/yarn.nodemanager.keytab"
+ }
+ },
+ {
+ "name": "/HIVE/HIVE_SERVER/hive_server_hive",
+ "principal": {
+ "configuration": "hive-interactive-site/hive.llap.daemon.service.principal"
+ },
+ "keytab": {
+ "configuration": "hive-interactive-site/hive.llap.daemon.keytab.file"
+ }
+ },
+ {
+ "name": "llap_zk_hive",
+ "principal": {
+ "value": "hive@${realm}",
+ "type" : "user",
+ "configuration": "hive-interactive-site/hive.llap.zk.sm.principal"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/hive.llap.zk.sm.keytab",
+ "owner": {
+ "name": "${yarn-env/yarn_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": "r"
+ },
+ "configuration": "hive-interactive-site/hive.llap.zk.sm.keytab.file"
+ }
+ },
+ {
+ "name": "/spnego",
+ "principal": {
+ "configuration": "yarn-site/yarn.nodemanager.webapp.spnego-principal"
+ },
+ "keytab": {
+ "configuration": "yarn-site/yarn.nodemanager.webapp.spnego-keytab-file"
+ }
+ }
+ ],
+ "configurations": [
+ {
+ "yarn-site": {
+ "yarn.nodemanager.container-executor.class": "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor"
+ }
+ }
+ ]
+ },
+ {
+ "name": "RESOURCEMANAGER",
+ "identities": [
+ {
+ "name": "resource_manager_rm",
+ "principal": {
+ "value": "rm/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "yarn-site/yarn.resourcemanager.principal",
+ "local_username": "${yarn-env/yarn_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/rm.service.keytab",
+ "owner": {
+ "name": "${yarn-env/yarn_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "yarn-site/yarn.resourcemanager.keytab"
+ }
+ },
+ {
+ "name": "/spnego",
+ "principal": {
+ "configuration": "yarn-site/yarn.resourcemanager.webapp.spnego-principal"
+ },
+ "keytab": {
+ "configuration": "yarn-site/yarn.resourcemanager.webapp.spnego-keytab-file"
+ }
+ }
+ ]
+ },
+ {
+ "name": "APP_TIMELINE_SERVER",
+ "identities": [
+ {
+ "name": "app_timeline_server_yarn",
+ "principal": {
+ "value": "yarn/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "yarn-site/yarn.timeline-service.principal",
+ "local_username": "${yarn-env/yarn_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/yarn.service.keytab",
+ "owner": {
+ "name": "${yarn-env/yarn_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "yarn-site/yarn.timeline-service.keytab"
+ }
+ },
+ {
+ "name": "/spnego",
+ "principal": {
+ "configuration": "yarn-site/yarn.timeline-service.http-authentication.kerberos.principal"
+ },
+ "keytab": {
+ "configuration": "yarn-site/yarn.timeline-service.http-authentication.kerberos.keytab"
+ }
+ },
+ {
+ "name": "/HDFS/NAMENODE/hdfs"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "MAPREDUCE2",
+ "identities": [
+ {
+ "name": "/spnego"
+ },
+ {
+ "name": "/smokeuser"
+ }
+ ],
+ "components": [
+ {
+ "name": "HISTORYSERVER",
+ "identities": [
+ {
+ "name": "/HDFS/NAMENODE/hdfs"
+ },
+ {
+ "name": "history_server_jhs",
+ "principal": {
+ "value": "jhs/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "mapred-site/mapreduce.jobhistory.principal",
+ "local_username": "${mapred-env/mapred_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/jhs.service.keytab",
+ "owner": {
+ "name": "${mapred-env/mapred_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "mapred-site/mapreduce.jobhistory.keytab"
+ }
+ },
+ {
+ "name": "/spnego",
+ "principal": {
+ "configuration": "mapred-site/mapreduce.jobhistory.webapp.spnego-principal"
+ },
+ "keytab": {
+ "configuration": "mapred-site/mapreduce.jobhistory.webapp.spnego-keytab-file"
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f61db3d/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 1812d05..7407da3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -121,6 +121,15 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
pass
pass
+ if 'hive-interactive-site' in services['configurations'] and \
+ services['configurations']['hive-interactive-site']['properties']['hive.llap.zk.sm.connectionString']:
+
+ # Fill the property 'hive.llap.zk.sm.connectionString' required by Hive Server Interactive (HiveServer2)
+ zookeeper_host_port = self.getZKHostPortString(services)
+ if zookeeper_host_port:
+ putHiveInteractiveSiteProperty = self.putProperty(configurations, "hive-interactive-site", services)
+ putHiveInteractiveSiteProperty("hive.llap.zk.sm.connectionString", zookeeper_host_port)
+
def recommendRangerConfigurations(self, configurations, clusterData, services, hosts):
super(HDP25StackAdvisor, self).recommendRangerConfigurations(configurations, clusterData, services, hosts)