You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by pa...@apache.org on 2016/06/29 15:08:24 UTC
ambari git commit: AMBARI-17406 Update Zeppelin service definition
(Renjith Kamath via pallavkul)
Repository: ambari
Updated Branches:
refs/heads/trunk e63317d8a -> f9c89885e
AMBARI-17406 Update Zeppelin service definition (Renjith Kamath via pallavkul)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f9c89885
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f9c89885
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f9c89885
Branch: refs/heads/trunk
Commit: f9c89885eae11163bcf471593bd35698f70eaeb0
Parents: e63317d
Author: Pallav Kulshreshtha <pa...@gmail.com>
Authored: Wed Jun 29 20:37:37 2016 +0530
Committer: Pallav Kulshreshtha <pa...@gmail.com>
Committed: Wed Jun 29 20:37:37 2016 +0530
----------------------------------------------------------------------
.../0.6.0.2.5/configuration/zeppelin-env.xml | 16 ++--
.../0.6.0.2.5/package/scripts/master.py | 82 +++++++++++++-------
.../0.6.0.2.5/package/scripts/params.py | 10 ++-
3 files changed, 75 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9c89885/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml
index 844ae98..ddca230 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml
@@ -168,10 +168,10 @@ export SPARK_YARN_USER_ENV="PYTHONPATH=${PYTHONPATH}"
#ldapRealm.userDnTemplate = uid={0},cn=users,cn=accounts,dc=hortonworks,dc=com
#ldapRealm.contextFactory.url = ldap://ldaphost:389
#ldapRealm.contextFactory.authenticationMechanism = SIMPLE
-sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager
-securityManager.sessionManager = $sessionManager
+#sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager
+#securityManager.sessionManager = $sessionManager
# 86,400,000 milliseconds = 24 hour
-securityManager.sessionManager.globalSessionTimeout = 86400000
+#securityManager.sessionManager.globalSessionTimeout = 86400000
shiro.loginUrl = /api/login
[urls]
@@ -224,7 +224,10 @@ log4j.appender.dailyfile.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m
<property>
<name>zeppelin.server.kerberos.principal</name>
- <value>none</value>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
<description>
Kerberos principal name for the Zeppelin.
</description>
@@ -232,7 +235,10 @@ log4j.appender.dailyfile.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m
</property>
<property>
<name>zeppelin.server.kerberos.keytab</name>
- <value>none</value>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
<description>
Location of the kerberos keytab file for the Zeppelin.
</description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9c89885/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
index f2899d0..fd6cbb6 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
@@ -161,8 +161,11 @@ class Master(Script):
not os.path.exists(params.conf_dir + "/interpreter.json"):
Execute(params.zeppelin_dir + '/bin/zeppelin-daemon.sh start >> '
+ params.zeppelin_log_file, user=params.zeppelin_user)
- time.sleep(20)
- self.update_zeppelin_interpreter()
+ time.sleep(20)
+ self.update_zeppelin_interpreter()
+
+ if params.security_enabled:
+ self.update_kerberos_properties()
Execute(params.zeppelin_dir + '/bin/zeppelin-daemon.sh restart >> '
+ params.zeppelin_log_file, user=params.zeppelin_user)
@@ -183,7 +186,7 @@ class Master(Script):
pid_file = ''
check_process_status(pid_file)
- def update_zeppelin_interpreter(self):
+ def get_interpreter_settings(self):
import params
import json
@@ -191,39 +194,66 @@ class Master(Script):
interpreter_config_file = open(interpreter_config, "r")
config_data = json.load(interpreter_config_file)
interpreter_config_file.close()
+ return config_data
+
+ def set_interpreter_settings(self, config_data):
+ import params
+ import json
+
+ interpreter_config = params.conf_dir + "/interpreter.json"
+ interpreter_config_file = open(interpreter_config, "w+")
+ interpreter_config_file.write(json.dumps(config_data, indent=2))
+ interpreter_config_file.close()
+
+ def update_kerberos_properties(self):
+ import params
+ config_data = self.get_interpreter_settings()
+ interpreter_settings = config_data['interpreterSettings']
+ for notebooks in interpreter_settings:
+ notebook = interpreter_settings[notebooks]
+ if notebook['group'] == 'livy' and params.livy_livyserver_host:
+ if params.zeppelin_kerberos_principal and params.zeppelin_kerberos_keytab:
+ notebook['properties']['zeppelin.livy.principal'] = params.zeppelin_kerberos_principal
+ notebook['properties']['zeppelin.livy.keytab'] = params.zeppelin_kerberos_keytab
+ elif notebook['group'] == 'spark':
+ if params.zeppelin_kerberos_principal and params.zeppelin_kerberos_keytab:
+ notebook['properties']['spark.yarn.principal'] = params.zeppelin_kerberos_principal
+ notebook['properties']['spark.yarn.keytab'] = params.zeppelin_kerberos_keytab
+
+ self.set_interpreter_settings(config_data)
+
+ def update_zeppelin_interpreter(self):
+ import params
+ config_data = self.get_interpreter_settings()
interpreter_settings = config_data['interpreterSettings']
for notebooks in interpreter_settings:
notebook = interpreter_settings[notebooks]
- if notebook['group'] == 'jdbc' and params.hive_server_host:
- notebook['properties']['default.url'] = 'jdbc:hive2://' +\
- params.hive_server_host +\
- ':' + params.hive_server_port
- notebook['properties']['default.driver'] = "org.apache.hive.jdbc.HiveDriver"
+ if notebook['group'] == 'jdbc':
notebook['dependencies'] = []
- notebook['dependencies'].append(
- {"groupArtifactVersion": "org.apache.hive:hive-jdbc:2.0.1", "local": "false"})
- notebook['dependencies'].append(
- {"groupArtifactVersion": "org.apache.hadoop:hadoop-common:2.7.2", "local": "false"})
- elif notebook['group'] == 'phoenix' and params.zookeeper_znode_parent \
- and params.hbase_zookeeper_quorum:
- notebook['properties']['phoenix.jdbc.url'] = "jdbc:phoenix:" +\
- params.hbase_zookeeper_quorum + ':' +\
- params.zookeeper_znode_parent
+ if params.hive_server_host:
+ notebook['properties']['hive.url'] = 'jdbc:hive2://' + \
+ params.hive_server_host + \
+ ':' + params.hive_server_port
+ notebook['dependencies'].append(
+ {"groupArtifactVersion": "org.apache.hive:hive-jdbc:2.0.1", "local": "false"})
+ notebook['dependencies'].append(
+ {"groupArtifactVersion": "org.apache.hadoop:hadoop-common:2.7.2", "local": "false"})
+
+ elif params.zookeeper_znode_parent \
+ and params.hbase_zookeeper_quorum:
+ notebook['properties']['phoenix.url'] = "jdbc:phoenix:" + \
+ params.hbase_zookeeper_quorum + ':' + \
+ params.zookeeper_znode_parent
+ notebook['dependencies'].append(
+ {"groupArtifactVersion": "org.apache.phoenix:phoenix-core:4.4.0-HBase-1.0", "local": "false"})
elif notebook['group'] == 'livy' and params.livy_livyserver_host:
notebook['properties']['livy.spark.master'] = "yarn-cluster"
- notebook['properties']['zeppelin.livy.principal'] = params.zeppelin_kerberos_principal
- notebook['properties']['zeppelin.livy.keytab'] = params.zeppelin_kerberos_keytab
notebook['properties']['zeppelin.livy.url'] = "http://" + params.livy_livyserver_host +\
":" + params.livy_livyserver_port
elif notebook['group'] == 'spark':
- notebook['properties']['master'] = "yarn-cluster"
- notebook['properties']['spark.yarn.principal'] = params.zeppelin_kerberos_principal
- notebook['properties']['spark.yarn.keytab'] = params.zeppelin_kerberos_keytab
-
- interpreter_config_file = open(interpreter_config, "w+")
- interpreter_config_file.write(json.dumps(config_data, indent=2))
- interpreter_config_file.close()
+ notebook['properties']['master'] = "yarn-client"
+ self.set_interpreter_settings(config_data)
if __name__ == "__main__":
Master().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9c89885/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
index 770f8f5..936df81 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
@@ -138,9 +138,15 @@ full_stack_version = default("/commandParams/version", None)
spark_client_version = get_stack_version('spark-client')
-if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY, stack_version_formatted):
- livy_livyserver_host = str(default("/clusterHostInfo/livy_server_hosts", [])[0])
+livy_hosts = default("/clusterHostInfo/livy_server_hosts", [])
+
+if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY, stack_version_formatted) and \
+ len(livy_hosts) != 0:
+ livy_livyserver_host = str(livy_hosts[0])
livy_livyserver_port = config['configurations']['livy-conf']['livy.server.port']
+else:
+ livy_livyserver_host = None
+ livy_livyserver_port = None
hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
security_enabled = config['configurations']['cluster-env']['security_enabled']