You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rk...@apache.org on 2017/04/04 06:52:56 UTC
[1/5] ambari git commit: AMBARI-20538: zeppelin spark/spark2 failing
due to failure to initialize spark context (Prabhjyot Singh via r-kamath)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 fb05bc7ed -> c8e937afe
AMBARI-20538: zeppelin spark/spark2 failing due to failure to initialize spark context (Prabhjyot Singh via r-kamath)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f6030f57
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f6030f57
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f6030f57
Branch: refs/heads/branch-2.5
Commit: f6030f57671ddf80ee7149e269396342860149ec
Parents: fb05bc7
Author: Renjith Kamath <re...@gmail.com>
Authored: Fri Mar 24 09:48:35 2017 +0530
Committer: Renjith Kamath <re...@gmail.com>
Committed: Tue Apr 4 12:20:32 2017 +0530
----------------------------------------------------------------------
.../ZEPPELIN/0.6.0.2.5/package/scripts/master.py | 8 --------
1 file changed, 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f6030f57/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
index 65c7f2b..be2f96c 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
@@ -162,14 +162,6 @@ class Master(Script):
owner=params.zeppelin_user, group=params.zeppelin_group)
self.create_zeppelin_hdfs_conf_dir(env)
- # copy hive-site.xml only if Spark 1.x is installed
- if 'spark-defaults' in params.config['configurations'] and params.is_hive_installed:
- XmlConfig("hive-site.xml",
- conf_dir=params.external_dependency_conf,
- configurations=params.spark_hive_properties,
- owner=params.zeppelin_user,
- group=params.zeppelin_group,
- mode=0644)
if len(params.hbase_master_hosts) > 0 and params.is_hbase_installed:
# copy hbase-site.xml
[4/5] ambari git commit: AMBARI-20564: configuring Zeppelin's
JDBC-phoenix interpreter requires hdfs-site.xml,
core-site.xml as well (Prabhjyot Singh via r-kamath)
Posted by rk...@apache.org.
AMBARI-20564: configuring Zeppelin's JDBC-phoenix interpreter requires hdfs-site.xml, core-site.xml as well (Prabhjyot Singh via r-kamath)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c486a4a4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c486a4a4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c486a4a4
Branch: refs/heads/branch-2.5
Commit: c486a4a4671caffa17c5e155bc7290cc78b8d6ee
Parents: 83e738f
Author: Renjith Kamath <re...@gmail.com>
Authored: Mon Mar 27 11:09:53 2017 +0530
Committer: Renjith Kamath <re...@gmail.com>
Committed: Tue Apr 4 12:20:48 2017 +0530
----------------------------------------------------------------------
.../ZEPPELIN/0.6.0.2.5/package/scripts/master.py | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c486a4a4/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
index ee693b6..82b67fb 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
@@ -171,8 +171,23 @@ class Master(Script):
configuration_attributes=params.config['configuration_attributes']['hbase-site'],
owner=params.zeppelin_user,
group=params.zeppelin_group,
- mode=0644
- )
+ mode=0644)
+
+ XmlConfig("hdfs-site.xml",
+ conf_dir=params.external_dependency_conf,
+ configurations=params.config['configurations']['hdfs-site'],
+ configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
+ owner=params.zeppelin_user,
+ group=params.zeppelin_group,
+ mode=0644)
+
+ XmlConfig("core-site.xml",
+ conf_dir=params.external_dependency_conf,
+ configurations=params.config['configurations']['core-site'],
+ configuration_attributes=params.config['configuration_attributes']['core-site'],
+ owner=params.zeppelin_user,
+ group=params.zeppelin_group,
+ mode=0644)
def stop(self, env, upgrade_type=None):
import params
[2/5] ambari git commit: AMBARI-20539. Interpreters that are not
listed under zeppelin.interpreter.group.order in zeppelin-site.xml should not
get installed. (Prabhjyot Singh via gauravn7)
Posted by rk...@apache.org.
AMBARI-20539. Interpreters that are not listed under zeppelin.interpreter.group.order in zeppelin-site.xml should not get installed. (Prabhjyot Singh via gauravn7)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/af55ef7a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/af55ef7a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/af55ef7a
Branch: refs/heads/branch-2.5
Commit: af55ef7a8e5fe986660a7c741c18ab9e9d1cabd4
Parents: f6030f5
Author: Gaurav Nagar <gr...@gmail.com>
Authored: Fri Mar 24 19:49:38 2017 +0530
Committer: Renjith Kamath <re...@gmail.com>
Committed: Tue Apr 4 12:20:38 2017 +0530
----------------------------------------------------------------------
.../ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml | 7 +++++++
.../ZEPPELIN/0.6.0.2.5/package/scripts/master.py | 5 +++++
.../ZEPPELIN/0.6.0.2.5/package/scripts/params.py | 1 +
ambari-server/src/test/python/stacks/2.6/configs/default.json | 3 ++-
4 files changed, 15 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/af55ef7a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml
index 039c701..85f1859 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml
@@ -93,6 +93,13 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
+ <name>zeppelin.interpreter.group.order</name>
+ <value>spark,angular,jdbc,livy,md,sh</value>
+ <description>Comma separated interpreter configurations. First interpreter become default
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
<name>zeppelin.interpreter.connect.timeout</name>
<value>30000</value>
<description>Interpreter process connect timeout in msec.</description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/af55ef7a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
index be2f96c..143d618 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
@@ -301,6 +301,11 @@ class Master(Script):
config_data = self.get_interpreter_settings()
interpreter_settings = config_data['interpreterSettings']
+ for setting_key in interpreter_settings.keys():
+ interpreter = interpreter_settings[setting_key]
+ if not (interpreter['group'] in params.zeppelin_interpreter):
+ del interpreter_settings[setting_key]
+
if 'spark2-defaults' in params.config['configurations']:
spark2_config = self.get_spark2_interpreter_config()
config_id = spark2_config["id"]
http://git-wip-us.apache.org/repos/asf/ambari/blob/af55ef7a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
index 9dfdea6..8ba1528 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
@@ -84,6 +84,7 @@ stack_name = default("/hostLevelParams/stack_name", None)
# params from zeppelin-config
zeppelin_port = str(config['configurations']['zeppelin-config']['zeppelin.server.port'])
+zeppelin_interpreter = str(config['configurations']['zeppelin-config']['zeppelin.interpreter.group.order']).split(",")
# params from zeppelin-env
zeppelin_user = config['configurations']['zeppelin-env']['zeppelin_user']
http://git-wip-us.apache.org/repos/asf/ambari/blob/af55ef7a/ambari-server/src/test/python/stacks/2.6/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/configs/default.json b/ambari-server/src/test/python/stacks/2.6/configs/default.json
index a1f89ce..54000e6 100644
--- a/ambari-server/src/test/python/stacks/2.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.6/configs/default.json
@@ -335,7 +335,8 @@
"zeppelin-config": {
"zeppelin.server.port": "9995",
"zeppelin.ssl.truststore.password": "change me",
- "zeppelin.interpreters": "org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter",
+ "zeppelin.interpreters": "org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter",
+ "zeppelin.interpreter.group.order": "spark,angular,jdbc,livy,md,sh",
"zeppelin.ssl.truststore.path": "conf/truststore",
"zeppelin.notebook.dir": "notebook",
"zeppelin.ssl.keystore.password": "change me",
[3/5] ambari git commit: AMBARI-20563. Zeppelin : livy2 interpreter
is not getting created by default.(Renjith Kamath via gauravn7)
Posted by rk...@apache.org.
AMBARI-20563. Zeppelin : livy2 interpreter is not getting created by default.(Renjith Kamath via gauravn7)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/83e738fe
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/83e738fe
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/83e738fe
Branch: refs/heads/branch-2.5
Commit: 83e738fe408dc5d6420d7e14a0936288ad9a816f
Parents: af55ef7
Author: Gaurav Nagar <gr...@gmail.com>
Authored: Sun Mar 26 01:08:07 2017 +0530
Committer: Renjith Kamath <re...@gmail.com>
Committed: Tue Apr 4 12:20:43 2017 +0530
----------------------------------------------------------------------
.../common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/83e738fe/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
index 143d618..ee693b6 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
@@ -311,7 +311,7 @@ class Master(Script):
config_id = spark2_config["id"]
interpreter_settings[config_id] = spark2_config
- if 'livy2-defaults' in params.config['configurations']:
+ if params.livy2_livyserver_host:
livy2_config = self.get_livy2_interpreter_config()
config_id = livy2_config["id"]
interpreter_settings[config_id] = livy2_config
[5/5] ambari git commit: AMBARI-20539. Interpreters that are not
listed under "zeppelin.interpreter.group.order" in zeppelin-site.xml should
not get installed. (Prabhjyot Singh via r-kamath)
Posted by rk...@apache.org.
AMBARI-20539. Interpreters that are not listed under "zeppelin.interpreter.group.order" in zeppelin-site.xml should not get installed. (Prabhjyot Singh via r-kamath)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c8e937af
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c8e937af
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c8e937af
Branch: refs/heads/branch-2.5
Commit: c8e937afef980c4f879969f3b8339bb11de2083b
Parents: c486a4a
Author: Renjith Kamath <re...@gmail.com>
Authored: Mon Mar 27 15:16:00 2017 +0530
Committer: Renjith Kamath <re...@gmail.com>
Committed: Tue Apr 4 12:20:54 2017 +0530
----------------------------------------------------------------------
.../ZEPPELIN/0.6.0.2.5/package/scripts/master.py | 16 ++++++++++------
.../ZEPPELIN/0.6.0.2.5/package/scripts/params.py | 5 ++++-
2 files changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c8e937af/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
index 82b67fb..fbf2246 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
@@ -81,7 +81,7 @@ class Master(Script):
)
spark_deps_full_path = self.get_zeppelin_spark_dependencies()[0]
- spark_dep_file_name = os.path.basename(spark_deps_full_path);
+ spark_dep_file_name = os.path.basename(spark_deps_full_path)
params.HdfsResource(params.spark_jar_dir + "/" + spark_dep_file_name,
type="file",
@@ -316,11 +316,6 @@ class Master(Script):
config_data = self.get_interpreter_settings()
interpreter_settings = config_data['interpreterSettings']
- for setting_key in interpreter_settings.keys():
- interpreter = interpreter_settings[setting_key]
- if not (interpreter['group'] in params.zeppelin_interpreter):
- del interpreter_settings[setting_key]
-
if 'spark2-defaults' in params.config['configurations']:
spark2_config = self.get_spark2_interpreter_config()
config_id = spark2_config["id"]
@@ -331,6 +326,15 @@ class Master(Script):
config_id = livy2_config["id"]
interpreter_settings[config_id] = livy2_config
+ if params.zeppelin_interpreter:
+ settings_to_delete = []
+ for settings_key, interpreter in interpreter_settings.items():
+ if interpreter['group'] not in params.zeppelin_interpreter:
+ settings_to_delete.append(settings_key)
+
+ for key in settings_to_delete:
+ del interpreter_settings[key]
+
hive_interactive_properties_key = 'hive_interactive'
for setting_key in interpreter_settings.keys():
interpreter = interpreter_settings[setting_key]
http://git-wip-us.apache.org/repos/asf/ambari/blob/c8e937af/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
index 8ba1528..fe3bbfb 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
@@ -84,7 +84,10 @@ stack_name = default("/hostLevelParams/stack_name", None)
# params from zeppelin-config
zeppelin_port = str(config['configurations']['zeppelin-config']['zeppelin.server.port'])
-zeppelin_interpreter = str(config['configurations']['zeppelin-config']['zeppelin.interpreter.group.order']).split(",")
+zeppelin_interpreter = None
+if 'zeppelin.interpreter.group.order' in config['configurations']['zeppelin-config']:
+ zeppelin_interpreter = str(config['configurations']['zeppelin-config']
+ ['zeppelin.interpreter.group.order']).split(",")
# params from zeppelin-env
zeppelin_user = config['configurations']['zeppelin-env']['zeppelin_user']