You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vs...@apache.org on 2017/09/20 06:32:45 UTC
ambari git commit: AMBARI-21987.Store Zeppelin's interpreter.json in
HDFS(Prabhjyot Singh via Venkata Sairam)
Repository: ambari
Updated Branches:
refs/heads/branch-2.6 f49cf54e9 -> c7b9348da
AMBARI-21987.Store Zeppelin's interpreter.json in HDFS(Prabhjyot Singh via Venkata Sairam)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c7b9348d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c7b9348d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c7b9348d
Branch: refs/heads/branch-2.6
Commit: c7b9348da34ce7d78c37eb93b254cdaf396bc061
Parents: f49cf54
Author: Venkata Sairam <ve...@gmail.com>
Authored: Wed Sep 20 12:02:29 2017 +0530
Committer: Venkata Sairam <ve...@gmail.com>
Committed: Wed Sep 20 12:02:29 2017 +0530
----------------------------------------------------------------------
.../0.6.0/configuration/zeppelin-config.xml | 2 +-
.../ZEPPELIN/0.6.0/package/scripts/master.py | 2 +-
.../0.7.0/configuration/zeppelin-config.xml | 8 ++-
.../ZEPPELIN/0.7.0/package/scripts/master.py | 67 ++++++++++++++++----
4 files changed, 63 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c7b9348d/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/configuration/zeppelin-config.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/configuration/zeppelin-config.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/configuration/zeppelin-config.xml
index bd6ad76..03ad5f7 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/configuration/zeppelin-config.xml
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/configuration/zeppelin-config.xml
@@ -77,7 +77,7 @@
</property>
<property>
<name>zeppelin.notebook.storage</name>
- <value>org.apache.zeppelin.notebook.repo.HdfsNotebookRepo</value>
+ <value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</value>
<description>notebook persistence layer implementation. If S3 is used, set this to
org.apache.zeppelin.notebook.repo.S3NotebookRepo instead. If S3 is used to store the
notebooks, it is necessary to use the following folder structure
http://git-wip-us.apache.org/repos/asf/ambari/blob/c7b9348d/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/package/scripts/master.py
index ba73d10..0fdc27c 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/package/scripts/master.py
@@ -242,7 +242,7 @@ class Master(Script):
os.path.join(params.zeppelin_dir, "notebook")), sudo=True)
if 'zeppelin.notebook.storage' in params.config['configurations']['zeppelin-config'] \
- and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.HdfsNotebookRepo':
+ and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo':
self.check_and_copy_notebook_in_hdfs(params)
if params.security_enabled:
http://git-wip-us.apache.org/repos/asf/ambari/blob/c7b9348d/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml
index bd6ad76..29821f2 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml
@@ -77,7 +77,7 @@
</property>
<property>
<name>zeppelin.notebook.storage</name>
- <value>org.apache.zeppelin.notebook.repo.HdfsNotebookRepo</value>
+ <value>org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo</value>
<description>notebook persistence layer implementation. If S3 is used, set this to
org.apache.zeppelin.notebook.repo.S3NotebookRepo instead. If S3 is used to store the
notebooks, it is necessary to use the following folder structure
@@ -86,6 +86,12 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
+ <name>zeppelin.config.fs.dir</name>
+ <value>conf</value>
+ <description>Location where interpreter.json should be installed</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>zeppelin.interpreter.dir</name>
<value>interpreter</value>
<description>Interpreter implementation base directory</description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/c7b9348d/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
index ba73d10..7d44d16 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
@@ -23,6 +23,7 @@ import os
from resource_management.core import shell, sudo
from resource_management.core.logger import Logger
+from resource_management.core.exceptions import Fail
from resource_management.core.resources import Directory
from resource_management.core.resources.system import Execute, File
from resource_management.core.source import InlineTemplate
@@ -242,7 +243,7 @@ class Master(Script):
os.path.join(params.zeppelin_dir, "notebook")), sudo=True)
if 'zeppelin.notebook.storage' in params.config['configurations']['zeppelin-config'] \
- and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.HdfsNotebookRepo':
+ and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo':
self.check_and_copy_notebook_in_hdfs(params)
if params.security_enabled:
@@ -294,23 +295,49 @@ class Master(Script):
self.set_interpreter_settings(config_data)
+ def pre_upgrade_restart(self, env, upgrade_type=None):
+ Logger.info("Executing Stack Upgrade pre-restart")
+ import params
+ env.set_params(params)
+
+ if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, format_stack_version(params.version)):
+ stack_select.select_packages(params.version)
+
+ def getZeppelinConfFS(self, params):
+ hdfs_interpreter_config = params.config['configurations']['zeppelin-config']['zeppelin.config.fs.dir'] + "/interpreter.json"
+
+ if not hdfs_interpreter_config.startswith("/"):
+ hdfs_interpreter_config = "/user/" + format("{zeppelin_user}") + "/" + hdfs_interpreter_config
+
+ return hdfs_interpreter_config
+
def get_interpreter_settings(self):
import params
import json
interpreter_config = os.path.join(params.conf_dir, "interpreter.json")
+ if 'zeppelin.notebook.storage' in params.config['configurations']['zeppelin-config'] \
+ and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo':
+
+ if 'zeppelin.config.fs.dir' in params.config['configurations']['zeppelin-config']:
+ try:
+ # copy from hdfs to /etc/zeppelin/conf/interpreter.json
+ params.HdfsResource(interpreter_config,
+ type="file",
+ action="download_on_execute",
+ source=self.getZeppelinConfFS(params),
+ group=params.zeppelin_group,
+ user=params.zeppelin_user)
+ except Fail as fail:
+ if "doesn't exist" not in fail.args[0]:
+ print "Error getting interpreter.json from HDFS"
+ print fail.args
+ raise Fail
+
config_content = sudo.read_file(interpreter_config)
config_data = json.loads(config_content)
return config_data
- def pre_upgrade_restart(self, env, upgrade_type=None):
- Logger.info("Executing Stack Upgrade pre-restart")
- import params
- env.set_params(params)
-
- if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, format_stack_version(params.version)):
- stack_select.select_packages(params.version)
-
def set_interpreter_settings(self, config_data):
import params
import json
@@ -319,8 +346,19 @@ class Master(Script):
File(interpreter_config,
group=params.zeppelin_group,
owner=params.zeppelin_user,
- content=json.dumps(config_data, indent=2)
- )
+ content=json.dumps(config_data, indent=2))
+
+ if 'zeppelin.notebook.storage' in params.config['configurations']['zeppelin-config'] \
+ and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo':
+
+ if 'zeppelin.config.fs.dir' in params.config['configurations']['zeppelin-config']:
+ params.HdfsResource(self.getZeppelinConfFS(params),
+ type="file",
+ action="create_on_execute",
+ source=interpreter_config,
+ group=params.zeppelin_group,
+ user=params.zeppelin_user,
+ replace_existing_files=True)
def update_kerberos_properties(self):
import params
@@ -499,8 +537,11 @@ class Master(Script):
import params
interpreter_json = interpreter_json_template.template
- File(format("{params.conf_dir}/interpreter.json"), content=interpreter_json,
- owner=params.zeppelin_user, group=params.zeppelin_group)
+ File(format("{params.conf_dir}/interpreter.json"),
+ content=interpreter_json,
+ owner=params.zeppelin_user,
+ group=params.zeppelin_group,
+ mode=0664)
def get_zeppelin_spark_dependencies(self):
import params