You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vs...@apache.org on 2017/09/04 09:15:31 UTC
ambari git commit: AMBARI-21835.Support for Zeppelin notebook storage
in HDFS(Prabhjyot Singh via Venkata Sairam)
Repository: ambari
Updated Branches:
refs/heads/trunk 7e222173b -> a679281f7
AMBARI-21835.Support for Zeppelin notebook storage in HDFS(Prabhjyot Singh via Venkata Sairam)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a679281f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a679281f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a679281f
Branch: refs/heads/trunk
Commit: a679281f7abe6768e0896e32df7c298b59877064
Parents: 7e22217
Author: Venkata Sairam <ve...@gmail.com>
Authored: Mon Sep 4 14:44:25 2017 +0530
Committer: Venkata Sairam <ve...@gmail.com>
Committed: Mon Sep 4 14:45:22 2017 +0530
----------------------------------------------------------------------
.../0.6.0.2.5/configuration/zeppelin-config.xml | 2 +-
.../0.6.0.2.5/package/scripts/master.py | 59 +++++++++++++++++---
.../0.6.0.2.5/package/scripts/params.py | 2 +
.../0.6.0.3.0/configuration/zeppelin-config.xml | 2 +-
4 files changed, 55 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a679281f/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml
index 03ad5f7..bd6ad76 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml
@@ -77,7 +77,7 @@
</property>
<property>
<name>zeppelin.notebook.storage</name>
- <value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</value>
+ <value>org.apache.zeppelin.notebook.repo.HdfsNotebookRepo</value>
<description>notebook persistence layer implementation. If S3 is used, set this to
org.apache.zeppelin.notebook.repo.S3NotebookRepo instead. If S3 is used to store the
notebooks, it is necessary to use the following folder structure
http://git-wip-us.apache.org/repos/asf/ambari/blob/a679281f/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
index 2142bb4..ba73d10 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
@@ -20,25 +20,25 @@ limitations under the License.
import glob
import os
-from resource_management.core.base import Fail
+
+from resource_management.core import shell, sudo
+from resource_management.core.logger import Logger
from resource_management.core.resources import Directory
from resource_management.core.resources.system import Execute, File
from resource_management.core.source import InlineTemplate
-from resource_management.core import sudo
-from resource_management.core.logger import Logger
-from resource_management.core.source import StaticFile
from resource_management.libraries import XmlConfig
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions import stack_select
from resource_management.libraries.functions.check_process_status import check_process_status
+from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.format import format
-from resource_management.libraries.functions import stack_select
-from resource_management.libraries.functions import StackFeature
-from resource_management.libraries.functions.decorator import retry
from resource_management.libraries.functions.stack_features import check_stack_feature
from resource_management.libraries.functions.version import format_stack_version
from resource_management.libraries.script.script import Script
-class Master(Script):
+class Master(Script):
def install(self, env):
import params
env.set_params(params)
@@ -185,6 +185,45 @@ class Master(Script):
group=params.zeppelin_group,
mode=0644)
+ def check_and_copy_notebook_in_hdfs(self, params):
+ if params.config['configurations']['zeppelin-config']['zeppelin.notebook.dir'].startswith("/"):
+ notebook_directory = params.config['configurations']['zeppelin-config']['zeppelin.notebook.dir']
+ else:
+ notebook_directory = "/user/" + format("{zeppelin_user}") + "/" + \
+ params.config['configurations']['zeppelin-config']['zeppelin.notebook.dir']
+
+ kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+ kinit_if_needed = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal};")
+
+ notebook_directory_exists = shell.call(format("{kinit_if_needed} hdfs --config {hadoop_conf_dir} dfs -test -e {notebook_directory};echo $?"),
+ user=params.zeppelin_user)[1]
+
+ #if there is no kerberos setup then the string will contain "-bash: kinit: command not found"
+ if "\n" in notebook_directory_exists:
+ notebook_directory_exists = notebook_directory_exists.split("\n")[1]
+
+ # '1' means it does not exists
+ if notebook_directory_exists == '1':
+ # hdfs dfs -mkdir {notebook_directory}
+ params.HdfsResource(format("{notebook_directory}"),
+ type="directory",
+ action="create_on_execute",
+ owner=params.zeppelin_user,
+ recursive_chown=True,
+ recursive_chmod=True
+ )
+
+ # hdfs dfs -put /usr/hdp/current/zeppelin-server/notebook/ {notebook_directory}
+ params.HdfsResource(format("{notebook_directory}"),
+ type="directory",
+ action="create_on_execute",
+ source=params.notebook_dir,
+ owner=params.zeppelin_user,
+ recursive_chown=True,
+ recursive_chmod=True
+ )
+
+
def stop(self, env, upgrade_type=None):
import params
self.create_zeppelin_log_dir(env)
@@ -202,6 +241,10 @@ class Master(Script):
Execute(("chown", "-R", format("{zeppelin_user}") + ":" + format("{zeppelin_group}"),
os.path.join(params.zeppelin_dir, "notebook")), sudo=True)
+ if 'zeppelin.notebook.storage' in params.config['configurations']['zeppelin-config'] \
+ and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.HdfsNotebookRepo':
+ self.check_and_copy_notebook_in_hdfs(params)
+
if params.security_enabled:
zeppelin_kinit_cmd = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal}; ")
Execute(zeppelin_kinit_cmd, user=params.zeppelin_user)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a679281f/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
index f5a2a37..3242f26 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
@@ -62,6 +62,8 @@ executor_mem = config['configurations']['zeppelin-env']['zeppelin.executor.mem']
executor_instances = config['configurations']['zeppelin-env'][
'zeppelin.executor.instances']
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+
spark_jar_dir = config['configurations']['zeppelin-env']['zeppelin.spark.jar.dir']
spark_jar = format("{spark_jar_dir}/zeppelin-spark-0.5.5-SNAPSHOT.jar")
setup_view = True
http://git-wip-us.apache.org/repos/asf/ambari/blob/a679281f/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.3.0/configuration/zeppelin-config.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.3.0/configuration/zeppelin-config.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.3.0/configuration/zeppelin-config.xml
index 2bfc0dc..ca6b295 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.3.0/configuration/zeppelin-config.xml
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.3.0/configuration/zeppelin-config.xml
@@ -77,7 +77,7 @@
</property>
<property>
<name>zeppelin.notebook.storage</name>
- <value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</value>
+ <value>org.apache.zeppelin.notebook.repo.HdfsNotebookRepo</value>
<description>notebook persistence layer implementation. If S3 is used, set this to
org.apache.zeppelin.notebook.repo.S3NotebookRepo instead. If S3 is used to store the
notebooks, it is necessary to use the following folder structure