You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by lp...@apache.org on 2017/10/12 17:27:21 UTC
[20/50] [abbrv] ambari git commit: AMBARI-22212.Alter logic for
storing interpreter.json(Prabhjyot Singh via Venkata Sairam)
AMBARI-22212.Alter logic for storing interpreter.json(Prabhjyot Singh via Venkata Sairam)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/be605cbf
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/be605cbf
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/be605cbf
Branch: refs/heads/feature-branch-AMBARI-21307
Commit: be605cbfdc977ff927df89fc772b8fbb1f589fce
Parents: 3c3b1b8
Author: Venkata Sairam <ve...@gmail.com>
Authored: Thu Oct 12 17:17:07 2017 +0530
Committer: Venkata Sairam <ve...@gmail.com>
Committed: Thu Oct 12 17:17:07 2017 +0530
----------------------------------------------------------------------
.../ZEPPELIN/0.7.0/package/scripts/master.py | 69 +++++++++-----------
.../ZEPPELIN/0.7.0/package/scripts/params.py | 5 ++
.../stacks/2.6/ZEPPELIN/test_zeppelin_070.py | 52 +++------------
3 files changed, 44 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be605cbf/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
index eaa2cb9..a8b1b32 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
@@ -242,6 +242,17 @@ class Master(Script):
if zeppelin_spark_dependencies and os.path.exists(zeppelin_spark_dependencies[0]):
self.create_zeppelin_dir(params)
+ if params.conf_stored_in_hdfs:
+ if not self.is_path_exists_in_HDFS(self.get_zeppelin_conf_FS_directory(params), params.zeppelin_user):
+ # hdfs dfs -mkdir {zeppelin's conf directory}
+ params.HdfsResource(self.get_zeppelin_conf_FS_directory(params),
+ type="directory",
+ action="create_on_execute",
+ owner=params.zeppelin_user,
+ recursive_chown=True,
+ recursive_chmod=True
+ )
+
# if first_setup:
if not glob.glob(params.conf_dir + "/interpreter.json") and \
not os.path.exists(params.conf_dir + "/interpreter.json"):
@@ -323,21 +334,16 @@ class Master(Script):
import json
interpreter_config = os.path.join(params.conf_dir, "interpreter.json")
- if 'zeppelin.notebook.storage' in params.config['configurations']['zeppelin-config'] \
- and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo':
-
- if 'zeppelin.config.fs.dir' in params.config['configurations']['zeppelin-config']:
- zeppelin_conf_fs = self.get_zeppelin_conf_FS(params)
-
- if self.is_path_exists_in_HDFS(zeppelin_conf_fs, params.zeppelin_user):
- # copy from hdfs to /etc/zeppelin/conf/interpreter.json
- kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',None))
- kinit_if_needed = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal};")
- shell.call(format("rm {interpreter_config};"
- "{kinit_if_needed} hdfs --config {hadoop_conf_dir} dfs -get {zeppelin_conf_fs} {interpreter_config}"),
- user=params.zeppelin_user)
- else:
- Logger.info(format("{zeppelin_conf_fs} does not exist. Skipping upload of DFS."))
+ if params.conf_stored_in_hdfs:
+ zeppelin_conf_fs = self.get_zeppelin_conf_FS(params)
+
+ if self.is_path_exists_in_HDFS(zeppelin_conf_fs, params.zeppelin_user):
+ # copy from hdfs to /etc/zeppelin/conf/interpreter.json
+ kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',None))
+ kinit_if_needed = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal};")
+ shell.call(format("rm {interpreter_config};"
+ "{kinit_if_needed} hdfs --config {hadoop_conf_dir} dfs -get {zeppelin_conf_fs} {interpreter_config}"),
+ user=params.zeppelin_user)
config_content = sudo.read_file(interpreter_config)
config_data = json.loads(config_content)
@@ -353,28 +359,15 @@ class Master(Script):
owner=params.zeppelin_user,
content=json.dumps(config_data, indent=2))
- if 'zeppelin.notebook.storage' in params.config['configurations']['zeppelin-config'] \
- and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo':
-
- if 'zeppelin.config.fs.dir' in params.config['configurations']['zeppelin-config']:
- if not self.is_path_exists_in_HDFS(self.get_zeppelin_conf_FS_directory(params), params.zeppelin_user):
- # hdfs dfs -mkdir {zeppelin's conf directory}
- params.HdfsResource(self.get_zeppelin_conf_FS_directory(params),
- type="directory",
- action="create_on_execute",
- owner=params.zeppelin_user,
- recursive_chown=True,
- recursive_chmod=True
- )
-
- params.HdfsResource(self.get_zeppelin_conf_FS(params),
- type="file",
- action="create_on_execute",
- source=interpreter_config,
- owner=params.zeppelin_user,
- recursive_chown=True,
- recursive_chmod=True,
- replace_existing_files=True)
+ if params.conf_stored_in_hdfs:
+ params.HdfsResource(self.get_zeppelin_conf_FS(params),
+ type="file",
+ action="create_on_execute",
+ source=interpreter_config,
+ owner=params.zeppelin_user,
+ recursive_chown=True,
+ recursive_chmod=True,
+ replace_existing_files=True)
def update_kerberos_properties(self):
import params
@@ -451,7 +444,7 @@ class Master(Script):
hive_interactive_properties_key = 'hive_interactive'
for setting_key in interpreter_settings.keys():
interpreter = interpreter_settings[setting_key]
- if interpreter['group'] == 'jdbc':
+ if interpreter['group'] == 'jdbc' and interpreter['name'] == 'jdbc':
interpreter['dependencies'] = []
if not params.hive_server_host and params.hive_server_interactive_hosts:
http://git-wip-us.apache.org/repos/asf/ambari/blob/be605cbf/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/params.py
index b7d3df0..e69037c 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/params.py
@@ -104,6 +104,11 @@ conf_dir = "/etc/zeppelin/conf"
external_dependency_conf = "/etc/zeppelin/conf/external-dependency-conf"
notebook_dir = os.path.join(*[install_dir, zeppelin_dirname, 'notebook'])
+conf_stored_in_hdfs = False
+if 'zeppelin.config.fs.dir' in config['configurations']['zeppelin-config'] and \
+ not config['configurations']['zeppelin-config']['zeppelin.config.fs.dir'].startswith('file://'):
+ conf_stored_in_hdfs = True
+
# zeppelin-env.sh
zeppelin_env_content = config['configurations']['zeppelin-env']['zeppelin_env_content']
http://git-wip-us.apache.org/repos/asf/ambari/blob/be605cbf/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py b/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py
index 3adb94c..e5d0240 100644
--- a/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py
+++ b/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py
@@ -341,12 +341,6 @@ class TestZeppelin070(RMFTestCase):
security_enabled=False,
)
- self.assertResourceCalled('File', '/etc/zeppelin/conf/interpreter.json',
- content=interpreter_json_generated.template_after_base,
- owner='zeppelin',
- group='zeppelin',
- )
-
self.assertResourceCalled('HdfsResource',
'/user/zeppelin/hdfs:///user/zeppelin/conf',
security_enabled=False,
@@ -359,13 +353,20 @@ class TestZeppelin070(RMFTestCase):
principal_name=UnknownConfigurationMock(),
recursive_chown=True,
recursive_chmod=True,
- owner='zeppelin',
user='hdfs',
+ owner='zeppelin',
hadoop_conf_dir='/usr/hdp/2.5.0.0-1235/hadoop/conf',
type='directory',
action=['create_on_execute'],
)
+ self.assertResourceCalled('File', '/etc/zeppelin/conf/interpreter.json',
+ content=interpreter_json_generated.template_after_base,
+ owner='zeppelin',
+ group='zeppelin',
+ )
+
+
self.assertResourceCalled('HdfsResource',
'/user/zeppelin/hdfs:///user/zeppelin/conf/interpreter.json',
security_enabled=False,
@@ -394,25 +395,6 @@ class TestZeppelin070(RMFTestCase):
)
self.assertResourceCalled('HdfsResource',
- '/user/zeppelin/hdfs:///user/zeppelin/conf',
- security_enabled=False,
- hadoop_bin_dir='/usr/hdp/2.5.0.0-1235/hadoop/bin',
- keytab=UnknownConfigurationMock(),
- default_fs='hdfs://c6401.ambari.apache.org:8020',
- recursive_chown=True,
- recursive_chmod=True,
- hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
- hdfs_site={u'a': u'b'},
- kinit_path_local='/usr/bin/kinit',
- principal_name=UnknownConfigurationMock(),
- user='hdfs',
- owner='zeppelin',
- hadoop_conf_dir='/usr/hdp/2.5.0.0-1235/hadoop/conf',
- type='directory',
- action=['create_on_execute'],
- )
-
- self.assertResourceCalled('HdfsResource',
'/user/zeppelin/hdfs:///user/zeppelin/conf/interpreter.json',
security_enabled=False,
hadoop_bin_dir='/usr/hdp/2.5.0.0-1235/hadoop/bin',
@@ -439,24 +421,6 @@ class TestZeppelin070(RMFTestCase):
group='zeppelin',
)
- self.assertResourceCalled('HdfsResource', '/user/zeppelin/hdfs:///user/zeppelin/conf',
- security_enabled = False,
- hadoop_bin_dir = '/usr/hdp/2.5.0.0-1235/hadoop/bin',
- keytab = UnknownConfigurationMock(),
- default_fs = 'hdfs://c6401.ambari.apache.org:8020',
- hdfs_resource_ignore_file = '/var/lib/ambari-agent/data/.hdfs_resource_ignore',
- hdfs_site = {u'a': u'b'},
- kinit_path_local = '/usr/bin/kinit',
- principal_name = UnknownConfigurationMock(),
- recursive_chown=True,
- recursive_chmod=True,
- user = 'hdfs',
- owner = 'zeppelin',
- hadoop_conf_dir = '/usr/hdp/2.5.0.0-1235/hadoop/conf',
- type = 'directory',
- action = ['create_on_execute'],
- )
-
self.assertResourceCalled('HdfsResource', '/user/zeppelin/hdfs:///user/zeppelin/conf/interpreter.json',
security_enabled = False,
hadoop_bin_dir = '/usr/hdp/2.5.0.0-1235/hadoop/bin',