You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by lp...@apache.org on 2017/10/12 17:27:04 UTC
[03/50] [abbrv] ambari git commit: AMBARI-22199.Zeppelin start fails
due to permission denied error during kinit(Venkata Sairam)
AMBARI-22199.Zeppelin start fails due to permission denied error during kinit(Venkata Sairam)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/33d67f3e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/33d67f3e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/33d67f3e
Branch: refs/heads/feature-branch-AMBARI-21307
Commit: 33d67f3ead24fcda4fcdb7d5920ccc76d1743657
Parents: 7e885a3
Author: Venkata Sairam <ve...@gmail.com>
Authored: Wed Oct 11 14:46:39 2017 +0530
Committer: Venkata Sairam <ve...@gmail.com>
Committed: Wed Oct 11 14:46:39 2017 +0530
----------------------------------------------------------------------
.../ZEPPELIN/0.7.0/package/scripts/master.py | 48 ++++---
.../stacks/2.6/ZEPPELIN/test_zeppelin_070.py | 140 ++++++++++++++++---
2 files changed, 147 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/33d67f3e/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
index d615d06..09944bd 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
@@ -192,8 +192,7 @@ class Master(Script):
notebook_directory = "/user/" + format("{zeppelin_user}") + "/" + \
params.config['configurations']['zeppelin-config']['zeppelin.notebook.dir']
-
- if self.is_path_exists_in_HDFS(notebook_directory, params.zeppelin_user):
+ if not self.is_path_exists_in_HDFS(notebook_directory, params.zeppelin_user):
# hdfs dfs -mkdir {notebook_directory}
params.HdfsResource(format("{notebook_directory}"),
type="directory",
@@ -231,14 +230,14 @@ class Master(Script):
Execute(("chown", "-R", format("{zeppelin_user}") + ":" + format("{zeppelin_group}"),
os.path.join(params.zeppelin_dir, "notebook")), sudo=True)
+ if params.security_enabled:
+ zeppelin_kinit_cmd = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal}; ")
+ Execute(zeppelin_kinit_cmd, user=params.zeppelin_user)
+
if 'zeppelin.notebook.storage' in params.config['configurations']['zeppelin-config'] \
and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo':
self.check_and_copy_notebook_in_hdfs(params)
- if params.security_enabled:
- zeppelin_kinit_cmd = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal}; ")
- Execute(zeppelin_kinit_cmd, user=params.zeppelin_user)
-
zeppelin_spark_dependencies = self.get_zeppelin_spark_dependencies()
if zeppelin_spark_dependencies and os.path.exists(zeppelin_spark_dependencies[0]):
self.create_zeppelin_dir(params)
@@ -292,14 +291,17 @@ class Master(Script):
if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, format_stack_version(params.version)):
stack_select.select_packages(params.version)
- def getZeppelinConfFS(self, params):
- hdfs_interpreter_config = params.config['configurations']['zeppelin-config']['zeppelin.config.fs.dir'] + "/interpreter.json"
+ def get_zeppelin_conf_FS_directory(self, params):
+ hdfs_interpreter_config = params.config['configurations']['zeppelin-config']['zeppelin.config.fs.dir']
if not hdfs_interpreter_config.startswith("/"):
hdfs_interpreter_config = "/user/" + format("{zeppelin_user}") + "/" + hdfs_interpreter_config
return hdfs_interpreter_config
+ def get_zeppelin_conf_FS(self, params):
+ return self.get_zeppelin_conf_FS_directory(params) + "/interpreter.json"
+
def is_path_exists_in_HDFS(self, path, as_user):
kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
kinit_if_needed = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal};")
@@ -325,17 +327,15 @@ class Master(Script):
and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo':
if 'zeppelin.config.fs.dir' in params.config['configurations']['zeppelin-config']:
- zeppelin_conf_fs = self.getZeppelinConfFS(params)
+ zeppelin_conf_fs = self.get_zeppelin_conf_FS(params)
if self.is_path_exists_in_HDFS(zeppelin_conf_fs, params.zeppelin_user):
# copy from hdfs to /etc/zeppelin/conf/interpreter.json
- params.HdfsResource(interpreter_config,
- type="file",
- action="download_on_execute",
- source=zeppelin_conf_fs,
- user=params.zeppelin_user,
- group=params.zeppelin_group,
- owner=params.zeppelin_user)
+ kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',None))
+ kinit_if_needed = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal};")
+ shell.call(format("rm {interpreter_config};"
+ "{kinit_if_needed} hdfs --config {hadoop_conf_dir} dfs -get {zeppelin_conf_fs} {interpreter_config}"),
+ user=params.zeppelin_user)
else:
Logger.info(format("{zeppelin_conf_fs} does not exist. Skipping upload of DFS."))
@@ -357,13 +357,23 @@ class Master(Script):
and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo':
if 'zeppelin.config.fs.dir' in params.config['configurations']['zeppelin-config']:
- params.HdfsResource(self.getZeppelinConfFS(params),
+ if not self.is_path_exists_in_HDFS(self.get_zeppelin_conf_FS_directory(params), params.zeppelin_user):
+ # hdfs dfs -mkdir {zeppelin's conf directory}
+ params.HdfsResource(self.get_zeppelin_conf_FS_directory(params),
+ type="directory",
+ action="create_on_execute",
+ owner=params.zeppelin_user,
+ recursive_chown=True,
+ recursive_chmod=True
+ )
+
+ params.HdfsResource(self.get_zeppelin_conf_FS(params),
type="file",
action="create_on_execute",
source=interpreter_config,
- user=params.zeppelin_user,
- group=params.zeppelin_group,
owner=params.zeppelin_user,
+ recursive_chown=True,
+ recursive_chmod=True,
replace_existing_files=True)
def update_kerberos_properties(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/33d67f3e/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py b/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py
index 3064880..3adb94c 100644
--- a/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py
+++ b/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py
@@ -221,6 +221,42 @@ class TestZeppelin070(RMFTestCase):
'/usr/hdp/current/zeppelin-server/notebook'),
sudo=True,
)
+ self.assertResourceCalled('HdfsResource', '/user/zeppelin/notebook',
+ hadoop_bin_dir='/usr/hdp/2.5.0.0-1235/hadoop/bin',
+ default_fs=u'hdfs://c6401.ambari.apache.org:8020',
+ hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ hdfs_site={u'a': u'b'},
+ keytab=UnknownConfigurationMock(),
+ kinit_path_local='/usr/bin/kinit',
+ user='hdfs',
+ owner='zeppelin',
+ principal_name=UnknownConfigurationMock(),
+ recursive_chown=True,
+ security_enabled=False,
+ hadoop_conf_dir='/usr/hdp/2.5.0.0-1235/hadoop/conf',
+ type='directory',
+ action=['create_on_execute'],
+ recursive_chmod=True
+ )
+ self.assertResourceCalled('HdfsResource', '/user/zeppelin/notebook',
+ hadoop_bin_dir='/usr/hdp/2.5.0.0-1235/hadoop/bin',
+ default_fs=u'hdfs://c6401.ambari.apache.org:8020',
+ hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ hdfs_site={u'a': u'b'},
+ keytab=UnknownConfigurationMock(),
+ kinit_path_local='/usr/bin/kinit',
+ user='hdfs',
+ owner='zeppelin',
+ principal_name=UnknownConfigurationMock(),
+ recursive_chown=True,
+ security_enabled=False,
+ source='/usr/hdp/current/zeppelin-server/notebook',
+ hadoop_conf_dir='/usr/hdp/2.5.0.0-1235/hadoop/conf',
+ type='directory',
+ action=['create_on_execute'],
+ recursive_chmod=True
+ )
+
self.assertResourceCalled('HdfsResource', '/user/zeppelin',
hadoop_bin_dir='/usr/hdp/2.5.0.0-1235/hadoop/bin',
default_fs=u'hdfs://c6401.ambari.apache.org:8020',
@@ -312,6 +348,25 @@ class TestZeppelin070(RMFTestCase):
)
self.assertResourceCalled('HdfsResource',
+ '/user/zeppelin/hdfs:///user/zeppelin/conf',
+ security_enabled=False,
+ hadoop_bin_dir='/usr/hdp/2.5.0.0-1235/hadoop/bin',
+ keytab=UnknownConfigurationMock(),
+ default_fs='hdfs://c6401.ambari.apache.org:8020',
+ hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ hdfs_site={u'a': u'b'},
+ kinit_path_local='/usr/bin/kinit',
+ principal_name=UnknownConfigurationMock(),
+ recursive_chown=True,
+ recursive_chmod=True,
+ owner='zeppelin',
+ user='hdfs',
+ hadoop_conf_dir='/usr/hdp/2.5.0.0-1235/hadoop/conf',
+ type='directory',
+ action=['create_on_execute'],
+ )
+
+ self.assertResourceCalled('HdfsResource',
'/user/zeppelin/hdfs:///user/zeppelin/conf/interpreter.json',
security_enabled=False,
hadoop_bin_dir='/usr/hdp/2.5.0.0-1235/hadoop/bin',
@@ -322,9 +377,10 @@ class TestZeppelin070(RMFTestCase):
hdfs_site={u'a': u'b'},
kinit_path_local='/usr/bin/kinit',
principal_name=UnknownConfigurationMock(),
- user='zeppelin',
+ recursive_chown=True,
+ recursive_chmod=True,
+ user='hdfs',
owner='zeppelin',
- group='zeppelin',
replace_existing_files=True,
hadoop_conf_dir='/usr/hdp/2.5.0.0-1235/hadoop/conf',
type='file',
@@ -337,24 +393,45 @@ class TestZeppelin070(RMFTestCase):
group='zeppelin',
)
- self.assertResourceCalled('HdfsResource', '/user/zeppelin/hdfs:///user/zeppelin/conf/interpreter.json',
- security_enabled = False,
- hadoop_bin_dir = '/usr/hdp/2.5.0.0-1235/hadoop/bin',
- keytab = UnknownConfigurationMock(),
- source = '/etc/zeppelin/conf/interpreter.json',
- default_fs = 'hdfs://c6401.ambari.apache.org:8020',
- replace_existing_files = True,
- hdfs_resource_ignore_file = '/var/lib/ambari-agent/data/.hdfs_resource_ignore',
- hdfs_site = {u'a': u'b'},
- kinit_path_local = '/usr/bin/kinit',
- principal_name = UnknownConfigurationMock(),
- user = 'zeppelin',
- owner = 'zeppelin',
- group = 'zeppelin',
- hadoop_conf_dir = '/usr/hdp/2.5.0.0-1235/hadoop/conf',
- type = 'file',
- action = ['create_on_execute'],
- )
+ self.assertResourceCalled('HdfsResource',
+ '/user/zeppelin/hdfs:///user/zeppelin/conf',
+ security_enabled=False,
+ hadoop_bin_dir='/usr/hdp/2.5.0.0-1235/hadoop/bin',
+ keytab=UnknownConfigurationMock(),
+ default_fs='hdfs://c6401.ambari.apache.org:8020',
+ recursive_chown=True,
+ recursive_chmod=True,
+ hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ hdfs_site={u'a': u'b'},
+ kinit_path_local='/usr/bin/kinit',
+ principal_name=UnknownConfigurationMock(),
+ user='hdfs',
+ owner='zeppelin',
+ hadoop_conf_dir='/usr/hdp/2.5.0.0-1235/hadoop/conf',
+ type='directory',
+ action=['create_on_execute'],
+ )
+
+ self.assertResourceCalled('HdfsResource',
+ '/user/zeppelin/hdfs:///user/zeppelin/conf/interpreter.json',
+ security_enabled=False,
+ hadoop_bin_dir='/usr/hdp/2.5.0.0-1235/hadoop/bin',
+ keytab=UnknownConfigurationMock(),
+ source='/etc/zeppelin/conf/interpreter.json',
+ default_fs='hdfs://c6401.ambari.apache.org:8020',
+ replace_existing_files=True,
+ hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ hdfs_site={u'a': u'b'},
+ kinit_path_local='/usr/bin/kinit',
+ principal_name=UnknownConfigurationMock(),
+ recursive_chmod=True,
+ recursive_chown=True,
+ user='hdfs',
+ owner='zeppelin',
+ hadoop_conf_dir='/usr/hdp/2.5.0.0-1235/hadoop/conf',
+ type='file',
+ action=['create_on_execute'],
+ )
self.assertResourceCalled('File', '/etc/zeppelin/conf/interpreter.json',
content=interpreter_json_generated.template_after_kerberos,
@@ -362,6 +439,24 @@ class TestZeppelin070(RMFTestCase):
group='zeppelin',
)
+ self.assertResourceCalled('HdfsResource', '/user/zeppelin/hdfs:///user/zeppelin/conf',
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/hdp/2.5.0.0-1235/hadoop/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ hdfs_resource_ignore_file = '/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ hdfs_site = {u'a': u'b'},
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = UnknownConfigurationMock(),
+ recursive_chown=True,
+ recursive_chmod=True,
+ user = 'hdfs',
+ owner = 'zeppelin',
+ hadoop_conf_dir = '/usr/hdp/2.5.0.0-1235/hadoop/conf',
+ type = 'directory',
+ action = ['create_on_execute'],
+ )
+
self.assertResourceCalled('HdfsResource', '/user/zeppelin/hdfs:///user/zeppelin/conf/interpreter.json',
security_enabled = False,
hadoop_bin_dir = '/usr/hdp/2.5.0.0-1235/hadoop/bin',
@@ -373,9 +468,10 @@ class TestZeppelin070(RMFTestCase):
kinit_path_local = '/usr/bin/kinit',
principal_name = UnknownConfigurationMock(),
replace_existing_files = True,
- user = 'zeppelin',
+ recursive_chown=True,
+ recursive_chmod=True,
+ user = 'hdfs',
owner = 'zeppelin',
- group = 'zeppelin',
hadoop_conf_dir = '/usr/hdp/2.5.0.0-1235/hadoop/conf',
type = 'file',
action = ['create_on_execute'],