You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2018/11/19 11:17:29 UTC
[ambari] branch trunk updated (72213a6 -> 5bc2e57)
This is an automated email from the ASF dual-hosted git repository.
aonishuk pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git.
from 72213a6 AMBARI-24905 Service display name on left navigation bar should be suffixed with "Client" if only client service component is present for a service
new 3da344a AMBARI-24920. LocalFS (file:///) directory creation fails (aonishuk)
new 5bc2e57 Delete hdfs_resource.py.rej
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../libraries/providers/hdfs_resource.py | 49 ++++++++++++++--------
1 file changed, 32 insertions(+), 17 deletions(-)
[ambari] 01/02: AMBARI-24920. LocalFS (file:///) directory creation
fails (aonishuk)
Posted by ao...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aonishuk pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 3da344ab8e7d878868998ee514dd0dbc99a2be55
Author: Andrew Onishuk <ao...@hortonworks.com>
AuthorDate: Mon Nov 19 12:16:10 2018 +0200
AMBARI-24920. LocalFS (file:///) directory creation fails (aonishuk)
---
.../libraries/providers/hdfs_resource.py | 49 ++++++++++++++--------
.../libraries/providers/hdfs_resource.py.rej | 11 +++++
2 files changed, 43 insertions(+), 17 deletions(-)
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
index 52b501d..33aa96a 100644
--- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
@@ -113,8 +113,14 @@ class HdfsResourceJar:
def action_delayed_for_nameservice(self, nameservice, action_name, main_resource):
resource = {}
env = Environment.get_instance()
- if not 'hdfs_files' in env.config:
- env.config['hdfs_files'] = []
+ env_dict_key = 'hdfs_files_sudo' if main_resource.create_as_root else 'hdfs_files'
+
+ if main_resource.create_as_root:
+ Logger.info("Will create {0} as root user".format(main_resource.resource.target))
+
+
+ if not env_dict_key in env.config:
+ env.config[env_dict_key] = []
# Put values in dictionary-resource
for field_name, json_field_name in RESOURCE_TO_JSON_FIELDS.iteritems():
@@ -130,22 +136,25 @@ class HdfsResourceJar:
resource['nameservice'] = nameservice
# Add resource to create
- env.config['hdfs_files'].append(resource)
+ env.config[env_dict_key].append(resource)
- def action_execute(self, main_resource):
+ def action_execute(self, main_resource, sudo=False):
env = Environment.get_instance()
+ env_dict_key = 'hdfs_files_sudo' if sudo else 'hdfs_files'
+ if not env_dict_key in env.config or not env.config[env_dict_key]:
+ return
+
# Check required parameters
- if main_resource.has_core_configs:
+ if not sudo:
main_resource.assert_parameter_is_set('user')
+ user = main_resource.resource.user
+ else:
+ user = None
+
- if not 'hdfs_files' in env.config or not env.config['hdfs_files']:
- Logger.info("No resources to create. 'create_on_execute' or 'delete_on_execute' or 'download_on_execute' wasn't triggered before this 'execute' action.")
- return
-
hadoop_bin_dir = main_resource.resource.hadoop_bin_dir
hadoop_conf_dir = main_resource.resource.hadoop_conf_dir
- user = main_resource.resource.user if main_resource.has_core_configs else None
security_enabled = main_resource.resource.security_enabled
keytab_file = main_resource.resource.keytab
kinit_path = main_resource.resource.kinit_path_local
@@ -161,18 +170,19 @@ class HdfsResourceJar:
# Write json file to disk
File(json_path,
owner = user,
- content = json.dumps(env.config['hdfs_files'])
+ content = json.dumps(env.config[env_dict_key])
)
# Execute jar to create/delete resources in hadoop
- Execute(format("hadoop --config {hadoop_conf_dir} jar {jar_path} {json_path}"),
+ Execute(('hadoop', '--config', hadoop_conf_dir, 'jar', jar_path, json_path),
user=user,
path=[hadoop_bin_dir],
logoutput=logoutput,
+ sudo=sudo,
)
# Clean
- env.config['hdfs_files'] = []
+ env.config[env_dict_key] = []
class WebHDFSCallException(Fail):
@@ -618,7 +628,8 @@ class HdfsResourceProvider(Provider):
self.has_core_configs = not is_empty(getattr(resource, 'default_fs'))
self.ignored_resources_list = HdfsResourceProvider.get_ignored_resources_list(self.resource.hdfs_resource_ignore_file)
-
+ self.create_as_root = False
+
if not self.has_core_configs:
self.webhdfs_enabled = False
self.fsType = None
@@ -670,10 +681,12 @@ class HdfsResourceProvider(Provider):
def action_delayed(self, action_name):
self.assert_parameter_is_set('type')
-
+
if self.has_core_configs:
path_protocol = urlparse(self.resource.target).scheme.lower()
default_fs_protocol = urlparse(self.resource.default_fs).scheme.lower()
+
+ self.create_as_root = path_protocol == 'file' or default_fs_protocol == 'file' and path_protocol == None
# for protocols which are different that defaultFs webhdfs will not be able to create directories
# so for them fast-hdfs-resource.jar should be used
@@ -682,6 +695,7 @@ class HdfsResourceProvider(Provider):
Logger.info("Cannot use webhdfs for {0} defaultFs = {1} has different protocol".format(self.resource.target, self.resource.default_fs))
else:
self.can_use_webhdfs = False
+ self.create_as_root = True
parsed_path = HdfsResourceProvider.parse_path(self.resource.target)
@@ -705,8 +719,9 @@ class HdfsResourceProvider(Provider):
def action_execute(self):
HdfsResourceWebHDFS().action_execute(self)
- HdfsResourceJar().action_execute(self)
-
+ HdfsResourceJar().action_execute(self, sudo=False)
+ HdfsResourceJar().action_execute(self, sudo=True)
+
def get_hdfs_resource_executor(self):
if self.can_use_webhdfs and WebHDFSUtil.is_webhdfs_available(self.webhdfs_enabled, self.fsType):
return HdfsResourceWebHDFS()
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py.rej b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py.rej
new file mode 100644
index 0000000..0220898
--- /dev/null
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py.rej
@@ -0,0 +1,11 @@
+--- ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
++++ ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
+@@ -686,7 +686,7 @@ class HdfsResourceProvider(Provider):
+ path_protocol = urlparse(self.resource.target).scheme.lower()
+ default_fs_protocol = urlparse(self.resource.default_fs).scheme.lower()
+
+- self.create_as_root = path_protocol == 'file' or default_fs_protocol == 'file' and path_protocol == None
++ self.create_as_root = path_protocol == 'file' or default_fs_protocol == 'file' and not path_protocol
+
+ # for protocols which are different that defaultFs webhdfs will not be able to create directories
+ # so for them fast-hdfs-resource.jar should be used
[ambari] 02/02: Delete hdfs_resource.py.rej
Posted by ao...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aonishuk pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 5bc2e578942ef3af46c7be86d11803a58fbb126c
Author: aonishuk <ao...@hortonworks.com>
AuthorDate: Mon Nov 19 12:34:24 2018 +0200
Delete hdfs_resource.py.rej
---
.../libraries/providers/hdfs_resource.py.rej | 11 -----------
1 file changed, 11 deletions(-)
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py.rej b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py.rej
deleted file mode 100644
index 0220898..0000000
--- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py.rej
+++ /dev/null
@@ -1,11 +0,0 @@
---- ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
-+++ ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
-@@ -686,7 +686,7 @@ class HdfsResourceProvider(Provider):
- path_protocol = urlparse(self.resource.target).scheme.lower()
- default_fs_protocol = urlparse(self.resource.default_fs).scheme.lower()
-
-- self.create_as_root = path_protocol == 'file' or default_fs_protocol == 'file' and path_protocol == None
-+ self.create_as_root = path_protocol == 'file' or default_fs_protocol == 'file' and not path_protocol
-
- # for protocols which are different that defaultFs webhdfs will not be able to create directories
- # so for them fast-hdfs-resource.jar should be used