You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2016/01/14 11:45:06 UTC
[1/2] ambari git commit: AMBARI-14660. HistoryServer upgrade times
out when /app-logs is too large (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/branch-2.1 c17e206f8 -> 2596e648f
refs/heads/trunk 99b800c59 -> d5d07e767
AMBARI-14660. HistoryServer upgrade times out when /app-logs is too large (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d5d07e76
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d5d07e76
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d5d07e76
Branch: refs/heads/trunk
Commit: d5d07e767f51b694bd068d82e1f1c52796b12303
Parents: 99b800c
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Jan 14 12:44:44 2016 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Jan 14 12:44:44 2016 +0200
----------------------------------------------------------------------
.../libraries/providers/hdfs_resource.py | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d5d07e76/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
index 731bce7..71c4d5a 100644
--- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
@@ -218,6 +218,12 @@ class HdfsResourceWebHDFS:
Since it's not available on non-hdfs FS and also can be disabled in scope of HDFS.
We should still have the other implementations for such a cases.
"""
+
+ # if we have more than this count of files to recursively chmod/chown
+ # webhdfs won't be used, but 'hadoop fs -chmod (or chown) -R ..' As it can really slow.
+ # (in one second ~17 files can be chmoded)
+ MAX_FILES_FOR_RECURSIVE_ACTION_VIA_WEBHDFS = 1000
+
def action_execute(self, main_resource):
pass
@@ -344,6 +350,12 @@ class HdfsResourceWebHDFS:
if self.main_resource.resource.recursive_chown:
self._fill_directories_list(self.main_resource.resource.target, results)
+
+ # if we don't do this, we can end up waiting real long, having a big result list.
+ if len(results) > HdfsResourceWebHDFS.MAX_FILES_FOR_RECURSIVE_ACTION_VIA_WEBHDFS:
+ shell.checked_call(["hadoop", "fs", "-chown", "-R", format("{owner}:{group}"), self.main_resource.resource.target], user=self.main_resource.resource.user)
+ results = []
+
if self.main_resource.resource.change_permissions_for_parents:
self._fill_in_parent_directories(self.main_resource.resource.target, results)
@@ -361,6 +373,12 @@ class HdfsResourceWebHDFS:
if self.main_resource.resource.recursive_chmod:
self._fill_directories_list(self.main_resource.resource.target, results)
+
+ # if we don't do this, we can end up waiting real long, having a big result list.
+ if len(results) > HdfsResourceWebHDFS.MAX_FILES_FOR_RECURSIVE_ACTION_VIA_WEBHDFS:
+ shell.checked_call(["hadoop", "fs", "-chmod", "-R", self.mode, self.main_resource.resource.target], user=self.main_resource.resource.user)
+ results = []
+
if self.main_resource.resource.change_permissions_for_parents:
self._fill_in_parent_directories(self.main_resource.resource.target, results)
[2/2] ambari git commit: AMBARI-14660. HistoryServer upgrade times
out when /app-logs is too large (aonishuk)
Posted by ao...@apache.org.
AMBARI-14660. HistoryServer upgrade times out when /app-logs is too large (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2596e648
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2596e648
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2596e648
Branch: refs/heads/branch-2.1
Commit: 2596e648f0804ae47ec7ef43d60e34119fee13c2
Parents: c17e206
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Jan 14 12:44:47 2016 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Jan 14 12:44:47 2016 +0200
----------------------------------------------------------------------
.../libraries/providers/hdfs_resource.py | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2596e648/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
index a016ad6..3c18b7d 100644
--- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
@@ -215,6 +215,12 @@ class HdfsResourceWebHDFS:
Since it's not available on non-hdfs FS and also can be disabled in scope of HDFS.
We should still have the other implementations for such a cases.
"""
+
+ # if we have more than this count of files to recursively chmod/chown
+ # webhdfs won't be used, but 'hadoop fs -chmod (or chown) -R ..' As it can really slow.
+ # (in one second ~17 files can be chmoded)
+ MAX_FILES_FOR_RECURSIVE_ACTION_VIA_WEBHDFS = 1000
+
def action_execute(self, main_resource):
pass
@@ -341,6 +347,12 @@ class HdfsResourceWebHDFS:
if self.main_resource.resource.recursive_chown:
self._fill_directories_list(self.main_resource.resource.target, results)
+
+ # if we don't do this, we can end up waiting real long, having a big result list.
+ if len(results) > HdfsResourceWebHDFS.MAX_FILES_FOR_RECURSIVE_ACTION_VIA_WEBHDFS:
+ shell.checked_call(["hadoop", "fs", "-chown", "-R", format("{owner}:{group}"), self.main_resource.resource.target], user=self.main_resource.resource.user)
+ results = []
+
if self.main_resource.resource.change_permissions_for_parents:
self._fill_in_parent_directories(self.main_resource.resource.target, results)
@@ -358,6 +370,12 @@ class HdfsResourceWebHDFS:
if self.main_resource.resource.recursive_chmod:
self._fill_directories_list(self.main_resource.resource.target, results)
+
+ # if we don't do this, we can end up waiting real long, having a big result list.
+ if len(results) > HdfsResourceWebHDFS.MAX_FILES_FOR_RECURSIVE_ACTION_VIA_WEBHDFS:
+ shell.checked_call(["hadoop", "fs", "-chmod", "-R", self.mode, self.main_resource.resource.target], user=self.main_resource.resource.user)
+ results = []
+
if self.main_resource.resource.change_permissions_for_parents:
self._fill_in_parent_directories(self.main_resource.resource.target, results)