You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2015/08/27 14:46:57 UTC
[1/2] ambari git commit: AMBARI-12900. Ambari overrides hot-fixed tez
tarball on HDFS everytime HiveServer is restarted (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/branch-2.1 69307276b -> 7184571c3
refs/heads/trunk 2ec1f8558 -> b0137be7a
AMBARI-12900. Ambari overrides hot-fixed tez tarball on HDFS everytime HiveServer is restarted (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b0137be7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b0137be7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b0137be7
Branch: refs/heads/trunk
Commit: b0137be7ab9e0388e6266b1911976ec253747ac9
Parents: 2ec1f85
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Aug 27 15:45:57 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Aug 27 15:45:57 2015 +0300
----------------------------------------------------------------------
.../resource_management/libraries/functions/copy_tarball.py | 5 +++--
.../resource_management/libraries/providers/hdfs_resource.py | 3 +++
.../resource_management/libraries/resources/hdfs_resource.py | 4 ++++
3 files changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b0137be7/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
index ad4aadc..792f019 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
@@ -96,7 +96,7 @@ def _get_single_version_from_hdp_select():
return hdp_version
def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=None, custom_dest_file=None, force_execute=False,
- use_ru_version_during_ru=True):
+ use_ru_version_during_ru=True, replace_existing_files=False):
"""
:param name: Tarball name, e.g., tez, hive, pig, sqoop.
:param user_group: Group to own the directory.
@@ -190,7 +190,8 @@ def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=Non
source=source_file,
group=user_group,
owner=owner,
- mode=0444
+ mode=0444,
+ replace_existing_files=replace_existing_files,
)
Logger.info("Will attempt to copy {0} tarball from {1} to DFS at {2}.".format(name, source_file, dest_file))
http://git-wip-us.apache.org/repos/asf/ambari/blob/b0137be7/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
index cb6fb21..1d1aeba 100644
--- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
@@ -301,6 +301,9 @@ class HdfsResourceWebHDFS:
if local_file_size == length:
Logger.info(format("DFS file {target} is identical to {source}, skipping the copying"))
return
+ elif not self.main_resource.resource.replace_existing_files:
+ Logger.info(format("Not replacing existing DFS file {target} which is different from {source}, due to replace_existing_files=False"))
+ return
else:
Logger.info(format("File {target} already exists in DFS, skipping the creation"))
return
http://git-wip-us.apache.org/repos/asf/ambari/blob/b0137be7/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py
index 3a991b0..69aeab0 100644
--- a/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py
+++ b/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py
@@ -63,6 +63,10 @@ class HdfsResource(Resource):
recursive_chown = BooleanArgument(default=False)
recursive_chmod = BooleanArgument(default=False)
change_permissions_for_parents = BooleanArgument(default=False)
+ """
+ If a file present in HDFS is different from source, should it be replaced?
+ """
+ replace_existing_files = BooleanArgument(default=True)
security_enabled = BooleanArgument(default=False)
principal_name = ResourceArgument()
[2/2] ambari git commit: AMBARI-12900. Ambari overrides hot-fixed tez
tarball on HDFS everytime HiveServer is restarted (aonishuk)
Posted by ao...@apache.org.
AMBARI-12900. Ambari overrides hot-fixed tez tarball on HDFS everytime HiveServer is restarted (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7184571c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7184571c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7184571c
Branch: refs/heads/branch-2.1
Commit: 7184571c3dff60c85fd7983c27dc638f0eb80c84
Parents: 6930727
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Aug 27 15:45:59 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Aug 27 15:45:59 2015 +0300
----------------------------------------------------------------------
.../resource_management/libraries/functions/copy_tarball.py | 5 +++--
.../resource_management/libraries/providers/hdfs_resource.py | 3 +++
.../resource_management/libraries/resources/hdfs_resource.py | 4 ++++
3 files changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7184571c/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
index ad4aadc..792f019 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
@@ -96,7 +96,7 @@ def _get_single_version_from_hdp_select():
return hdp_version
def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=None, custom_dest_file=None, force_execute=False,
- use_ru_version_during_ru=True):
+ use_ru_version_during_ru=True, replace_existing_files=False):
"""
:param name: Tarball name, e.g., tez, hive, pig, sqoop.
:param user_group: Group to own the directory.
@@ -190,7 +190,8 @@ def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=Non
source=source_file,
group=user_group,
owner=owner,
- mode=0444
+ mode=0444,
+ replace_existing_files=replace_existing_files,
)
Logger.info("Will attempt to copy {0} tarball from {1} to DFS at {2}.".format(name, source_file, dest_file))
http://git-wip-us.apache.org/repos/asf/ambari/blob/7184571c/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
index cb6fb21..1d1aeba 100644
--- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
@@ -301,6 +301,9 @@ class HdfsResourceWebHDFS:
if local_file_size == length:
Logger.info(format("DFS file {target} is identical to {source}, skipping the copying"))
return
+ elif not self.main_resource.resource.replace_existing_files:
+ Logger.info(format("Not replacing existing DFS file {target} which is different from {source}, due to replace_existing_files=False"))
+ return
else:
Logger.info(format("File {target} already exists in DFS, skipping the creation"))
return
http://git-wip-us.apache.org/repos/asf/ambari/blob/7184571c/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py
index 3a991b0..69aeab0 100644
--- a/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py
+++ b/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py
@@ -63,6 +63,10 @@ class HdfsResource(Resource):
recursive_chown = BooleanArgument(default=False)
recursive_chmod = BooleanArgument(default=False)
change_permissions_for_parents = BooleanArgument(default=False)
+ """
+ If a file present in HDFS is different from source, should it be replaced?
+ """
+ replace_existing_files = BooleanArgument(default=True)
security_enabled = BooleanArgument(default=False)
principal_name = ResourceArgument()