[ambari] branch trunk updated: [AMBARI-23741] - Starting JHS Takes Too Long Due To Tarball Extraction (#1158)

[jo...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jonathanhurley pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git


The following commit(s) were added to refs/heads/trunk by this push:
     new bd071a0  [AMBARI-23741] - Starting JHS Takes Too Long Due To Tarball Extraction (#1158)
bd071a0 is described below

commit bd071a0fa4bcb876bf6f4260ef8e5c8666a057d8
Author: Jonathan Hurley <jo...@apache.org>
AuthorDate: Thu May 3 12:21:19 2018 -0400

    [AMBARI-23741] - Starting JHS Takes Too Long Due To Tarball Extraction (#1158)
---
 .../resource_management/libraries/functions/copy_tarball.py    |  4 ++--
 .../resource_management/libraries/functions/tar_archive.py     | 10 ++++++----
 ambari-server/src/main/python/ambari_server/setupMpacks.py     |  5 +++--
 ambari-server/src/test/python/TestMpacks.py                    |  8 ++++----
 4 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
index eed90ec..63bd065 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
@@ -70,7 +70,7 @@ def _prepare_tez_tarball():
   sudo.chmod(tez_temp_dir, 0777)
 
   Logger.info("Extracting {0} to {1}".format(mapreduce_source_file, mapreduce_temp_dir))
-  tar_archive.extract_archive(mapreduce_source_file, mapreduce_temp_dir)
+  tar_archive.untar_archive(mapreduce_source_file, mapreduce_temp_dir)
 
   Logger.info("Extracting {0} to {1}".format(tez_source_file, tez_temp_dir))
   tar_archive.untar_archive(tez_source_file, tez_temp_dir)
@@ -169,7 +169,7 @@ def _prepare_mapreduce_tarball():
     raise Fail("Unable to seed the mapreduce tarball with native LZO libraries since the source Hadoop native lib directory {0} does not exist".format(hadoop_lib_native_source_dir))
 
   Logger.info("Extracting {0} to {1}".format(mapreduce_source_file, mapreduce_temp_dir))
-  tar_archive.extract_archive(mapreduce_source_file, mapreduce_temp_dir)
+  tar_archive.untar_archive(mapreduce_source_file, mapreduce_temp_dir)
 
   mapreduce_lib_dir = os.path.join(mapreduce_temp_dir, "hadoop", "lib")
 
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
index 372a894..3be1ab6 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
@@ -48,7 +48,13 @@ def archive_directory_dereference(archive, directory):
 
 def untar_archive(archive, directory, silent=True):
   """
+  Extracts a tarball using the system's tar utility. This is more
+  efficient than Python 2.x's tarfile module.
+
   :param directory:   can be a symlink and is followed
+  :param silent:  True if the output should be suppressed. This is a good
+  idea in most cases as the streamed output of a huge tarball can cause
+  a performance degredation
   """
   options = "-xf" if silent else "-xvf"
 
@@ -58,10 +64,6 @@ def untar_archive(archive, directory, silent=True):
     try_sleep = 1,
   )
 
-def extract_archive(archive, directory):
-  with closing(tarfile.open(archive, mode(archive))) as tar:
-    tar.extractall(directory)
-
 def get_archive_root_dir(archive):
   root_dir = None
   with closing(tarfile.open(archive, mode(archive))) as tar:
diff --git a/ambari-server/src/main/python/ambari_server/setupMpacks.py b/ambari-server/src/main/python/ambari_server/setupMpacks.py
index 7a9b2b8..a0dde5a 100755
--- a/ambari-server/src/main/python/ambari_server/setupMpacks.py
+++ b/ambari-server/src/main/python/ambari_server/setupMpacks.py
@@ -39,7 +39,7 @@ from ambari_server.userInput import get_YN_input
 from ambari_server.dbConfiguration import ensure_jdbc_driver_is_installed, LINUX_DBMS_KEYS_LIST
 
 from resource_management.core import sudo
-from resource_management.libraries.functions.tar_archive import extract_archive, get_archive_root_dir
+from resource_management.libraries.functions.tar_archive import untar_archive, get_archive_root_dir
 from resource_management.libraries.functions.version import compare_versions
 
 
@@ -157,7 +157,8 @@ def expand_mpack(archive_path):
   print_info_msg("Expand management pack at temp location {0}".format(tmp_root_dir))
   if os.path.exists(tmp_root_dir):
     sudo.rmtree(tmp_root_dir)
-  extract_archive(archive_path, tmpdir)
+
+  untar_archive(archive_path, tmpdir)
 
   if not os.path.exists(tmp_root_dir):
     print_error_msg("Malformed management pack. Failed to expand management pack!")
diff --git a/ambari-server/src/test/python/TestMpacks.py b/ambari-server/src/test/python/TestMpacks.py
index b290665..1a45038 100644
--- a/ambari-server/src/test/python/TestMpacks.py
+++ b/ambari-server/src/test/python/TestMpacks.py
@@ -215,11 +215,11 @@ class TestMpacks(TestCase):
     os_path_exists_mock.assert_has_calls(os_path_exists_calls)
 
   @patch("os.path.exists")
-  @patch("ambari_server.setupMpacks.extract_archive")
+  @patch("ambari_server.setupMpacks.untar_archive")
   @patch("ambari_server.setupMpacks.get_archive_root_dir")
   @patch("ambari_server.setupMpacks.download_mpack")
   @patch("ambari_server.setupMpacks.get_ambari_properties")
-  def test_install_mpack_with_malformed_mpack(self, get_ambari_properties_mock, download_mpack_mock, get_archive_root_dir_mock, extract_archive_mock, os_path_exists_mock):
+  def test_install_mpack_with_malformed_mpack(self, get_ambari_properties_mock, download_mpack_mock, get_archive_root_dir_mock, untar_archive_mock, os_path_exists_mock):
     options = self._create_empty_options_mock()
     options.mpack_path = "/path/to/mpack.tar.gz"
     download_mpack_mock.return_value = "/tmp/mpack.tar.gz"
@@ -237,7 +237,7 @@ class TestMpacks(TestCase):
 
     get_archive_root_dir_mock.return_value = "mpack"
     os_path_exists_mock.side_effect = [True, True, False, False]
-    extract_archive_mock.return_value = None
+    untar_archive_mock.return_value = None
     fail = False
     try:
       install_mpack(options)
@@ -248,7 +248,7 @@ class TestMpacks(TestCase):
 
     get_archive_root_dir_mock.return_value = "mpack"
     os_path_exists_mock.side_effect = [True, True, False, True, False]
-    extract_archive_mock.return_value = None
+    untar_archive_mock.return_value = None
     fail = False
     try:
       install_mpack(options)

-- 
To stop receiving notification emails like this one, please contact
jonathanhurley@apache.org.