You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2015/07/08 16:48:29 UTC
[2/2] ambari git commit: AMBARI-12331. on non-root agent RU Oozie
restart fails (aonishuk)
AMBARI-12331. on non-root agent RU Oozie restart fails (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6ea0b83c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6ea0b83c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6ea0b83c
Branch: refs/heads/branch-2.1
Commit: 6ea0b83c84bcd23a84577996d6e5b7e7a4af98ba
Parents: 58fc42e
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Wed Jul 8 17:48:03 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Wed Jul 8 17:48:03 2015 +0300
----------------------------------------------------------------------
.../libraries/functions/tar_archive.py | 40 ++---
.../1.4.0.2.0/package/scripts/flume_upgrade.py | 5 +-
.../0.5.0.2.2/package/scripts/knox_gateway.py | 6 +-
.../package/scripts/oozie_server_upgrade.py | 23 +--
.../stacks/2.0.6/HDFS/test_hdfs_client.py | 3 +
.../stacks/2.0.6/OOZIE/test_oozie_server.py | 124 +++++++++------
.../stacks/2.1/FALCON/test_falcon_server.py | 158 ++++++++++++++++++-
.../python/stacks/2.2/KNOX/test_knox_gateway.py | 85 ++++++----
8 files changed, 309 insertions(+), 135 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6ea0b83c/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
index f1fb597..49caf88 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
@@ -18,16 +18,12 @@ See the License for the specific language governing permissions and
limitations under the License.
'''
-import os
-import tarfile
-from contextlib import closing
+from resource_management.core.resources.system import Execute
def archive_dir(output_filename, input_dir):
- with closing(tarfile.open(output_filename, "w:gz")) as tar:
- try:
- tar.add(input_dir, arcname=os.path.basename("."))
- finally:
- tar.close()
+ Execute(('tar', '-zcvf', output_filename, input_dir),
+ sudo = True,
+ )
def archive_directory_dereference(archive, directory):
@@ -38,19 +34,15 @@ def archive_directory_dereference(archive, directory):
:param directory: the directory to include
:return: None
"""
- tarball = None
- try:
- # !!! dereference must be TRUE since the conf is a symlink and we want
- # its contents instead of the actual symlink
- tarball = tarfile.open(archive, mode="w", dereference=True)
-
- # tar the files, chopping off everything in front of directory
- # /foo/bar/conf/a, /foo/bar/conf/b, /foo/bar/conf/dir/c
- # becomes
- # a
- # b
- # dir/c
- tarball.add(directory, arcname=os.path.relpath(directory, start=directory))
- finally:
- if tarball:
- tarball.close()
\ No newline at end of file
+
+ Execute(('tar', '-zcvhf', archive, directory),
+ sudo = True,
+ )
+
+def untar_archive(archive, directory):
+ """
+ :param directory: can be a symlink and is followed
+ """
+ Execute(('tar','-xvf',archive,'-C',directory+"/"),
+ sudo = True,
+ )
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/6ea0b83c/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py
index 32cc1b9..7743619 100644
--- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py
@@ -70,10 +70,7 @@ def pre_start_restore():
directoryMappings[directory])
if os.path.isfile(archive):
- Logger.info('Extracting {0} to {1}'.format(archive, directory))
- Execute(('tar','-xvf',archive,'-C',directory+"/"),
- sudo = True,
- )
+ tar_archive.untar_archive(archive, directory)
# cleanup
if os.path.exists(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)):
http://git-wip-us.apache.org/repos/asf/ambari/blob/6ea0b83c/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py
index 38a55bc..8117312 100644
--- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py
@@ -21,7 +21,7 @@ import os
import tarfile
from resource_management.libraries.script.script import Script
-from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import conf_select, tar_archive
from resource_management.libraries.functions import hdp_select
from resource_management.libraries.functions.check_process_status import check_process_status
from resource_management.libraries.functions import format
@@ -134,9 +134,7 @@ class KnoxGatewayDefault(KnoxGateway):
sudo = True,
)
- Execute(('tar','-xvf',conf_tar_source_path,'-C',extract_dir),
- sudo = True,
- )
+ tar_archive.untar_archive(conf_tar_source_path, extract_dir)
File(conf_tar_dest_path,
action = "delete",
http://git-wip-us.apache.org/repos/asf/ambari/blob/6ea0b83c/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
index fb19ddf..6225eae 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
@@ -25,7 +25,7 @@ import tempfile
from resource_management.core import shell
from resource_management.core.logger import Logger
from resource_management.core.exceptions import Fail
-from resource_management.core.resources.system import Execute
+from resource_management.core.resources.system import Execute, Directory
from resource_management.libraries.functions import Direction
from resource_management.libraries.functions import format
from resource_management.libraries.functions import compare_versions
@@ -87,17 +87,12 @@ class OozieUpgrade(Script):
Logger.info('Extracting {0} to {1}'.format(archive, directory))
- tarball = None
- try:
- tarball = tarfile.open(archive, "r")
- tarball.extractall(directory)
- finally:
- if tarball:
- tarball.close()
+ tar_archive.untar_archive(archive, directory)
# cleanup
- shutil.rmtree(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR))
-
+ Directory(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR),
+ action="delete",
+ )
@staticmethod
def prepare_libext_directory():
@@ -112,11 +107,9 @@ class OozieUpgrade(Script):
target_version_needs_compression_libraries = compare_versions(
format_hdp_stack_version(params.version), '2.2.1.0') >= 0
- if not os.path.isdir(params.oozie_libext_customer_dir):
- os.makedirs(params.oozie_libext_customer_dir, 0o777)
-
- # ensure that it's rwx for all
- os.chmod(params.oozie_libext_customer_dir, 0o777)
+ Directory(params.oozie_libext_customer_dir,
+ mode = 0777,
+ )
# get all hadooplzo* JAR files
# hdp-select set hadoop-client has not run yet, therefore we cannot use
http://git-wip-us.apache.org/repos/asf/ambari/blob/6ea0b83c/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
index 41fd771..06d94e9 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
@@ -64,6 +64,9 @@ class Test(RMFTestCase):
self.assertResourceCalled('PropertiesFile', '/tmp/123/startup.properties',
properties = UnknownConfigurationMock(),
)
+ self.assertResourceCalled('Execute', ('tar', '-zcvf', u'/tmp/HDFS_CLIENT-configs.tar.gz', '/tmp/123'),
+ sudo = True,
+ )
self.assertResourceCalled('Directory', '/tmp/123',
action = ['delete'],
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/6ea0b83c/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
index 71744c3..39fd14b 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
@@ -26,6 +26,7 @@ from resource_management.core.exceptions import Fail
from resource_management.libraries import functions
from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil
import hashlib
+import tempfile
md5_mock = MagicMock()
md5_mock.hexdigest.return_value = "abc123hash"
@@ -33,6 +34,7 @@ md5_mock.hexdigest.return_value = "abc123hash"
@patch("platform.linux_distribution", new = MagicMock(return_value="Linux"))
@patch.object(hashlib, "md5", new=MagicMock(return_value=md5_mock))
@patch.object(WebHDFSUtil, "run_command", new=MagicMock(return_value={}))
+@patch.object(tempfile, "gettempdir", new=MagicMock(return_value="/tmp"))
class TestOozieServer(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "OOZIE/4.0.0.2.0/package"
STACK_VERSION = "2.0.6"
@@ -913,17 +915,15 @@ class TestOozieServer(RMFTestCase):
put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
- @patch("tarfile.open")
@patch("os.path.isdir")
@patch("os.path.exists")
@patch("os.path.isfile")
@patch("os.remove")
- @patch("os.chmod")
@patch("shutil.rmtree", new = MagicMock())
@patch("glob.iglob")
@patch("shutil.copy2", new = MagicMock())
- def test_upgrade(self, glob_mock, chmod_mock, remove_mock,
- isfile_mock, exists_mock, isdir_mock, tarfile_open_mock):
+ def test_upgrade(self, glob_mock, remove_mock,
+ isfile_mock, exists_mock, isdir_mock):
def exists_mock_side_effect(path):
if path == '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar':
@@ -945,23 +945,7 @@ class TestOozieServer(RMFTestCase):
target = RMFTestCase.TARGET_COMMON_SERVICES,
call_mocks = [(0, prepare_war_stdout)]
)
-
- # 2 calls to tarfile.open (1 directories, read + write)
- self.assertTrue(tarfile_open_mock.called)
- self.assertEqual(tarfile_open_mock.call_count,2)
-
- # check the call args for creation of the tarfile
- call_args = tarfile_open_mock.call_args_list[0]
- call_argument_tarfile = call_args[0][0]
- call_kwargs = call_args[1]
-
- self.assertTrue("oozie-upgrade-backup/oozie-conf-backup.tar" in call_argument_tarfile)
- self.assertEquals(True, call_kwargs["dereference"])
-
- self.assertTrue(chmod_mock.called)
- self.assertEqual(chmod_mock.call_count,1)
- chmod_mock.assert_called_once_with('/usr/hdp/current/oozie-server/libext-customer', 511)
-
+
self.assertTrue(isfile_mock.called)
self.assertEqual(isfile_mock.call_count,3)
isfile_mock.assert_called_with('/usr/share/HDP-oozie/ext-2.2.zip')
@@ -970,20 +954,39 @@ class TestOozieServer(RMFTestCase):
self.assertEqual(glob_mock.call_count,1)
glob_mock.assert_called_with('/usr/hdp/2.2.1.0-2135/hadoop/lib/hadoop-lzo*.jar')
- self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', '2.2.1.0-2135'), sudo=True,)
+ self.assertResourceCalled('Execute', ('tar',
+ '-zcvhf',
+ '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar',
+ '/usr/hdp/current/oozie-server/conf/'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', u'2.2.1.0-2135'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('tar',
+ '-xvf',
+ '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar',
+ '-C',
+ '/usr/hdp/current/oozie-server/conf//'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Directory', '/tmp/oozie-upgrade-backup',
+ action = ['delete'],
+ )
+ self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext-customer',
+ mode = 0777,
+ )
self.assertNoMoreResources()
- @patch("tarfile.open")
@patch("os.path.isdir")
@patch("os.path.exists")
@patch("os.path.isfile")
@patch("os.remove")
- @patch("os.chmod")
@patch("shutil.rmtree", new = MagicMock())
@patch("glob.iglob")
@patch("shutil.copy2", new = MagicMock())
- def test_upgrade_23(self, glob_mock, chmod_mock, remove_mock,
- isfile_mock, exists_mock, isdir_mock, tarfile_open_mock):
+ def test_upgrade_23(self, glob_mock, remove_mock,
+ isfile_mock, exists_mock, isdir_mock):
def exists_mock_side_effect(path):
if path == '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar':
@@ -1016,14 +1019,6 @@ class TestOozieServer(RMFTestCase):
mocks_dict = mocks_dict
)
- # 2 calls to tarfile.open (1 directories, read + write)
- self.assertTrue(tarfile_open_mock.called)
- self.assertEqual(tarfile_open_mock.call_count,2)
-
- self.assertTrue(chmod_mock.called)
- self.assertEqual(chmod_mock.call_count,1)
- chmod_mock.assert_called_once_with('/usr/hdp/current/oozie-server/libext-customer', 511)
-
self.assertTrue(isfile_mock.called)
self.assertEqual(isfile_mock.call_count,3)
isfile_mock.assert_called_with('/usr/share/HDP-oozie/ext-2.2.zip')
@@ -1032,7 +1027,28 @@ class TestOozieServer(RMFTestCase):
self.assertEqual(glob_mock.call_count,1)
glob_mock.assert_called_with('/usr/hdp/2.3.0.0-1234/hadoop/lib/hadoop-lzo*.jar')
- self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', '2.3.0.0-1234'), sudo=True)
+ self.assertResourceCalled('Execute', ('tar',
+ '-zcvhf',
+ '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar',
+ '/usr/hdp/current/oozie-server/conf/'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', '2.3.0.0-1234'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('tar',
+ '-xvf',
+ '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar',
+ '-C',
+ '/usr/hdp/current/oozie-server/conf//'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Directory', '/tmp/oozie-upgrade-backup',
+ action = ['delete'],
+ )
+ self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext-customer',
+ mode = 0777,
+ )
self.assertNoMoreResources()
self.assertEquals(2, mocks_dict['call'].call_count)
@@ -1045,16 +1061,14 @@ class TestOozieServer(RMFTestCase):
mocks_dict['call'].call_args_list[0][0][0])
- @patch("tarfile.open")
@patch("os.path.isdir")
@patch("os.path.exists")
@patch("os.path.isfile")
@patch("os.remove")
- @patch("os.chmod")
@patch("shutil.rmtree", new = MagicMock())
@patch("shutil.copy2", new = MagicMock())
- def test_downgrade_no_compression_library_copy(self, chmod_mock, remove_mock,
- isfile_mock, exists_mock, isdir_mock, tarfile_open_mock):
+ def test_downgrade_no_compression_library_copy(self, remove_mock,
+ isfile_mock, exists_mock, isdir_mock):
isdir_mock.return_value = True
exists_mock.return_value = False
@@ -1069,19 +1083,31 @@ class TestOozieServer(RMFTestCase):
target = RMFTestCase.TARGET_COMMON_SERVICES,
call_mocks = [(0, prepare_war_stdout)])
- # 2 calls to tarfile.open (1 directories, read + write)
- self.assertTrue(tarfile_open_mock.called)
- self.assertEqual(tarfile_open_mock.call_count,2)
-
- self.assertTrue(chmod_mock.called)
- self.assertEqual(chmod_mock.call_count,1)
- chmod_mock.assert_called_once_with('/usr/hdp/current/oozie-server/libext-customer', 511)
-
self.assertTrue(isfile_mock.called)
self.assertEqual(isfile_mock.call_count,2)
isfile_mock.assert_called_with('/usr/share/HDP-oozie/ext-2.2.zip')
-
- self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', '2.2.0.0-0000'), sudo=True)
+ self.assertResourceCalled('Execute', ('tar',
+ '-zcvhf',
+ '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar',
+ '/usr/hdp/current/oozie-server/conf/'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', u'2.2.0.0-0000'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('tar',
+ '-xvf',
+ '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar',
+ '-C',
+ '/usr/hdp/current/oozie-server/conf//'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Directory', '/tmp/oozie-upgrade-backup',
+ action = ['delete'],
+ )
+ self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext-customer',
+ mode = 0777,
+ )
self.assertNoMoreResources()
http://git-wip-us.apache.org/repos/asf/ambari/blob/6ea0b83c/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
index 087b72c..ba2c026 100644
--- a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
+++ b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
@@ -23,9 +23,10 @@ from mock.mock import MagicMock, patch
import shutil
from stacks.utils.RMFTestCase import *
import tarfile
-
+import tempfile
@patch("platform.linux_distribution", new = MagicMock(return_value="Linux"))
+@patch.object(tempfile, "gettempdir", new=MagicMock(return_value="/tmp"))
class TestFalconServer(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "FALCON/0.5.0.2.1/package"
STACK_VERSION = "2.1"
@@ -191,7 +192,7 @@ class TestFalconServer(RMFTestCase):
tarfile_open_mock):
isdir_mock.return_value = True
- exists_mock.side_effect = [False,False,True]
+ exists_mock.side_effect = [False,False,True, True]
isfile_mock.return_value = True
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py",
@@ -207,11 +208,159 @@ class TestFalconServer(RMFTestCase):
self.assertResourceCalled('File', '/var/run/falcon/falcon.pid',
action = ['delete'])
- self.assertResourceCalled('Execute', ('hdp-select', 'set', 'falcon-server', '2.2.1.0-2135'), sudo=True)
+ self.assertResourceCalled('Execute', ('tar',
+ '-zcvhf',
+ '/tmp/falcon-upgrade-backup/falcon-conf-backup.tar',
+ '/usr/hdp/current/falcon-server/conf'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('tar',
+ '-zcvhf',
+ '/tmp/falcon-upgrade-backup/falcon-local-backup.tar',
+ u'/hadoop/falcon'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('hdp-select', 'set', 'falcon-server', u'2.2.1.0-2135'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Directory', '/var/run/falcon',
+ owner = 'falcon',
+ recursive = True,
+ )
+ self.assertResourceCalled('Directory', '/var/log/falcon',
+ owner = 'falcon',
+ recursive = True,
+ )
+ self.assertResourceCalled('Directory', '/usr/hdp/current/falcon-server/webapp',
+ owner = 'falcon',
+ recursive = True,
+ )
+ self.assertResourceCalled('Directory', '/usr/hdp/current/falcon-server',
+ owner = 'falcon',
+ recursive = True,
+ )
+ self.assertResourceCalled('Directory', '/etc/falcon',
+ recursive = True,
+ mode = 0755,
+ )
+ self.assertResourceCalled('Directory', '/usr/hdp/current/falcon-server/conf',
+ owner = 'falcon',
+ recursive = True,
+ )
+ self.assertResourceCalled('File', '/usr/hdp/current/falcon-server/conf/falcon-env.sh',
+ owner = 'falcon',
+ content = InlineTemplate(self.getConfig()['configurations']['falcon-env']['content']),
+ )
+ self.assertResourceCalled('File', '/usr/hdp/current/falcon-server/conf/client.properties',
+ owner = 'falcon',
+ content = Template('client.properties.j2'),
+ mode = 0644,
+ )
+ self.assertResourceCalled('PropertiesFile', '/usr/hdp/current/falcon-server/conf/runtime.properties',
+ owner = 'falcon',
+ mode = 0644,
+ properties = {u'*.domain': u'${falcon.app.type}',
+ u'*.log.cleanup.frequency.days.retention': u'days(7)',
+ u'*.log.cleanup.frequency.hours.retention': u'minutes(1)',
+ u'*.log.cleanup.frequency.minutes.retention': u'hours(6)',
+ u'*.log.cleanup.frequency.months.retention': u'months(3)'},
+ )
+ self.assertResourceCalled('PropertiesFile', '/usr/hdp/current/falcon-server/conf/startup.properties',
+ owner = 'falcon',
+ mode = 0644,
+ properties = self.getConfig()['configurations']['falcon-startup.properties'],
+ )
+ self.assertResourceCalled('Directory', '/hadoop/falcon/data/lineage/graphdb',
+ owner = 'falcon',
+ recursive = True,
+ group = 'hadoop',
+ mode = 0775,
+ cd_access = 'a',
+ )
+ self.assertResourceCalled('Directory', '/hadoop/falcon/data/lineage',
+ owner = 'falcon',
+ recursive = True,
+ group = 'hadoop',
+ mode = 0775,
+ cd_access = 'a',
+ )
+ self.assertResourceCalled('Directory', '/hadoop/falcon/store',
+ owner = 'falcon',
+ recursive = True,
+ )
+ self.assertResourceCalled('HdfsResource', '/apps/falcon',
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = UnknownConfigurationMock(),
+ user = 'hdfs',
+ owner = 'falcon',
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ type = 'directory',
+ action = ['create_on_execute'],
+ mode = 0777,
+ )
+ self.assertResourceCalled('Directory', '/hadoop/falcon/store',
+ owner = 'falcon',
+ recursive = True,
+ )
+ self.assertResourceCalled('HdfsResource', '/apps/data-mirroring',
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ keytab = UnknownConfigurationMock(),
+ source = '/usr/hdp/current/falcon-server/data-mirroring',
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ user = 'hdfs',
+ hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = UnknownConfigurationMock(),
+ recursive_chmod = True,
+ recursive_chown = True,
+ owner = 'falcon',
+ group = 'users',
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ type = 'directory',
+ action = ['create_on_execute'],
+ mode = 0770,
+ )
+ self.assertResourceCalled('HdfsResource', None,
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = UnknownConfigurationMock(),
+ user = 'hdfs',
+ action = ['execute'],
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ )
+ self.assertResourceCalled('Directory', '/hadoop/falcon',
+ owner = 'falcon',
+ recursive = True,
+ cd_access = 'a',
+ )
+ self.assertResourceCalled('Directory', '/hadoop/falcon/embeddedmq',
+ owner = 'falcon',
+ recursive = True,
+ )
+ self.assertResourceCalled('Directory', '/hadoop/falcon/embeddedmq/data',
+ owner = 'falcon',
+ recursive = True,
+ )
+ self.assertResourceCalled('Execute', '/usr/hdp/current/falcon-server/bin/falcon-start -port 15000',
+ environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client'},
+ path = ['/usr/hdp/current/hadoop-client/bin'],
+ user = 'falcon',
+ )
+ self.assertNoMoreResources()
# 4 calls to tarfile.open (2 directories * read + write)
self.assertTrue(tarfile_open_mock.called)
- self.assertEqual(tarfile_open_mock.call_count,4)
+ self.assertEqual(tarfile_open_mock.call_count,2)
# check the call args for creation of the tarfile
call_args = tarfile_open_mock.call_args_list[0]
@@ -219,7 +368,6 @@ class TestFalconServer(RMFTestCase):
call_kwargs = call_args[1]
self.assertTrue("falcon-upgrade-backup/falcon-conf-backup.tar" in call_argument_tarfile)
- self.assertEquals(True, call_kwargs["dereference"])
@patch("resource_management.libraries.functions.security_commons.build_expectations")
@patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem")
http://git-wip-us.apache.org/repos/asf/ambari/blob/6ea0b83c/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
index 0c4b292..5ca5251 100644
--- a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
+++ b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
@@ -18,31 +18,13 @@ See the License for the specific language governing permissions and
limitations under the License.
'''
import json
+import tempfile
from resource_management import *
from stacks.utils.RMFTestCase import *
from mock.mock import patch
from mock.mock import MagicMock
-
-class TarfileFake:
- """
- Dummy class to mock the calls to tarfile module.
- """
- def __init__(self):
- self.extractall_count = 0
- self.add_count = 0
- self.close_count = 0
-
- def extractall(self, path=".", members=None):
- self.extractall_count += 1
-
- def add(self, name, arcname=None, recursive=True, exclude=None, filter=None):
- self.add_count += 1
-
- def close(self):
- self.close_count += 1
-
-
+@patch.object(tempfile, "gettempdir", new=MagicMock(return_value="/tmp"))
@patch("platform.linux_distribution", new = MagicMock(return_value="Linux"))
class TestKnoxGateway(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "KNOX/0.5.0.2.2/package"
@@ -238,9 +220,8 @@ class TestKnoxGateway(RMFTestCase):
)
put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
- @patch("tarfile.open")
@patch("os.path.isdir")
- def test_pre_rolling_restart(self, isdir_mock, tarfile_open_mock):
+ def test_pre_rolling_restart(self, isdir_mock):
isdir_mock.return_value = True
config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/knox_upgrade.json"
with open(config_file, "r") as f:
@@ -255,16 +236,28 @@ class TestKnoxGateway(RMFTestCase):
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES)
- self.assertTrue(tarfile_open_mock.called)
-
- self.assertResourceCalled("Execute", ('hdp-select', 'set', 'knox-server', version), sudo=True)
+ self.assertResourceCalled('Execute', ('tar',
+ '-zcvhf',
+ '/tmp/knox-upgrade-backup/knox-conf-backup.tar',
+ '/usr/hdp/current/knox-server/conf/'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('tar',
+ '-zcvhf',
+ '/tmp/knox-upgrade-backup/knox-data-backup.tar',
+ '/var/lib/knox/data'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('hdp-select', 'set', 'knox-server', '2.2.1.0-3242'),
+ sudo = True,
+ )
+ self.assertNoMoreResources()
@patch("os.remove")
@patch("os.path.exists")
- @patch("tarfile.open")
@patch("os.path.isdir")
@patch("resource_management.core.shell.call")
- def test_pre_rolling_restart_23(self, call_mock, isdir_mock, tarfile_open_mock, path_exists_mock, remove_mock):
+ def test_pre_rolling_restart_23(self, call_mock, isdir_mock, path_exists_mock, remove_mock):
isdir_mock.return_value = True
config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/knox_upgrade.json"
with open(config_file, "r") as f:
@@ -273,8 +266,6 @@ class TestKnoxGateway(RMFTestCase):
json_content['commandParams']['version'] = version
path_exists_mock.return_value = True
- knox_conf_tarfile = TarfileFake()
- tarfile_open_mock.return_value = knox_conf_tarfile
mocks_dict = {}
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/knox_gateway.py",
@@ -286,9 +277,37 @@ class TestKnoxGateway(RMFTestCase):
call_mocks = [(0, None), (0, None)],
mocks_dict = mocks_dict)
- self.assertTrue(tarfile_open_mock.called)
-
- self.assertResourceCalled("Execute", ('hdp-select', 'set', 'knox-server', version), sudo=True)
+ self.assertResourceCalled('Execute', ('tar',
+ '-zcvhf',
+ '/tmp/knox-upgrade-backup/knox-conf-backup.tar',
+ '/usr/hdp/current/knox-server/conf/'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('tar',
+ '-zcvhf',
+ '/tmp/knox-upgrade-backup/knox-data-backup.tar',
+ '/var/lib/knox/data'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('hdp-select', 'set', 'knox-server', '2.3.0.0-1234'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('cp',
+ '/tmp/knox-upgrade-backup/knox-conf-backup.tar',
+ '/usr/hdp/current/knox-server/conf/knox-conf-backup.tar'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('tar',
+ '-xvf',
+ '/tmp/knox-upgrade-backup/knox-conf-backup.tar',
+ '-C',
+ '/usr/hdp/current/knox-server/conf/'),
+ sudo = True,
+ )
+ self.assertResourceCalled('File', '/usr/hdp/current/knox-server/conf/knox-conf-backup.tar',
+ action = ['delete'],
+ )
+ self.assertNoMoreResources()
self.assertEquals(1, mocks_dict['call'].call_count)
self.assertEquals(1, mocks_dict['checked_call'].call_count)
@@ -299,8 +318,6 @@ class TestKnoxGateway(RMFTestCase):
('conf-select', 'create-conf-dir', '--package', 'knox', '--stack-version', '2.3.0.0-1234', '--conf-version', '0'),
mocks_dict['call'].call_args_list[0][0][0])
- self.assertTrue(2, knox_conf_tarfile.close_count)
-
@patch("os.path.islink")
@patch("os.path.realpath")
@patch("os.unlink")