You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2015/09/14 14:01:19 UTC
[1/2] ambari git commit: Verify if restricting acls on
/var/lib/ambari-agent/data will be OK (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/trunk e837d4f08 -> b8388e12b
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-server/src/test/python/TestBootstrap.py.orig
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestBootstrap.py.orig b/ambari-server/src/test/python/TestBootstrap.py.orig
new file mode 100644
index 0000000..792d99d
--- /dev/null
+++ b/ambari-server/src/test/python/TestBootstrap.py.orig
@@ -0,0 +1,879 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from stacks.utils.RMFTestCase import *
+import bootstrap
+import time
+import subprocess
+import os
+import logging
+import tempfile
+import pprint
+
+from ambari_commons.os_check import OSCheck
+from bootstrap import PBootstrap, Bootstrap, BootstrapDefault, SharedState, HostLog, SCP, SSH
+from unittest import TestCase
+from subprocess import Popen
+from bootstrap import AMBARI_PASSPHRASE_VAR_NAME
+from mock.mock import MagicMock, call
+from mock.mock import patch
+from mock.mock import create_autospec
+from only_for_platform import not_for_platform, os_distro_value, PLATFORM_WINDOWS
+
+@not_for_platform(PLATFORM_WINDOWS)
+class TestBootstrap(TestCase):
+
+ def setUp(self):
+ logging.basicConfig(level=logging.ERROR)
+
+
+ def test_getRemoteName(self):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6", None, "8440", "root")
+ res = bootstrap_obj = Bootstrap("hostname", shared_state)
+ utime1 = 1234
+ utime2 = 12345
+ bootstrap_obj.getUtime = MagicMock(return_value=utime1)
+ remote1 = bootstrap_obj.getRemoteName("/tmp/setupAgent.sh")
+ self.assertEquals(remote1, "/tmp/setupAgent{0}.sh".format(utime1))
+
+ bootstrap_obj.getUtime.return_value=utime2
+ remote1 = bootstrap_obj.getRemoteName("/tmp/setupAgent.sh")
+ self.assertEquals(remote1, "/tmp/setupAgent{0}.sh".format(utime1))
+
+ remote2 = bootstrap_obj.getRemoteName("/tmp/host_pass")
+ self.assertEquals(remote2, "/tmp/host_pass{0}".format(utime2))
+
+
+ # TODO: Test bootstrap timeout
+
+ # TODO: test_return_error_message_for_missing_sudo_package
+
+ def test_getAmbariPort(self):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ self.assertEquals(bootstrap_obj.getAmbariPort(),"8440")
+ shared_state.server_port = None
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ self.assertEquals(bootstrap_obj.getAmbariPort(),"null")
+
+
+ @patch.object(subprocess, "Popen")
+ @patch("sys.stderr")
+ @patch("sys.exit")
+ @patch.object(PBootstrap, "run")
+ @patch("os.path.dirname")
+ @patch("os.path.realpath")
+ def test_bootstrap_main(self, dirname_mock, realpath_mock, run_mock, exit_mock, stderr_mock, subprocess_Popen_mock):
+ bootstrap.main(["bootstrap.py", "hostname,hostname2", "/tmp/bootstrap", "root", "sshkey_file", "setupAgent.py", "ambariServer", \
+ "centos6", "1.1.1", "8440", "root", "passwordfile"])
+ self.assertTrue(run_mock.called)
+ run_mock.reset_mock()
+ bootstrap.main(["bootstrap.py", "hostname,hostname2", "/tmp/bootstrap", "root", "sshkey_file", "setupAgent.py", "ambariServer", \
+ "centos6", "1.1.1", "8440", "root", None])
+ self.assertTrue(run_mock.called)
+ run_mock.reset_mock()
+ def side_effect(retcode):
+ raise Exception(retcode, "sys.exit")
+ exit_mock.side_effect = side_effect
+ try:
+ bootstrap.main(["bootstrap.py","hostname,hostname2", "/tmp/bootstrap"])
+ self.fail("sys.exit(2)")
+ except Exception:
+ # Expected
+ pass
+ self.assertTrue(exit_mock.called)
+
+
+ @patch("os.environ")
+ def test_getRunSetupWithPasswordCommand(self, environ_mock):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ environ_mock.__getitem__.return_value = "TEST_PASSPHRASE"
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ utime = 1234
+ bootstrap_obj.getUtime = MagicMock(return_value=utime)
+ ret = bootstrap_obj.getRunSetupWithPasswordCommand("hostname")
+ expected = "sudo -S python /var/lib/ambari-agent/data/tmp/setupAgent{0}.py hostname TEST_PASSPHRASE " \
+ "ambariServer root 8440 < /var/lib/ambari-agent/data/tmp/host_pass{0}".format(utime)
+ self.assertEquals(ret, expected)
+
+
+ def test_generateRandomFileName(self):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ self.assertTrue(bootstrap_obj.generateRandomFileName(None) == bootstrap_obj.getUtime())
+
+
+
+ @patch.object(OSCheck, "is_redhat_family")
+ @patch.object(OSCheck, "is_suse_family")
+ def test_getRepoDir(self, is_suse_family, is_redhat_family):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ # Suse
+ is_redhat_family.return_value = False
+ is_suse_family.return_value = True
+ res = bootstrap_obj.getRepoDir()
+ self.assertEquals(res, "/etc/zypp/repos.d")
+ # non-Suse
+ is_suse_family.return_value = False
+ is_redhat_family.return_value = True
+ res = bootstrap_obj.getRepoDir()
+ self.assertEquals(res, "/etc/yum.repos.d")
+
+ def test_getSetupScript(self):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ self.assertEquals(bootstrap_obj.shared_state.script_dir, "scriptDir")
+
+
+ def test_run_setup_agent_command_ends_with_project_version(self):
+ os.environ[AMBARI_PASSPHRASE_VAR_NAME] = ""
+ version = "1.1.1"
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ version, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ runSetupCommand = bootstrap_obj.getRunSetupCommand("hostname")
+ self.assertTrue(runSetupCommand.endswith(version + " 8440"))
+
+
+ def test_agent_setup_command_without_project_version(self):
+ os.environ[AMBARI_PASSPHRASE_VAR_NAME] = ""
+ version = None
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ version, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ runSetupCommand = bootstrap_obj.getRunSetupCommand("hostname")
+ self.assertTrue(runSetupCommand.endswith(" 8440"))
+
+
+ # TODO: test_os_check_fail_fails_bootstrap_execution
+
+
+ def test_host_log(self):
+ tmp_file, tmp_filename = tempfile.mkstemp()
+ dummy_log = HostLog(tmp_filename)
+ # First write to log
+ dummy_log.write("a\nb\nc")
+ # Read it
+ with open(tmp_filename) as f:
+ s = f.read()
+ etalon = "a\nb\nc\n"
+ self.assertEquals(s, etalon)
+ # Next write
+ dummy_log.write("Yet another string")
+ # Read it
+ with open(tmp_filename) as f:
+ s = f.read()
+ etalon = "a\nb\nc\nYet another string\n"
+ self.assertEquals(s, etalon)
+ # Should not append line end if it already exists
+ dummy_log.write("line break->\n")
+ # Read it
+ with open(tmp_filename) as f:
+ s = f.read()
+ etalon = "a\nb\nc\nYet another string\nline break->\n"
+ self.assertEquals(s, etalon)
+ # Cleanup
+ os.unlink(tmp_filename)
+
+
+ @patch("subprocess.Popen")
+ def test_SCP(self, popenMock):
+ params = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ "1.2.1", "8440", "root")
+ host_log_mock = MagicMock()
+ log = {'text': ""}
+ def write_side_effect(text):
+ log['text'] = log['text'] + text
+
+ host_log_mock.write.side_effect = write_side_effect
+ scp = SCP(params.user, params.sshkey_file, "dummy-host", "src/file",
+ "dst/file", params.bootdir, host_log_mock)
+ log_sample = "log_sample"
+ error_sample = "error_sample"
+ # Successful run
+ process = MagicMock()
+ popenMock.return_value = process
+ process.communicate.return_value = (log_sample, error_sample)
+ process.returncode = 0
+
+ retcode = scp.run()
+
+ self.assertTrue(popenMock.called)
+ self.assertTrue(log_sample in log['text'])
+ self.assertTrue(error_sample in log['text'])
+ command_str = str(popenMock.call_args[0][0])
+ self.assertEquals(command_str, "['scp', '-r', '-o', 'ConnectTimeout=60', '-o', "
+ "'BatchMode=yes', '-o', 'StrictHostKeyChecking=no', '-i', 'sshkey_file',"
+ " 'src/file', 'root@dummy-host:dst/file']")
+ self.assertEqual(retcode["exitstatus"], 0)
+
+ log['text'] = ""
+ #unsuccessfull run
+ process.returncode = 1
+
+ retcode = scp.run()
+
+ self.assertTrue(log_sample in log['text'])
+ self.assertTrue(error_sample in log['text'])
+ self.assertEqual(retcode["exitstatus"], 1)
+
+
+ @patch("subprocess.Popen")
+ def test_SSH(self, popenMock):
+ params = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ "1.2.1", "8440", "root")
+ host_log_mock = MagicMock()
+ log = {'text': ""}
+ def write_side_effect(text):
+ log['text'] = log['text'] + text
+
+ host_log_mock.write.side_effect = write_side_effect
+ ssh = SSH(params.user, params.sshkey_file, "dummy-host", "dummy-command",
+ params.bootdir, host_log_mock)
+ log_sample = "log_sample"
+ error_sample = "error_sample"
+ # Successful run
+ process = MagicMock()
+ popenMock.return_value = process
+ process.communicate.return_value = (log_sample, error_sample)
+ process.returncode = 0
+
+ retcode = ssh.run()
+
+ self.assertTrue(popenMock.called)
+ self.assertTrue(log_sample in log['text'])
+ self.assertTrue(error_sample in log['text'])
+ command_str = str(popenMock.call_args[0][0])
+ self.assertEquals(command_str, "['ssh', '-o', 'ConnectTimeOut=60', '-o', "
+ "'StrictHostKeyChecking=no', '-o', 'BatchMode=yes', '-tt', '-i', "
+ "'sshkey_file', 'root@dummy-host', 'dummy-command']")
+ self.assertEqual(retcode["exitstatus"], 0)
+
+ log['text'] = ""
+ #unsuccessfull run
+ process.returncode = 1
+
+ retcode = ssh.run()
+
+ self.assertTrue(log_sample in log['text'])
+ self.assertTrue(error_sample in log['text'])
+ self.assertEqual(retcode["exitstatus"], 1)
+
+ log['text'] = ""
+ # unsuccessful run with error message
+ process.returncode = 1
+
+ dummy_error_message = "dummy_error_message"
+ ssh = SSH(params.user, params.sshkey_file, "dummy-host", "dummy-command",
+ params.bootdir, host_log_mock, errorMessage= dummy_error_message)
+ retcode = ssh.run()
+
+ self.assertTrue(log_sample in log['text'])
+ self.assertTrue(error_sample in log['text'])
+ self.assertTrue(dummy_error_message in log['text'])
+ self.assertEqual(retcode["exitstatus"], 1)
+
+
+ def test_getOsCheckScript(self):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ ocs = bootstrap_obj.getOsCheckScript()
+ self.assertEquals(ocs, "scriptDir/os_check_type.py")
+
+
+ @patch.object(BootstrapDefault, "getRemoteName")
+ def test_getOsCheckScriptRemoteLocation(self, getRemoteName_mock):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ v = "/tmp/os_check_type1374259902.py"
+ getRemoteName_mock.return_value = v
+ ocs = bootstrap_obj.getOsCheckScriptRemoteLocation()
+ self.assertEquals(ocs, v)
+
+
+ @patch.object(BootstrapDefault, "is_suse")
+ def test_getRepoFile(self, is_suse_mock):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ is_suse_mock.return_value = False
+ rf = bootstrap_obj.getRepoFile()
+ self.assertEquals(rf, "/etc/yum.repos.d/ambari.repo")
+
+
+ @patch.object(SSH, "__init__")
+ @patch.object(SSH, "run")
+ @patch.object(HostLog, "write")
+ def test_createTargetDir(self, write_mock, run_mock,
+ init_mock):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ expected = 42
+ init_mock.return_value = None
+ run_mock.return_value = expected
+ res = bootstrap_obj.createTargetDir()
+ self.assertEquals(res, expected)
+ command = str(init_mock.call_args[0][3])
+ self.assertEqual(command,
+ "sudo mkdir -p /var/lib/ambari-agent/data/tmp ; "
+ "sudo chown -R root /var/lib/ambari-agent/data/tmp ; "
+ "sudo chmod 755 /var/lib/ambari-agent ; "
+ "sudo chmod 755 /var/lib/ambari-agent/data ; "
+ "sudo chmod 777 /var/lib/ambari-agent/data/tmp")
+
+ @patch.object(BootstrapDefault, "getOsCheckScript")
+ @patch.object(BootstrapDefault, "getOsCheckScriptRemoteLocation")
+ @patch.object(SCP, "__init__")
+ @patch.object(SCP, "run")
+ @patch.object(HostLog, "write")
+ def test_copyOsCheckScript(self, write_mock, run_mock, init_mock,
+ getOsCheckScriptRemoteLocation_mock, getOsCheckScript_mock):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ getOsCheckScript_mock.return_value = "OsCheckScript"
+ getOsCheckScriptRemoteLocation_mock.return_value = "OsCheckScriptRemoteLocation"
+ expected = 42
+ init_mock.return_value = None
+ run_mock.return_value = expected
+ res = bootstrap_obj.copyOsCheckScript()
+ self.assertEquals(res, expected)
+ input_file = str(init_mock.call_args[0][3])
+ remote_file = str(init_mock.call_args[0][4])
+ self.assertEqual(input_file, "OsCheckScript")
+ self.assertEqual(remote_file, "OsCheckScriptRemoteLocation")
+
+
+ @patch.object(BootstrapDefault, "getRemoteName")
+ @patch.object(BootstrapDefault, "hasPassword")
+ @patch.object(OSCheck, "is_suse_family")
+ @patch.object(OSCheck, "is_ubuntu_family")
+ @patch.object(OSCheck, "is_redhat_family")
+ def test_getRepoFile(self, is_redhat_family, is_ubuntu_family, is_suse_family, hasPassword_mock, getRemoteName_mock):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ is_redhat_family.return_value = True
+ is_ubuntu_family.return_value = False
+ is_suse_family.return_value = False
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ # Without password
+ hasPassword_mock.return_value = False
+ getRemoteName_mock.return_value = "RemoteName"
+ rf = bootstrap_obj.getMoveRepoFileCommand("target")
+ self.assertEquals(rf, "sudo mv RemoteName target/ambari.repo")
+ # With password
+ hasPassword_mock.return_value = True
+ getRemoteName_mock.return_value = "RemoteName"
+ rf = bootstrap_obj.getMoveRepoFileCommand("target")
+ self.assertEquals(rf, "sudo -S mv RemoteName target/ambari.repo < RemoteName")
+
+ @patch("os.path.exists")
+ @patch.object(OSCheck, "is_suse_family")
+ @patch.object(OSCheck, "is_ubuntu_family")
+ @patch.object(OSCheck, "is_redhat_family")
+ @patch.object(BootstrapDefault, "getMoveRepoFileCommand")
+ @patch.object(BootstrapDefault, "getRepoDir")
+ @patch.object(BootstrapDefault, "getRepoFile")
+ @patch.object(BootstrapDefault, "getRemoteName")
+ @patch.object(SCP, "__init__")
+ @patch.object(SCP, "run")
+ @patch.object(SSH, "__init__")
+ @patch.object(SSH, "run")
+ @patch.object(HostLog, "write")
+ def test_copyNeededFiles(self, write_mock, ssh_run_mock, ssh_init_mock,
+ scp_run_mock, scp_init_mock,
+ getRemoteName_mock, getRepoFile_mock, getRepoDir,
+ getMoveRepoFileCommand, is_redhat_family, is_ubuntu_family, is_suse_family,
+ os_path_exists_mock):
+ #
+ # Ambari repo file exists
+ #
+ def os_path_exists_side_effect(*args, **kwargs):
+ if args[0] == getRepoFile_mock():
+ return True
+ else:
+ return False
+
+ os_path_exists_mock.side_effect = os_path_exists_side_effect
+ os_path_exists_mock.return_value = None
+
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ is_redhat_family.return_value = True
+ is_ubuntu_family.return_value = False
+ is_suse_family.return_value = False
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ getMoveRepoFileCommand.return_value = "MoveRepoFileCommand"
+ getRepoDir.return_value = "RepoDir"
+ getRemoteName_mock.return_value = "RemoteName"
+ getRepoFile_mock.return_value = "RepoFile"
+ expected1 = {"exitstatus": 42, "log": "log42", "errormsg": "errorMsg"}
+ expected2 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"}
+ expected3 = {"exitstatus": 1, "log": "log1", "errormsg": "errorMsg"}
+ expected4 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"}
+ scp_init_mock.return_value = None
+ ssh_init_mock.return_value = None
+ # Testing max retcode return
+ scp_run_mock.side_effect = [expected1, expected3]
+ ssh_run_mock.side_effect = [expected2, expected4]
+ res = bootstrap_obj.copyNeededFiles()
+ self.assertEquals(res, expected1["exitstatus"])
+ input_file = str(scp_init_mock.call_args[0][3])
+ remote_file = str(scp_init_mock.call_args[0][4])
+ self.assertEqual(input_file, "setupAgentFile")
+ self.assertEqual(remote_file, "RemoteName")
+ command = str(ssh_init_mock.call_args[0][3])
+ self.assertEqual(command, "sudo chmod 644 RepoFile")
+ # Another order
+ expected1 = {"exitstatus": 0, "log": "log0", "errormsg": "errorMsg"}
+ expected2 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"}
+ expected3 = {"exitstatus": 1, "log": "log1", "errormsg": "errorMsg"}
+ expected4 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"}
+ scp_run_mock.side_effect = [expected1, expected3]
+ ssh_run_mock.side_effect = [expected2, expected4]
+ res = bootstrap_obj.copyNeededFiles()
+ self.assertEquals(res, expected2["exitstatus"])
+ # yet another order
+ expected1 = {"exitstatus": 33, "log": "log33", "errormsg": "errorMsg"}
+ expected2 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"}
+ expected3 = {"exitstatus": 42, "log": "log42", "errormsg": "errorMsg"}
+ expected4 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"}
+ scp_run_mock.side_effect = [expected1, expected3]
+ ssh_run_mock.side_effect = [expected2, expected4]
+ res = bootstrap_obj.copyNeededFiles()
+ self.assertEquals(res, expected3["exitstatus"])
+
+ #
+ #Ambari repo file does not exist
+ #
+ os_path_exists_mock.side_effect = None
+ os_path_exists_mock.return_value = False
+
+ #Expectations:
+ # SSH will not be called at all
+ # SCP will be called once for copying the setup script file
+ scp_run_mock.reset_mock()
+ ssh_run_mock.reset_mock()
+ expectedResult = {"exitstatus": 33, "log": "log33", "errormsg": "errorMsg"}
+ scp_run_mock.side_effect = [expectedResult]
+ res = bootstrap_obj.copyNeededFiles()
+ self.assertFalse(ssh_run_mock.called)
+ self.assertEquals(res, expectedResult["exitstatus"])
+
+ @patch.object(BootstrapDefault, "getOsCheckScriptRemoteLocation")
+ @patch.object(SSH, "__init__")
+ @patch.object(SSH, "run")
+ @patch.object(HostLog, "write")
+ def test_runOsCheckScript(self, write_mock, run_mock,
+ init_mock, getOsCheckScriptRemoteLocation_mock):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ getOsCheckScriptRemoteLocation_mock.return_value = "OsCheckScriptRemoteLocation"
+ expected = 42
+ init_mock.return_value = None
+ run_mock.return_value = expected
+ res = bootstrap_obj.runOsCheckScript()
+ self.assertEquals(res, expected)
+ command = str(init_mock.call_args[0][3])
+ self.assertEqual(command,
+ "chmod a+x OsCheckScriptRemoteLocation && "
+ "env PYTHONPATH=$PYTHONPATH:/var/lib/ambari-agent/data/tmp OsCheckScriptRemoteLocation centos6")
+
+
+ @patch.object(SSH, "__init__")
+ @patch.object(BootstrapDefault, "getRunSetupCommand")
+ @patch.object(SSH, "run")
+ @patch.object(HostLog, "write")
+ def test_runSetupAgent(self, write_mock, run_mock,
+ getRunSetupCommand_mock, init_mock):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ getRunSetupCommand_mock.return_value = "RunSetupCommand"
+ expected = 42
+ init_mock.return_value = None
+ run_mock.return_value = expected
+ res = bootstrap_obj.runSetupAgent()
+ self.assertEquals(res, expected)
+ command = str(init_mock.call_args[0][3])
+ self.assertEqual(command, "RunSetupCommand")
+
+
+ @patch.object(BootstrapDefault, "hasPassword")
+ @patch.object(BootstrapDefault, "getRunSetupWithPasswordCommand")
+ @patch.object(BootstrapDefault, "getRunSetupWithoutPasswordCommand")
+ def test_getRunSetupCommand(self, getRunSetupWithoutPasswordCommand_mock,
+ getRunSetupWithPasswordCommand_mock,
+ hasPassword_mock):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ # With password
+ hasPassword_mock.return_value = True
+ getRunSetupWithPasswordCommand_mock.return_value = "RunSetupWithPasswordCommand"
+ getRunSetupWithoutPasswordCommand_mock.return_value = "RunSetupWithoutPasswordCommand"
+ res = bootstrap_obj.getRunSetupCommand("dummy-host")
+ self.assertEqual(res, "RunSetupWithPasswordCommand")
+ # Without password
+ hasPassword_mock.return_value = False
+ res = bootstrap_obj.getRunSetupCommand("dummy-host")
+ self.assertEqual(res, "RunSetupWithoutPasswordCommand")
+
+
+ @patch.object(HostLog, "write")
+ def test_createDoneFile(self, write_mock):
+ tmp_dir = tempfile.gettempdir()
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", tmp_dir,
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ done_file = os.path.join(tmp_dir, "hostname.done")
+ expected = 42
+ bootstrap_obj.createDoneFile(expected)
+ with open(done_file) as df:
+ res = df.read()
+ self.assertEqual(res, str(expected))
+ os.unlink(done_file)
+
+ @patch.object(OSCheck, "is_suse_family")
+ @patch.object(OSCheck, "is_ubuntu_family")
+ @patch.object(OSCheck, "is_redhat_family")
+ @patch.object(SSH, "__init__")
+ @patch.object(SSH, "run")
+ @patch.object(HostLog, "write")
+ def test_checkSudoPackage(self, write_mock, run_mock, init_mock, is_redhat_family, is_ubuntu_family, is_suse_family):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ expected = 42
+ init_mock.return_value = None
+ run_mock.return_value = expected
+ is_redhat_family.return_value = True
+ is_ubuntu_family.return_value = False
+ is_suse_family.return_value = False
+ res = bootstrap_obj.checkSudoPackage()
+ self.assertEquals(res, expected)
+ command = str(init_mock.call_args[0][3])
+ self.assertEqual(command, "rpm -qa | grep -e '^sudo\-'")
+
+ @patch.object(OSCheck, "is_suse_family")
+ @patch.object(OSCheck, "is_ubuntu_family")
+ @patch.object(OSCheck, "is_redhat_family")
+ @patch.object(SSH, "__init__")
+ @patch.object(SSH, "run")
+ @patch.object(HostLog, "write")
+ def test_checkSudoPackageUbuntu(self, write_mock, run_mock, init_mock,
+ is_redhat_family, is_ubuntu_family, is_suse_family):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "ubuntu12",
+ None, "8440", "root")
+ is_redhat_family.return_value = False
+ is_ubuntu_family.return_value = True
+ is_suse_family.return_value = False
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ expected = 42
+ init_mock.return_value = None
+ run_mock.return_value = expected
+ res = bootstrap_obj.checkSudoPackage()
+ self.assertEquals(res, expected)
+ command = str(init_mock.call_args[0][3])
+ self.assertEqual(command, "dpkg --get-selections|grep -e '^sudo\s*install'")
+
+
+ @patch.object(SSH, "__init__")
+ @patch.object(SSH, "run")
+ @patch.object(HostLog, "write")
+ @patch.object(BootstrapDefault, "getPasswordFile")
+ def test_deletePasswordFile(self, getPasswordFile_mock, write_mock, run_mock,
+ init_mock):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ expected = 42
+ getPasswordFile_mock.return_value = "PasswordFile"
+ init_mock.return_value = None
+ run_mock.return_value = expected
+ res = bootstrap_obj.deletePasswordFile()
+ self.assertEquals(res, expected)
+ command = str(init_mock.call_args[0][3])
+ self.assertEqual(command, "rm PasswordFile")
+
+
+ @patch.object(BootstrapDefault, "getPasswordFile")
+ @patch.object(SCP, "__init__")
+ @patch.object(SCP, "run")
+ @patch.object(SSH, "__init__")
+ @patch.object(SSH, "run")
+ @patch.object(HostLog, "write")
+ def test_copyPasswordFile(self, write_mock, ssh_run_mock,
+ ssh_init_mock, scp_run_mock,
+ scp_init_mock, getPasswordFile_mock):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root", password_file="PasswordFile")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ getPasswordFile_mock.return_value = "PasswordFile"
+ # Testing max retcode return
+ expected1 = {"exitstatus": 42, "log": "log42", "errormsg": "errorMsg"}
+ expected2 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"}
+ scp_init_mock.return_value = None
+ scp_run_mock.return_value = expected1
+ ssh_init_mock.return_value = None
+ ssh_run_mock.return_value = expected2
+ res = bootstrap_obj.copyPasswordFile()
+ self.assertEquals(res, expected1["exitstatus"])
+ input_file = str(scp_init_mock.call_args[0][3])
+ remote_file = str(scp_init_mock.call_args[0][4])
+ self.assertEqual(input_file, "PasswordFile")
+ self.assertEqual(remote_file, "PasswordFile")
+ command = str(ssh_init_mock.call_args[0][3])
+ self.assertEqual(command, "chmod 600 PasswordFile")
+ # Another order
+ expected1 = {"exitstatus": 0, "log": "log0", "errormsg": "errorMsg"}
+ expected2 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"}
+ scp_run_mock.return_value = expected1
+ ssh_run_mock.return_value = expected2
+
+
+ @patch.object(SSH, "__init__")
+ @patch.object(SSH, "run")
+ @patch.object(HostLog, "write")
+ @patch.object(BootstrapDefault, "getPasswordFile")
+ def test_changePasswordFileModeOnHost(self, getPasswordFile_mock, write_mock,
+ run_mock, init_mock):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ expected = 42
+ getPasswordFile_mock.return_value = "PasswordFile"
+ init_mock.return_value = None
+ run_mock.return_value = expected
+ res = bootstrap_obj.changePasswordFileModeOnHost()
+ self.assertEquals(res, expected)
+ command = str(init_mock.call_args[0][3])
+ self.assertEqual(command, "chmod 600 PasswordFile")
+
+
+ @patch.object(HostLog, "write")
+ def test_try_to_execute(self, write_mock):
+ expected = 43
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ # Normal case
+ def act_normal_return_int():
+ return 43
+ ret = bootstrap_obj.try_to_execute(act_normal_return_int)
+ self.assertEqual(ret["exitstatus"], expected)
+ self.assertFalse(write_mock.called)
+ write_mock.reset_mock()
+ def act_normal_return():
+ return {"exitstatus": 43}
+ ret = bootstrap_obj.try_to_execute(act_normal_return)
+ self.assertEqual(ret["exitstatus"], expected)
+ self.assertFalse(write_mock.called)
+ write_mock.reset_mock()
+ # Exception scenario
+ def act():
+ raise IOError()
+ ret = bootstrap_obj.try_to_execute(act)
+ self.assertEqual(ret["exitstatus"], 177)
+ self.assertTrue(write_mock.called)
+
+
+ @patch.object(BootstrapDefault, "try_to_execute")
+ @patch.object(BootstrapDefault, "hasPassword")
+ @patch.object(BootstrapDefault, "createDoneFile")
+ @patch.object(HostLog, "write")
+ @patch("logging.warn")
+ @patch("logging.error")
+ def test_run(self, error_mock, warn_mock, write_mock, createDoneFile_mock,
+ hasPassword_mock, try_to_execute_mock):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ # Testing workflow without password
+ bootstrap_obj.copied_password_file = False
+ hasPassword_mock.return_value = False
+ try_to_execute_mock.return_value = {"exitstatus": 0, "log":"log0", "errormsg":"errormsg0"}
+ bootstrap_obj.run()
+ self.assertEqual(try_to_execute_mock.call_count, 7) # <- Adjust if changed
+ self.assertTrue(createDoneFile_mock.called)
+ self.assertEqual(bootstrap_obj.getStatus()["return_code"], 0)
+
+ try_to_execute_mock.reset_mock()
+ createDoneFile_mock.reset_mock()
+ # Testing workflow with password
+ bootstrap_obj.copied_password_file = True
+ hasPassword_mock.return_value = True
+ try_to_execute_mock.return_value = {"exitstatus": 0, "log":"log0", "errormsg":"errormsg0"}
+ bootstrap_obj.run()
+ self.assertEqual(try_to_execute_mock.call_count, 10) # <- Adjust if changed
+ self.assertTrue(createDoneFile_mock.called)
+ self.assertEqual(bootstrap_obj.getStatus()["return_code"], 0)
+
+ error_mock.reset_mock()
+ write_mock.reset_mock()
+ try_to_execute_mock.reset_mock()
+ createDoneFile_mock.reset_mock()
+ # Testing workflow when some action failed before copying password
+ bootstrap_obj.copied_password_file = False
+ hasPassword_mock.return_value = False
+ try_to_execute_mock.side_effect = [{"exitstatus": 0, "log":"log0", "errormsg":"errormsg0"}, {"exitstatus": 1, "log":"log1", "errormsg":"errormsg1"}]
+ bootstrap_obj.run()
+ self.assertEqual(try_to_execute_mock.call_count, 2) # <- Adjust if changed
+ self.assertTrue("ERROR" in error_mock.call_args[0][0])
+ self.assertTrue("ERROR" in write_mock.call_args[0][0])
+ self.assertTrue(createDoneFile_mock.called)
+ self.assertEqual(bootstrap_obj.getStatus()["return_code"], 1)
+
+ try_to_execute_mock.reset_mock()
+ createDoneFile_mock.reset_mock()
+ # Testing workflow when some action failed after copying password
+ bootstrap_obj.copied_password_file = True
+ hasPassword_mock.return_value = True
+ try_to_execute_mock.side_effect = [{"exitstatus": 0, "log":"log0", "errormsg":"errormsg0"}, {"exitstatus": 42, "log":"log42", "errormsg":"errormsg42"}, {"exitstatus": 0, "log":"log0", "errormsg":"errormsg0"}]
+ bootstrap_obj.run()
+ self.assertEqual(try_to_execute_mock.call_count, 3) # <- Adjust if changed
+ self.assertTrue(createDoneFile_mock.called)
+ self.assertEqual(bootstrap_obj.getStatus()["return_code"], 42)
+
+ error_mock.reset_mock()
+ write_mock.reset_mock()
+ try_to_execute_mock.reset_mock()
+ createDoneFile_mock.reset_mock()
+ # Testing workflow when some action failed after copying password and
+ # removing password failed too
+ bootstrap_obj.copied_password_file = True
+ hasPassword_mock.return_value = True
+ try_to_execute_mock.side_effect = [{"exitstatus": 0, "log":"log0", "errormsg":"errormsg0"}, {"exitstatus": 17, "log":"log17", "errormsg":"errormsg17"}, {"exitstatus": 19, "log":"log19", "errormsg":"errormsg19"}]
+ bootstrap_obj.run()
+ self.assertEqual(try_to_execute_mock.call_count, 3) # <- Adjust if changed
+ self.assertTrue("ERROR" in write_mock.call_args_list[0][0][0])
+ self.assertTrue("ERROR" in error_mock.call_args[0][0])
+ self.assertTrue("WARNING" in write_mock.call_args_list[1][0][0])
+ self.assertTrue("WARNING" in warn_mock.call_args[0][0])
+ self.assertTrue(createDoneFile_mock.called)
+ self.assertEqual(bootstrap_obj.getStatus()["return_code"], 17)
+
+
+ @patch.object(BootstrapDefault, "createDoneFile")
+ @patch.object(HostLog, "write")
+ def test_interruptBootstrap(self, write_mock, createDoneFile_mock):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ bootstrap_obj = Bootstrap("hostname", shared_state)
+ bootstrap_obj.interruptBootstrap()
+ self.assertTrue(createDoneFile_mock.called)
+
+
+ @patch("time.sleep")
+ @patch("time.time")
+ @patch("logging.warn")
+ @patch("logging.info")
+ @patch.object(BootstrapDefault, "start")
+ @patch.object(BootstrapDefault, "interruptBootstrap")
+ @patch.object(BootstrapDefault, "getStatus")
+ def test_PBootstrap(self, getStatus_mock, interruptBootstrap_mock, start_mock,
+ info_mock, warn_mock, time_mock, sleep_mock):
+ shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+ "setupAgentFile", "ambariServer", "centos6",
+ None, "8440", "root")
+ n = 180
+ time = 100500
+ time_mock.return_value = time
+ hosts = []
+ for i in range(0, n):
+ hosts.append("host" + str(i))
+ # Testing normal case
+ getStatus_mock.return_value = {"return_code": 0,
+ "start_time": time + 999}
+ pbootstrap_obj = PBootstrap(hosts, shared_state)
+ pbootstrap_obj.run()
+ self.assertEqual(start_mock.call_count, n)
+ self.assertEqual(interruptBootstrap_mock.call_count, 0)
+
+ start_mock.reset_mock()
+ getStatus_mock.reset_mock()
+ # Testing case of timeout
+ def fake_return_code_generator():
+ call_number = 0
+ while True:
+ call_number += 1
+ if call_number % 5 != 0: # ~80% of hosts finish successfully
+ yield 0
+ else:
+ yield None
+
+ def fake_start_time_generator():
+ while True:
+ yield time - bootstrap.HOST_BOOTSTRAP_TIMEOUT - 1
+
+ return_code_generator = fake_return_code_generator()
+ start_time_generator = fake_start_time_generator()
+
+ def status_get_item_mock(item):
+ if item == "return_code":
+ return return_code_generator.next()
+ elif item == "start_time":
+ return start_time_generator.next()
+
+ dict_mock = MagicMock()
+ dict_mock.__getitem__.side_effect = status_get_item_mock
+ getStatus_mock.return_value = dict_mock
+
+ pbootstrap_obj.run()
+ self.assertEqual(start_mock.call_count, n)
+ self.assertEqual(interruptBootstrap_mock.call_count, n / 5)
+
[2/2] ambari git commit: Verify if restricting acls on
/var/lib/ambari-agent/data will be OK (aonishuk)
Posted by ao...@apache.org.
Verify if restricting acls on /var/lib/ambari-agent/data will be OK (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b8388e12
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b8388e12
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b8388e12
Branch: refs/heads/trunk
Commit: b8388e12b348e3989490e812dd2fe993b2ca603e
Parents: e837d4f
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Mon Sep 14 15:01:01 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Mon Sep 14 15:01:01 2015 +0300
----------------------------------------------------------------------
ambari-agent/conf/unix/ambari-agent.ini | 1 -
ambari-agent/conf/unix/install-helper.sh | 5 +-
ambari-agent/pom.xml | 4 +-
.../src/main/python/ambari_agent/Constants.py | 21 +
.../ambari_agent/CustomServiceOrchestrator.py | 3 +-
.../python/ambari_agent/alerts/metric_alert.py | 3 +-
.../python/ambari_agent/alerts/script_alert.py | 3 +-
.../python/ambari_agent/alerts/web_alert.py | 3 +-
.../src/main/python/ambari_agent/security.py | 11 +-
.../python/ambari_agent/TestCertGeneration.py | 6 +-
.../ambari_agent/TestCertGeneration.py.orig | 50 ++
.../TestCustomServiceOrchestrator.py | 1 -
.../TestCustomServiceOrchestrator.py.orig | 643 ++++++++++++++
.../test/python/ambari_agent/TestSecurity.py | 6 +-
.../python/ambari_agent/TestSecurity.py.orig | 389 ++++++++
.../libraries/script/script.py | 2 +-
ambari-server/src/main/python/bootstrap.py | 2 +-
.../HDFS/2.1.0.2.0/package/scripts/namenode.py | 2 +-
ambari-server/src/test/python/TestBootstrap.py | 12 +-
.../src/test/python/TestBootstrap.py.orig | 879 +++++++++++++++++++
20 files changed, 2020 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/conf/unix/ambari-agent.ini
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/unix/ambari-agent.ini b/ambari-agent/conf/unix/ambari-agent.ini
index abfde62..3b7631c 100644
--- a/ambari-agent/conf/unix/ambari-agent.ini
+++ b/ambari-agent/conf/unix/ambari-agent.ini
@@ -19,7 +19,6 @@ secured_url_port=8441
[agent]
prefix=/var/lib/ambari-agent/data
-tmp_dir=/var/lib/ambari-agent/data/tmp
;loglevel=(DEBUG/INFO)
loglevel=INFO
data_cleanup_interval=86400
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/conf/unix/install-helper.sh
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/unix/install-helper.sh b/ambari-agent/conf/unix/install-helper.sh
index 48391d5..35c67fb 100644
--- a/ambari-agent/conf/unix/install-helper.sh
+++ b/ambari-agent/conf/unix/install-helper.sh
@@ -58,8 +58,9 @@ do_install(){
# on nano Ubuntu, when umask=027 those folders are created without 'x' bit for 'others'.
# which causes failures when hadoop users try to access tmp_dir
chmod a+x /var/lib/ambari-agent
- chmod a+x /var/lib/ambari-agent/data
- chmod 777 /var/lib/ambari-agent/data/tmp
+
+ chmod 777 /var/lib/ambari-agent/tmp
+ chmod 700 /var/lib/ambari-agent/data
}
do_remove(){
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-agent/pom.xml b/ambari-agent/pom.xml
index 24927e2..0096162 100644
--- a/ambari-agent/pom.xml
+++ b/ambari-agent/pom.xml
@@ -391,7 +391,7 @@
<groupname>root</groupname>
</mapping>
<mapping>
- <directory>/var/lib/${project.artifactId}/data/tmp</directory>
+ <directory>/var/lib/${project.artifactId}/tmp</directory>
<filemode>777</filemode>
<username>root</username>
<groupname>root</groupname>
@@ -596,7 +596,7 @@
<paths>
<path>${package.pid.dir}</path>
<path>/var/lib/${project.artifactId}/data</path>
- <path>/var/lib/${project.artifactId}/data/tmp</path>
+ <path>/var/lib/${project.artifactId}/tmp</path>
<path>/var/lib/${project.artifactId}/keys</path>
<path>${package.log.dir}</path>
<path>/var/lib/${project.artifactId}/lib</path>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/main/python/ambari_agent/Constants.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/Constants.py b/ambari-agent/src/main/python/ambari_agent/Constants.py
new file mode 100644
index 0000000..6b80f00
--- /dev/null
+++ b/ambari-agent/src/main/python/ambari_agent/Constants.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+AGENT_TMP_DIR = "/var/lib/ambari-agent/tmp"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
index 6ee929c..6c1a161 100644
--- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
+++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
@@ -29,6 +29,7 @@ from FileCache import FileCache
from AgentException import AgentException
from PythonExecutor import PythonExecutor
from PythonReflectiveExecutor import PythonReflectiveExecutor
+import Constants
import hostname
@@ -61,7 +62,7 @@ class CustomServiceOrchestrator():
def __init__(self, config, controller):
self.config = config
self.tmp_dir = config.get('agent', 'prefix')
- self.exec_tmp_dir = config.get('agent', 'tmp_dir')
+ self.exec_tmp_dir = Constants.AGENT_TMP_DIR
self.file_cache = FileCache(config)
self.status_commands_stdout = os.path.join(self.tmp_dir,
'status_command_stdout.txt')
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
index aa4ad75..8de49cd 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
@@ -30,6 +30,7 @@ from alerts.base_alert import BaseAlert
from ambari_commons.urllib_handlers import RefreshHeaderProcessor
from resource_management.libraries.functions.get_port_from_url import get_port_from_url
from resource_management.libraries.functions.curl_krb_request import curl_krb_request
+from ambari_agent import Constants
logger = logging.getLogger()
@@ -199,7 +200,7 @@ class MetricAlert(BaseAlert):
content = ''
try:
if kerberos_principal is not None and kerberos_keytab is not None and security_enabled:
- tmp_dir = self.config.get('agent', 'tmp_dir')
+ tmp_dir = Constants.AGENT_TMP_DIR
if tmp_dir is None:
tmp_dir = gettempdir()
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py
index 76afbc9..e70dc22 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py
@@ -25,6 +25,7 @@ import re
from alerts.base_alert import BaseAlert
from resource_management.core.environment import Environment
from resource_management.core.logger import Logger
+from ambari_agent import Constants
logger = logging.getLogger(__name__)
@@ -99,7 +100,7 @@ class ScriptAlert(BaseAlert):
matchObj = re.match( r'((.*)services(.*)package)', self.path_to_script)
if matchObj:
basedir = matchObj.group(1)
- with Environment(basedir, tmp_dir=self.config.get('agent', 'tmp_dir')) as env:
+ with Environment(basedir, tmp_dir=Constants.AGENT_TMP_DIR) as env:
return cmd_module.execute(configurations, self.parameters, self.host_name)
else:
return cmd_module.execute(configurations, self.parameters, self.host_name)
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
index b76d5e0..0d627c3 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
@@ -31,6 +31,7 @@ from resource_management.libraries.functions.get_port_from_url import get_port_f
from resource_management.libraries.functions.curl_krb_request import curl_krb_request
from ambari_commons import OSCheck
from ambari_commons.inet_utils import resolve_address
+from ambari_agent import Constants
# hashlib is supplied as of Python 2.5 as the replacement interface for md5
# and other secure hashes. In 2.6, md5 is deprecated. Import hashlib if
@@ -171,7 +172,7 @@ class WebAlert(BaseAlert):
# Create the kerberos credentials cache (ccache) file and set it in the environment to use
# when executing curl. Use the md5 hash of the combination of the principal and keytab file
# to generate a (relatively) unique cache filename so that we can use it as needed.
- tmp_dir = self.config.get('agent', 'tmp_dir')
+ tmp_dir = Constants.AGENT_TMP_DIR
if tmp_dir is None:
tmp_dir = gettempdir()
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/main/python/ambari_agent/security.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/security.py b/ambari-agent/src/main/python/ambari_agent/security.py
index bfaf134..dc1f141 100644
--- a/ambari-agent/src/main/python/ambari_agent/security.py
+++ b/ambari-agent/src/main/python/ambari_agent/security.py
@@ -35,6 +35,7 @@ logger = logging.getLogger(__name__)
GEN_AGENT_KEY = 'openssl req -new -newkey rsa:1024 -nodes -keyout "%(keysdir)s' \
+ os.sep + '%(hostname)s.key" -subj /OU=%(hostname)s/ ' \
'-out "%(keysdir)s' + os.sep + '%(hostname)s.csr"'
+KEY_FILENAME = '%(hostname)s.key'
class VerifiedHTTPSConnection(httplib.HTTPSConnection):
@@ -190,7 +191,7 @@ class CertificateManager():
if not agent_key_exists:
logger.info("Agent key not exists, generating request")
- self.genAgentCrtReq()
+ self.genAgentCrtReq(self.getAgentKeyName())
else:
logger.info("Agent key exists, ok")
@@ -252,10 +253,12 @@ class CertificateManager():
'\nExiting..')
raise ssl.SSLError
- def genAgentCrtReq(self):
+ def genAgentCrtReq(self, keyname):
+ keysdir = os.path.abspath(self.config.get('security', 'keysdir'))
generate_script = GEN_AGENT_KEY % {
'hostname': hostname.hostname(self.config),
- 'keysdir': os.path.abspath(self.config.get('security', 'keysdir'))}
+ 'keysdir': keysdir}
+
logger.info(generate_script)
if platform.system() == 'Windows':
p = subprocess.Popen(generate_script, stdout=subprocess.PIPE)
@@ -264,6 +267,8 @@ class CertificateManager():
p = subprocess.Popen([generate_script], shell=True,
stdout=subprocess.PIPE)
p.communicate()
+ # this is required to be 600 for security concerns.
+ os.chmod(keyname, 0600)
def initSecurity(self):
self.checkCertExists()
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py b/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py
index 2461281..1cf7866 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py
@@ -41,8 +41,10 @@ class TestCertGeneration(TestCase):
config.set('security', 'server_crt', 'ca.crt')
self.certMan = CertificateManager(config)
- def test_generation(self):
- self.certMan.genAgentCrtReq()
+ @patch.object(os, "chmod")
+ def test_generation(self, chmod_mock):
+ self.certMan.genAgentCrtReq('/dummy_dir/hostname.key')
+ self.assertTrue(chmod_mock.called)
self.assertTrue(os.path.exists(self.certMan.getAgentKeyName()))
self.assertTrue(os.path.exists(self.certMan.getAgentCrtReqName()))
def tearDown(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py.orig
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py.orig b/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py.orig
new file mode 100644
index 0000000..2461281
--- /dev/null
+++ b/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py.orig
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import os
+import tempfile
+import shutil
+from unittest import TestCase
+
+from ambari_agent.security import CertificateManager
+from ambari_agent import AmbariConfig
+from mock.mock import patch, MagicMock
+from ambari_commons import OSCheck
+from only_for_platform import os_distro_value
+
+class TestCertGeneration(TestCase):
+ @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+ def setUp(self):
+ self.tmpdir = tempfile.mkdtemp()
+ config = AmbariConfig.AmbariConfig()
+ #config.add_section('server')
+ config.set('server', 'hostname', 'example.com')
+ config.set('server', 'url_port', '777')
+ #config.add_section('security')
+ config.set('security', 'keysdir', self.tmpdir)
+ config.set('security', 'server_crt', 'ca.crt')
+ self.certMan = CertificateManager(config)
+
+ def test_generation(self):
+ self.certMan.genAgentCrtReq()
+ self.assertTrue(os.path.exists(self.certMan.getAgentKeyName()))
+ self.assertTrue(os.path.exists(self.certMan.getAgentCrtReqName()))
+ def tearDown(self):
+ shutil.rmtree(self.tmpdir)
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
index 8957b44..a4025eb 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
@@ -56,7 +56,6 @@ class TestCustomServiceOrchestrator(TestCase):
self.config = ConfigParser.RawConfigParser()
self.config.add_section('agent')
self.config.set('agent', 'prefix', tmpdir)
- self.config.set('agent', 'tmp_dir', exec_tmp_dir)
self.config.set('agent', 'cache_dir', "/cachedir")
self.config.add_section('python')
self.config.set('python', 'custom_actions_dir', tmpdir)
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py.orig
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py.orig b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py.orig
new file mode 100644
index 0000000..8957b44
--- /dev/null
+++ b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py.orig
@@ -0,0 +1,643 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import ConfigParser
+from multiprocessing.pool import ThreadPool
+import os
+
+import pprint
+from ambari_commons import shell
+
+from unittest import TestCase
+import threading
+import tempfile
+import time
+from threading import Thread
+
+from mock.mock import MagicMock, patch
+import StringIO
+import sys
+
+from ambari_agent.ActionQueue import ActionQueue
+from ambari_agent.AgentException import AgentException
+from ambari_agent.AmbariConfig import AmbariConfig
+from ambari_agent.BackgroundCommandExecutionHandle import BackgroundCommandExecutionHandle
+from ambari_agent.CustomServiceOrchestrator import CustomServiceOrchestrator
+from ambari_agent.FileCache import FileCache
+from ambari_agent.PythonExecutor import PythonExecutor
+from ambari_commons import OSCheck
+from only_for_platform import get_platform, os_distro_value, PLATFORM_WINDOWS
+
+class TestCustomServiceOrchestrator(TestCase):
+
+ def setUp(self):
+ # disable stdout
+ out = StringIO.StringIO()
+ sys.stdout = out
+ # generate sample config
+ tmpdir = tempfile.gettempdir()
+ exec_tmp_dir = os.path.join(tmpdir, 'tmp')
+ self.config = ConfigParser.RawConfigParser()
+ self.config.add_section('agent')
+ self.config.set('agent', 'prefix', tmpdir)
+ self.config.set('agent', 'tmp_dir', exec_tmp_dir)
+ self.config.set('agent', 'cache_dir', "/cachedir")
+ self.config.add_section('python')
+ self.config.set('python', 'custom_actions_dir', tmpdir)
+
+
+ @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+ @patch.object(FileCache, "__init__")
+ def test_add_reg_listener_to_controller(self, FileCache_mock):
+ FileCache_mock.return_value = None
+ dummy_controller = MagicMock()
+ config = AmbariConfig().getConfig()
+ tempdir = tempfile.gettempdir()
+ config.set('agent', 'prefix', tempdir)
+ CustomServiceOrchestrator(config, dummy_controller)
+ self.assertTrue(dummy_controller.registration_listeners.append.called)
+
+
+ @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+ @patch.object(CustomServiceOrchestrator, 'decompressClusterHostInfo')
+ @patch("ambari_agent.hostname.public_hostname")
+ @patch("os.path.isfile")
+ @patch("os.unlink")
+ @patch.object(FileCache, "__init__")
+ def test_dump_command_to_json(self, FileCache_mock, unlink_mock,
+ isfile_mock, hostname_mock,
+ decompress_cluster_host_info_mock):
+ FileCache_mock.return_value = None
+ hostname_mock.return_value = "test.hst"
+ command = {
+ 'commandType': 'EXECUTION_COMMAND',
+ 'role': u'DATANODE',
+ 'roleCommand': u'INSTALL',
+ 'commandId': '1-1',
+ 'taskId': 3,
+ 'clusterName': u'cc',
+ 'serviceName': u'HDFS',
+ 'configurations':{'global' : {}},
+ 'configurationTags':{'global' : { 'tag': 'v1' }},
+ 'clusterHostInfo':{'namenode_host' : ['1'],
+ 'slave_hosts' : ['0', '1'],
+ 'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
+ 'all_ping_ports': ['8670:0,1']},
+ 'hostLevelParams':{}
+ }
+
+ decompress_cluster_host_info_mock.return_value = {'namenode_host' : ['h2.hortonworks.com'],
+ 'slave_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
+ 'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
+ 'all_ping_ports': ['8670', '8670']}
+
+ config = AmbariConfig().getConfig()
+ tempdir = tempfile.gettempdir()
+ config.set('agent', 'prefix', tempdir)
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ isfile_mock.return_value = True
+ # Test dumping EXECUTION_COMMAND
+ json_file = orchestrator.dump_command_to_json(command)
+ self.assertTrue(os.path.exists(json_file))
+ self.assertTrue(os.path.getsize(json_file) > 0)
+ if get_platform() != PLATFORM_WINDOWS:
+ self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
+ self.assertTrue(json_file.endswith("command-3.json"))
+ self.assertTrue(decompress_cluster_host_info_mock.called)
+ os.unlink(json_file)
+ # Test dumping STATUS_COMMAND
+ command['commandType']='STATUS_COMMAND'
+ decompress_cluster_host_info_mock.reset_mock()
+ json_file = orchestrator.dump_command_to_json(command)
+ self.assertTrue(os.path.exists(json_file))
+ self.assertTrue(os.path.getsize(json_file) > 0)
+ if get_platform() != PLATFORM_WINDOWS:
+ self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
+ self.assertTrue(json_file.endswith("status_command.json"))
+ self.assertFalse(decompress_cluster_host_info_mock.called)
+ os.unlink(json_file)
+ # Testing side effect of dump_command_to_json
+ self.assertEquals(command['public_hostname'], "test.hst")
+ self.assertTrue(unlink_mock.called)
+
+
+ @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+ @patch("ambari_agent.hostname.public_hostname")
+ @patch("os.path.isfile")
+ @patch("os.unlink")
+ @patch.object(FileCache, "__init__")
+ def test_dump_command_to_json_with_retry(self, FileCache_mock, unlink_mock,
+ isfile_mock, hostname_mock):
+ FileCache_mock.return_value = None
+ hostname_mock.return_value = "test.hst"
+ command = {
+ 'commandType': 'EXECUTION_COMMAND',
+ 'role': u'DATANODE',
+ 'roleCommand': u'INSTALL',
+ 'commandId': '1-1',
+ 'taskId': 3,
+ 'clusterName': u'cc',
+ 'serviceName': u'HDFS',
+ 'configurations':{'global' : {}},
+ 'configurationTags':{'global' : { 'tag': 'v1' }},
+ 'clusterHostInfo':{'namenode_host' : ['1'],
+ 'slave_hosts' : ['0', '1'],
+ 'all_racks' : [u'/default-rack:0'],
+ 'ambari_server_host' : 'a.b.c',
+ 'all_ipv4_ips' : [u'192.168.12.101:0'],
+ 'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
+ 'all_ping_ports': ['8670:0,1']},
+ 'hostLevelParams':{}
+ }
+
+ config = AmbariConfig().getConfig()
+ tempdir = tempfile.gettempdir()
+ config.set('agent', 'prefix', tempdir)
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ isfile_mock.return_value = True
+ # Test dumping EXECUTION_COMMAND
+ json_file = orchestrator.dump_command_to_json(command)
+ self.assertTrue(os.path.exists(json_file))
+ self.assertTrue(os.path.getsize(json_file) > 0)
+ if get_platform() != PLATFORM_WINDOWS:
+ self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
+ self.assertTrue(json_file.endswith("command-3.json"))
+ os.unlink(json_file)
+ # Test dumping STATUS_COMMAND
+ json_file = orchestrator.dump_command_to_json(command, True)
+ self.assertTrue(os.path.exists(json_file))
+ self.assertTrue(os.path.getsize(json_file) > 0)
+ if get_platform() != PLATFORM_WINDOWS:
+ self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
+ self.assertTrue(json_file.endswith("command-3.json"))
+ os.unlink(json_file)
+ # Testing side effect of dump_command_to_json
+ self.assertEquals(command['public_hostname'], "test.hst")
+ self.assertTrue(unlink_mock.called)
+
+ @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+ @patch("os.path.exists")
+ @patch.object(FileCache, "__init__")
+ def test_resolve_script_path(self, FileCache_mock, exists_mock):
+ FileCache_mock.return_value = None
+ dummy_controller = MagicMock()
+ config = AmbariConfig().getConfig()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ # Testing existing path
+ exists_mock.return_value = True
+ path = orchestrator.\
+ resolve_script_path(os.path.join("HBASE", "package"), os.path.join("scripts", "hbase_master.py"))
+ self.assertEqual(os.path.join("HBASE", "package", "scripts", "hbase_master.py"), path)
+ # Testing not existing path
+ exists_mock.return_value = False
+ try:
+ orchestrator.resolve_script_path("/HBASE",
+ os.path.join("scripts", "hbase_master.py"))
+ self.fail('ExpectedException not thrown')
+ except AgentException:
+ pass # Expected
+
+
+ @patch.object(CustomServiceOrchestrator, "resolve_script_path")
+ @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
+ @patch.object(FileCache, "get_host_scripts_base_dir")
+ @patch.object(FileCache, "get_service_base_dir")
+ @patch.object(FileCache, "get_hook_base_dir")
+ @patch.object(CustomServiceOrchestrator, "dump_command_to_json")
+ @patch.object(PythonExecutor, "run_file")
+ @patch.object(FileCache, "__init__")
+ def test_runCommand(self, FileCache_mock,
+ run_file_mock, dump_command_to_json_mock,
+ get_hook_base_dir_mock, get_service_base_dir_mock,
+ get_host_scripts_base_dir_mock,
+ resolve_hook_script_path_mock,
+ resolve_script_path_mock):
+
+ FileCache_mock.return_value = None
+ command = {
+ 'role' : 'REGION_SERVER',
+ 'hostLevelParams' : {
+ 'stack_name' : 'HDP',
+ 'stack_version' : '2.0.7',
+ 'jdk_location' : 'some_location'
+ },
+ 'commandParams': {
+ 'script_type': 'PYTHON',
+ 'script': 'scripts/hbase_regionserver.py',
+ 'command_timeout': '600',
+ 'service_package_folder' : 'HBASE'
+ },
+ 'taskId' : '3',
+ 'roleCommand': 'INSTALL'
+ }
+
+ get_host_scripts_base_dir_mock.return_value = "/host_scripts"
+ get_service_base_dir_mock.return_value = "/basedir/"
+ resolve_script_path_mock.return_value = "/basedir/scriptpath"
+ resolve_hook_script_path_mock.return_value = \
+ ('/hooks_dir/prefix-command/scripts/hook.py',
+ '/hooks_dir/prefix-command')
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+ unix_process_id = 111
+ orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
+ get_hook_base_dir_mock.return_value = "/hooks/"
+ # normal run case
+ run_file_mock.return_value = {
+ 'stdout' : 'sss',
+ 'stderr' : 'eee',
+ 'exitcode': 0,
+ }
+ ret = orchestrator.runCommand(command, "out.txt", "err.txt")
+ self.assertEqual(ret['exitcode'], 0)
+ self.assertTrue(run_file_mock.called)
+ self.assertEqual(run_file_mock.call_count, 3)
+
+ run_file_mock.reset_mock()
+
+ # Case when we force another command
+ run_file_mock.return_value = {
+ 'stdout' : 'sss',
+ 'stderr' : 'eee',
+ 'exitcode': 0,
+ }
+ ret = orchestrator.runCommand(command, "out.txt", "err.txt",
+ forced_command_name=CustomServiceOrchestrator.SCRIPT_TYPE_PYTHON)
+ ## Check that override_output_files was true only during first call
+ print run_file_mock
+ self.assertEquals(run_file_mock.call_args_list[0][0][8], True)
+ self.assertEquals(run_file_mock.call_args_list[1][0][8], False)
+ self.assertEquals(run_file_mock.call_args_list[2][0][8], False)
+ ## Check that forced_command_name was taken into account
+ self.assertEqual(run_file_mock.call_args_list[0][0][1][0],
+ CustomServiceOrchestrator.SCRIPT_TYPE_PYTHON)
+
+ run_file_mock.reset_mock()
+
+ # unknown script type case
+ command['commandParams']['script_type'] = "SOME_TYPE"
+ ret = orchestrator.runCommand(command, "out.txt", "err.txt")
+ self.assertEqual(ret['exitcode'], 1)
+ self.assertFalse(run_file_mock.called)
+ self.assertTrue("Unknown script type" in ret['stdout'])
+
+ #By default returns empty dictionary
+ self.assertEqual(ret['structuredOut'], '{}')
+
+ pass
+
+ @patch("ambari_commons.shell.kill_process_with_children")
+ @patch.object(CustomServiceOrchestrator, "resolve_script_path")
+ @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
+ @patch.object(FileCache, "get_host_scripts_base_dir")
+ @patch.object(FileCache, "get_service_base_dir")
+ @patch.object(FileCache, "get_hook_base_dir")
+ @patch.object(CustomServiceOrchestrator, "dump_command_to_json")
+ @patch.object(PythonExecutor, "run_file")
+ @patch.object(FileCache, "__init__")
+ def test_cancel_command(self, FileCache_mock,
+ run_file_mock, dump_command_to_json_mock,
+ get_hook_base_dir_mock, get_service_base_dir_mock,
+ get_host_scripts_base_dir_mock,
+ resolve_hook_script_path_mock, resolve_script_path_mock,
+ kill_process_with_children_mock):
+ FileCache_mock.return_value = None
+ command = {
+ 'role' : 'REGION_SERVER',
+ 'hostLevelParams' : {
+ 'stack_name' : 'HDP',
+ 'stack_version' : '2.0.7',
+ 'jdk_location' : 'some_location'
+ },
+ 'commandParams': {
+ 'script_type': 'PYTHON',
+ 'script': 'scripts/hbase_regionserver.py',
+ 'command_timeout': '600',
+ 'service_package_folder' : 'HBASE'
+ },
+ 'taskId' : '3',
+ 'roleCommand': 'INSTALL'
+ }
+
+ get_host_scripts_base_dir_mock.return_value = "/host_scripts"
+ get_service_base_dir_mock.return_value = "/basedir/"
+ resolve_script_path_mock.return_value = "/basedir/scriptpath"
+ resolve_hook_script_path_mock.return_value = \
+ ('/hooks_dir/prefix-command/scripts/hook.py',
+ '/hooks_dir/prefix-command')
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+ unix_process_id = 111
+ orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
+ get_hook_base_dir_mock.return_value = "/hooks/"
+ run_file_mock_return_value = {
+ 'stdout' : 'killed',
+ 'stderr' : 'killed',
+ 'exitcode': 1,
+ }
+ def side_effect(*args, **kwargs):
+ time.sleep(0.2)
+ return run_file_mock_return_value
+ run_file_mock.side_effect = side_effect
+
+ _, out = tempfile.mkstemp()
+ _, err = tempfile.mkstemp()
+ pool = ThreadPool(processes=1)
+ async_result = pool.apply_async(orchestrator.runCommand, (command, out, err))
+
+ time.sleep(0.1)
+ orchestrator.cancel_command(command['taskId'], 'reason')
+
+ ret = async_result.get()
+
+ self.assertEqual(ret['exitcode'], 1)
+ self.assertEquals(ret['stdout'], 'killed\nCommand aborted. reason')
+ self.assertEquals(ret['stderr'], 'killed\nCommand aborted. reason')
+
+ self.assertTrue(kill_process_with_children_mock.called)
+ self.assertFalse(command['taskId'] in orchestrator.commands_in_progress.keys())
+ self.assertTrue(os.path.exists(out))
+ self.assertTrue(os.path.exists(err))
+ try:
+ os.remove(out)
+ os.remove(err)
+ except:
+ pass
+
+ from ambari_agent.StackVersionsFileHandler import StackVersionsFileHandler
+
+ @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+ @patch.object(CustomServiceOrchestrator, "get_py_executor")
+ @patch("ambari_commons.shell.kill_process_with_children")
+ @patch.object(FileCache, "__init__")
+ @patch.object(CustomServiceOrchestrator, "resolve_script_path")
+ @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
+ @patch.object(StackVersionsFileHandler, "read_stack_version")
+ def test_cancel_backgound_command(self, read_stack_version_mock, resolve_hook_script_path_mock,
+ resolve_script_path_mock, FileCache_mock, kill_process_with_children_mock,
+ get_py_executor_mock):
+ FileCache_mock.return_value = None
+ FileCache_mock.cache_dir = MagicMock()
+ resolve_hook_script_path_mock.return_value = None
+# shell.kill_process_with_children = MagicMock()
+ dummy_controller = MagicMock()
+ cfg = AmbariConfig()
+ cfg.set('agent', 'tolerate_download_failures', 'true')
+ cfg.set('agent', 'prefix', '.')
+ cfg.set('agent', 'cache_dir', 'background_tasks')
+
+ actionQueue = ActionQueue(cfg, dummy_controller)
+
+ dummy_controller.actionQueue = actionQueue
+ orchestrator = CustomServiceOrchestrator(cfg, dummy_controller)
+ orchestrator.file_cache = MagicMock()
+ def f (a, b):
+ return ""
+ orchestrator.file_cache.get_service_base_dir = f
+ actionQueue.customServiceOrchestrator = orchestrator
+
+ import TestActionQueue
+ import copy
+
+ pyex = PythonExecutor(actionQueue.customServiceOrchestrator.tmp_dir, actionQueue.customServiceOrchestrator.config)
+ TestActionQueue.patch_output_file(pyex)
+ pyex.prepare_process_result = MagicMock()
+ get_py_executor_mock.return_value = pyex
+ orchestrator.dump_command_to_json = MagicMock()
+
+ lock = threading.RLock()
+ complete_done = threading.Condition(lock)
+
+ complete_was_called = {}
+ def command_complete_w(process_condenced_result, handle):
+ with lock:
+ complete_was_called['visited']= ''
+ complete_done.wait(3)
+
+ actionQueue.on_background_command_complete_callback = TestActionQueue.wraped(actionQueue.on_background_command_complete_callback, command_complete_w, None)
+ execute_command = copy.deepcopy(TestActionQueue.TestActionQueue.background_command)
+ actionQueue.put([execute_command])
+ actionQueue.processBackgroundQueueSafeEmpty()
+
+ time.sleep(.1)
+
+ orchestrator.cancel_command(19,'')
+ self.assertTrue(kill_process_with_children_mock.called)
+ kill_process_with_children_mock.assert_called_with(33)
+
+ with lock:
+ complete_done.notifyAll()
+
+ with lock:
+ self.assertTrue(complete_was_called.has_key('visited'))
+
+ time.sleep(.1)
+
+ runningCommand = actionQueue.commandStatuses.get_command_status(19)
+ self.assertTrue(runningCommand is not None)
+ self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS)
+
+
+ @patch.object(CustomServiceOrchestrator, "dump_command_to_json")
+ @patch.object(PythonExecutor, "run_file")
+ @patch.object(FileCache, "__init__")
+ @patch.object(FileCache, "get_custom_actions_base_dir")
+ def test_runCommand_custom_action(self, get_custom_actions_base_dir_mock,
+ FileCache_mock,
+ run_file_mock, dump_command_to_json_mock):
+ FileCache_mock.return_value = None
+ get_custom_actions_base_dir_mock.return_value = "some path"
+ _, script = tempfile.mkstemp()
+ command = {
+ 'role' : 'any',
+ 'commandParams': {
+ 'script_type': 'PYTHON',
+ 'script': 'some_custom_action.py',
+ 'command_timeout': '600',
+ 'jdk_location' : 'some_location'
+ },
+ 'taskId' : '3',
+ 'roleCommand': 'ACTIONEXECUTE'
+ }
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+ unix_process_id = 111
+ orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
+ # normal run case
+ run_file_mock.return_value = {
+ 'stdout' : 'sss',
+ 'stderr' : 'eee',
+ 'exitcode': 0,
+ }
+ ret = orchestrator.runCommand(command, "out.txt", "err.txt")
+ self.assertEqual(ret['exitcode'], 0)
+ self.assertTrue(run_file_mock.called)
+ # Hoooks are not supported for custom actions,
+ # that's why run_file() should be called only once
+ self.assertEqual(run_file_mock.call_count, 1)
+
+
+ @patch("os.path.isfile")
+ @patch.object(FileCache, "__init__")
+ def test_resolve_hook_script_path(self, FileCache_mock, isfile_mock):
+ FileCache_mock.return_value = None
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+ # Testing None param
+ res1 = orchestrator.resolve_hook_script_path(None, "prefix", "command",
+ "script_type")
+ self.assertEqual(res1, None)
+ # Testing existing hook script
+ isfile_mock.return_value = True
+ res2 = orchestrator.resolve_hook_script_path("hooks_dir", "prefix", "command",
+ "script_type")
+ self.assertEqual(res2, (os.path.join('hooks_dir', 'prefix-command', 'scripts', 'hook.py'),
+ os.path.join('hooks_dir', 'prefix-command')))
+ # Testing not existing hook script
+ isfile_mock.return_value = False
+ res3 = orchestrator.resolve_hook_script_path("hooks_dir", "prefix", "command",
+ "script_type")
+ self.assertEqual(res3, None)
+
+
+ @patch.object(CustomServiceOrchestrator, "runCommand")
+ @patch.object(FileCache, "__init__")
+ def test_requestComponentStatus(self, FileCache_mock, runCommand_mock):
+ FileCache_mock.return_value = None
+ status_command = {
+ "serviceName" : 'HDFS',
+ "commandType" : "STATUS_COMMAND",
+ "clusterName" : "",
+ "componentName" : "DATANODE",
+ 'configurations':{}
+ }
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+ # Test alive case
+ runCommand_mock.return_value = {
+ "exitcode" : 0
+ }
+
+ status = orchestrator.requestComponentStatus(status_command)
+ self.assertEqual(runCommand_mock.return_value, status)
+
+ # Test dead case
+ runCommand_mock.return_value = {
+ "exitcode" : 1
+ }
+ status = orchestrator.requestComponentStatus(status_command)
+ self.assertEqual(runCommand_mock.return_value, status)
+
+ @patch.object(CustomServiceOrchestrator, "runCommand")
+ @patch.object(FileCache, "__init__")
+ def test_requestComponentSecurityState(self, FileCache_mock, runCommand_mock):
+ FileCache_mock.return_value = None
+ status_command = {
+ "serviceName" : 'HDFS',
+ "commandType" : "STATUS_COMMAND",
+ "clusterName" : "",
+ "componentName" : "DATANODE",
+ 'configurations':{}
+ }
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+ # Test securityState
+ runCommand_mock.return_value = {
+ 'exitcode' : 0,
+ 'structuredOut' : {'securityState': 'UNSECURED'}
+ }
+
+ status = orchestrator.requestComponentSecurityState(status_command)
+ self.assertEqual('UNSECURED', status)
+
+ # Test case where exit code indicates failure
+ runCommand_mock.return_value = {
+ "exitcode" : 1
+ }
+ status = orchestrator.requestComponentSecurityState(status_command)
+ self.assertEqual('UNKNOWN', status)
+
+ @patch.object(FileCache, "__init__")
+ def test_requestComponentSecurityState_realFailure(self, FileCache_mock):
+ '''
+ Tests the case where the CustomServiceOrchestrator attempts to call a service's security_status
+ method, but fails to do so because the script or method was not found.
+ :param FileCache_mock:
+ :return:
+ '''
+ FileCache_mock.return_value = None
+ status_command = {
+ "serviceName" : 'BOGUS_SERVICE',
+ "commandType" : "STATUS_COMMAND",
+ "clusterName" : "",
+ "componentName" : "DATANODE",
+ 'configurations':{}
+ }
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+
+ status = orchestrator.requestComponentSecurityState(status_command)
+ self.assertEqual('UNKNOWN', status)
+
+
+ @patch.object(CustomServiceOrchestrator, "get_py_executor")
+ @patch.object(CustomServiceOrchestrator, "dump_command_to_json")
+ @patch.object(FileCache, "__init__")
+ @patch.object(FileCache, "get_custom_actions_base_dir")
+ def test_runCommand_background_action(self, get_custom_actions_base_dir_mock,
+ FileCache_mock,
+ dump_command_to_json_mock,
+ get_py_executor_mock):
+ FileCache_mock.return_value = None
+ get_custom_actions_base_dir_mock.return_value = "some path"
+ _, script = tempfile.mkstemp()
+ command = {
+ 'role' : 'any',
+ 'commandParams': {
+ 'script_type': 'PYTHON',
+ 'script': 'some_custom_action.py',
+ 'command_timeout': '600',
+ 'jdk_location' : 'some_location'
+ },
+ 'taskId' : '13',
+ 'roleCommand': 'ACTIONEXECUTE',
+ 'commandType': 'BACKGROUND_EXECUTION_COMMAND',
+ '__handle': BackgroundCommandExecutionHandle({'taskId': '13'}, 13,
+ MagicMock(), MagicMock())
+ }
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+
+ import TestActionQueue
+ pyex = PythonExecutor(orchestrator.tmp_dir, orchestrator.config)
+ TestActionQueue.patch_output_file(pyex)
+ pyex.condenseOutput = MagicMock()
+ get_py_executor_mock.return_value = pyex
+ orchestrator.dump_command_to_json = MagicMock()
+
+ ret = orchestrator.runCommand(command, "out.txt", "err.txt")
+ self.assertEqual(ret['exitcode'], 777)
+
+ def tearDown(self):
+ # enable stdout
+ sys.stdout = sys.__stdout__
+
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/test/python/ambari_agent/TestSecurity.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestSecurity.py b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py
index 0e1e4ee..c1b7812 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestSecurity.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py
@@ -343,12 +343,14 @@ class TestSecurity(unittest.TestCase):
@patch("subprocess.Popen")
@patch("subprocess.Popen.communicate")
- def test_genAgentCrtReq(self, communicate_mock, popen_mock):
+ @patch.object(os, "chmod")
+ def test_genAgentCrtReq(self, chmod_mock, communicate_mock, popen_mock):
man = CertificateManager(self.config)
p = MagicMock(spec=subprocess.Popen)
p.communicate = communicate_mock
popen_mock.return_value = p
- man.genAgentCrtReq()
+ man.genAgentCrtReq('/dummy-keysdir/hostname.key')
+ self.assertTrue(chmod_mock.called)
self.assertTrue(popen_mock.called)
self.assertTrue(communicate_mock.called)
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/test/python/ambari_agent/TestSecurity.py.orig
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestSecurity.py.orig b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py.orig
new file mode 100644
index 0000000..0e1e4ee
--- /dev/null
+++ b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py.orig
@@ -0,0 +1,389 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import StringIO
+import sys, subprocess
+from mock.mock import MagicMock, patch, ANY
+import mock.mock
+import unittest
+import logging
+import signal
+import ConfigParser
+import ssl
+import os
+import tempfile
+
+from ambari_commons import OSCheck
+from only_for_platform import os_distro_value
+
+with patch("platform.linux_distribution", return_value = ('Suse','11','Final')):
+ from ambari_agent import NetUtil
+ from ambari_agent.security import CertificateManager
+ from ambari_agent import ProcessHelper, main
+ from ambari_agent.AmbariConfig import AmbariConfig
+ from ambari_agent.Controller import Controller
+ from ambari_agent import security
+
+aa = mock.mock.mock_open()
+class TestSecurity(unittest.TestCase):
+
+ @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+ def setUp(self):
+ # disable stdout
+ out = StringIO.StringIO()
+ sys.stdout = out
+ # Create config
+ self.config = AmbariConfig()
+ # Instantiate CachedHTTPSConnection (skip connect() call)
+ with patch.object(security.VerifiedHTTPSConnection, "connect"):
+ self.cachedHTTPSConnection = security.CachedHTTPSConnection(self.config)
+
+
+ def tearDown(self):
+ # enable stdout
+ sys.stdout = sys.__stdout__
+
+
+ ### VerifiedHTTPSConnection ###
+
+ @patch.object(security.CertificateManager, "initSecurity")
+ @patch("socket.create_connection")
+ @patch("ssl.wrap_socket")
+ def test_VerifiedHTTPSConnection_connect(self, wrap_socket_mock,
+ create_connection_mock,
+ init_security_mock):
+ init_security_mock.return_value = None
+ self.config.set('security', 'keysdir', '/dummy-keysdir')
+ connection = security.VerifiedHTTPSConnection("example.com",
+ self.config.get('server', 'secured_url_port'), self.config)
+ connection._tunnel_host = False
+ connection.sock = None
+ connection.connect()
+ self.assertTrue(wrap_socket_mock.called)
+
+ ### VerifiedHTTPSConnection with no certificates creation
+ @patch.object(security.CertificateManager, "initSecurity")
+ @patch("socket.create_connection")
+ @patch("ssl.wrap_socket")
+ def test_Verified_HTTPSConnection_non_secure_connect(self, wrap_socket_mock,
+ create_connection_mock,
+ init_security_mock):
+ connection = security.VerifiedHTTPSConnection("example.com",
+ self.config.get('server', 'secured_url_port'), self.config)
+ connection._tunnel_host = False
+ connection.sock = None
+ connection.connect()
+ self.assertFalse(init_security_mock.called)
+
+ ### VerifiedHTTPSConnection with two-way SSL authentication enabled
+ @patch.object(security.CertificateManager, "initSecurity")
+ @patch("socket.create_connection")
+ @patch("ssl.wrap_socket")
+ def test_Verified_HTTPSConnection_two_way_ssl_connect(self, wrap_socket_mock,
+ create_connection_mock,
+ init_security_mock):
+ wrap_socket_mock.side_effect=ssl.SSLError()
+ connection = security.VerifiedHTTPSConnection("example.com",
+ self.config.get('server', 'secured_url_port'), self.config)
+ connection._tunnel_host = False
+ connection.sock = None
+ try:
+ connection.connect()
+ except ssl.SSLError:
+ pass
+ self.assertTrue(init_security_mock.called)
+
+ ### CachedHTTPSConnection ###
+
+ @patch.object(security.VerifiedHTTPSConnection, "connect")
+ def test_CachedHTTPSConnection_connect(self, vhc_connect_mock):
+ self.config.set('server', 'hostname', 'dummy.server.hostname')
+ self.config.set('server', 'secured_url_port', '443')
+ # Testing not connected case
+ self.cachedHTTPSConnection.connected = False
+ self.cachedHTTPSConnection.connect()
+ self.assertTrue(vhc_connect_mock.called)
+ vhc_connect_mock.reset_mock()
+ # Testing already connected case
+ self.cachedHTTPSConnection.connect()
+ self.assertFalse(vhc_connect_mock.called)
+
+
+ @patch.object(security.CachedHTTPSConnection, "connect")
+ def test_forceClear(self, connect_mock):
+ # Testing if httpsconn instance changed
+ old = self.cachedHTTPSConnection.httpsconn
+ self.cachedHTTPSConnection.forceClear()
+ self.assertNotEqual(old, self.cachedHTTPSConnection.httpsconn)
+
+
+ @patch.object(security.CachedHTTPSConnection, "connect")
+ def test_request(self, connect_mock):
+ httpsconn_mock = MagicMock(create = True)
+ self.cachedHTTPSConnection.httpsconn = httpsconn_mock
+
+ dummy_request = MagicMock(create = True)
+ dummy_request.get_method.return_value = "dummy_get_method"
+ dummy_request.get_full_url.return_value = "dummy_full_url"
+ dummy_request.get_data.return_value = "dummy_get_data"
+ dummy_request.headers = "dummy_headers"
+
+ responce_mock = MagicMock(create = True)
+ responce_mock.read.return_value = "dummy responce"
+ httpsconn_mock.getresponse.return_value = responce_mock
+
+ # Testing normal case
+ responce = self.cachedHTTPSConnection.request(dummy_request)
+
+ self.assertEqual(responce, responce_mock.read.return_value)
+ httpsconn_mock.request.assert_called_once_with(
+ dummy_request.get_method.return_value,
+ dummy_request.get_full_url.return_value,
+ dummy_request.get_data.return_value,
+ dummy_request.headers)
+
+ # Testing case of exception
+ try:
+ def side_eff():
+ raise Exception("Dummy exception")
+ httpsconn_mock.read.side_effect = side_eff
+ responce = self.cachedHTTPSConnection.request(dummy_request)
+ self.fail("Should raise IOError")
+ except Exception, err:
+ # Expected
+ pass
+
+
+ ### CertificateManager ###
+
+
+ @patch("ambari_agent.hostname.hostname")
+ def test_getAgentKeyName(self, hostname_mock):
+ hostname_mock.return_value = "dummy.hostname"
+ self.config.set('security', 'keysdir', '/dummy-keysdir')
+ man = CertificateManager(self.config)
+ res = man.getAgentKeyName()
+ self.assertEquals(res, os.path.abspath("/dummy-keysdir/dummy.hostname.key"))
+
+
+ @patch("ambari_agent.hostname.hostname")
+ def test_getAgentCrtName(self, hostname_mock):
+ hostname_mock.return_value = "dummy.hostname"
+ self.config.set('security', 'keysdir', '/dummy-keysdir')
+ man = CertificateManager(self.config)
+ res = man.getAgentCrtName()
+ self.assertEquals(res, os.path.abspath("/dummy-keysdir/dummy.hostname.crt"))
+
+
+ @patch("ambari_agent.hostname.hostname")
+ def test_getAgentCrtReqName(self, hostname_mock):
+ hostname_mock.return_value = "dummy.hostname"
+ self.config.set('security', 'keysdir', '/dummy-keysdir')
+ man = CertificateManager(self.config)
+ res = man.getAgentCrtReqName()
+ self.assertEquals(res, os.path.abspath("/dummy-keysdir/dummy.hostname.csr"))
+
+
+ def test_getSrvrCrtName(self):
+ self.config.set('security', 'keysdir', '/dummy-keysdir')
+ man = CertificateManager(self.config)
+ res = man.getSrvrCrtName()
+ self.assertEquals(res, os.path.abspath("/dummy-keysdir/ca.crt"))
+
+
+ @patch("os.path.exists")
+ @patch.object(security.CertificateManager, "loadSrvrCrt")
+ @patch.object(security.CertificateManager, "getAgentKeyName")
+ @patch.object(security.CertificateManager, "genAgentCrtReq")
+ @patch.object(security.CertificateManager, "getAgentCrtName")
+ @patch.object(security.CertificateManager, "reqSignCrt")
+ def test_checkCertExists(self, reqSignCrt_mock, getAgentCrtName_mock,
+ genAgentCrtReq_mock, getAgentKeyName_mock,
+ loadSrvrCrt_mock, exists_mock):
+ self.config.set('security', 'keysdir', '/dummy-keysdir')
+ getAgentKeyName_mock.return_value = "dummy AgentKeyName"
+ getAgentCrtName_mock.return_value = "dummy AgentCrtName"
+ man = CertificateManager(self.config)
+
+ # Case when all files exist
+ exists_mock.side_effect = [True, True, True]
+ man.checkCertExists()
+ self.assertFalse(loadSrvrCrt_mock.called)
+ self.assertFalse(genAgentCrtReq_mock.called)
+ self.assertFalse(reqSignCrt_mock.called)
+
+ # Absent server cert
+ exists_mock.side_effect = [False, True, True]
+ man.checkCertExists()
+ self.assertTrue(loadSrvrCrt_mock.called)
+ self.assertFalse(genAgentCrtReq_mock.called)
+ self.assertFalse(reqSignCrt_mock.called)
+ loadSrvrCrt_mock.reset_mock()
+
+ # Absent agent key
+ exists_mock.side_effect = [True, False, True]
+ man.checkCertExists()
+ self.assertFalse(loadSrvrCrt_mock.called)
+ self.assertTrue(genAgentCrtReq_mock.called)
+ self.assertFalse(reqSignCrt_mock.called)
+ genAgentCrtReq_mock.reset_mock()
+
+ # Absent agent cert
+ exists_mock.side_effect = [True, True, False]
+ man.checkCertExists()
+ self.assertFalse(loadSrvrCrt_mock.called)
+ self.assertFalse(genAgentCrtReq_mock.called)
+ self.assertTrue(reqSignCrt_mock.called)
+ reqSignCrt_mock.reset_mock()
+
+
+
+ @patch("urllib2.OpenerDirector.open")
+ @patch.object(security.CertificateManager, "getSrvrCrtName")
+ def test_loadSrvrCrt(self, getSrvrCrtName_mock, urlopen_mock):
+ read_mock = MagicMock(create=True)
+ read_mock.read.return_value = "dummy_cert"
+ urlopen_mock.return_value = read_mock
+ _, tmpoutfile = tempfile.mkstemp()
+ getSrvrCrtName_mock.return_value = tmpoutfile
+
+ man = CertificateManager(self.config)
+ man.loadSrvrCrt()
+
+ # Checking file contents
+ saved = open(tmpoutfile, 'r').read()
+ self.assertEqual(saved, read_mock.read.return_value)
+ try:
+ os.unlink(tmpoutfile)
+ except:
+ pass
+
+
+ @patch("ambari_agent.hostname.hostname")
+ @patch('__builtin__.open', create=True, autospec=True)
+ @patch.dict('os.environ', {'DUMMY_PASSPHRASE': 'dummy-passphrase'})
+ @patch('ambari_simplejson.dumps')
+ @patch('urllib2.Request')
+ @patch("urllib2.OpenerDirector.open")
+ @patch('ambari_simplejson.loads')
+ def test_reqSignCrt(self, loads_mock, urlopen_mock, request_mock, dumps_mock, open_mock, hostname_mock):
+ self.config.set('security', 'keysdir', '/dummy-keysdir')
+ self.config.set('security', 'passphrase_env_var_name', 'DUMMY_PASSPHRASE')
+ man = CertificateManager(self.config)
+ hostname_mock.return_value = "dummy-hostname"
+
+ open_mock.return_value.read.return_value = "dummy_request"
+ urlopen_mock.return_value.read.return_value = "dummy_server_request"
+ loads_mock.return_value = {
+ 'result': 'OK',
+ 'signedCa': 'dummy-crt'
+ }
+
+ # Test normal server interaction
+ man.reqSignCrt()
+
+ self.assertEqual(dumps_mock.call_args[0][0], {
+ 'csr' : 'dummy_request',
+ 'passphrase' : 'dummy-passphrase'
+ })
+ self.assertEqual(open_mock.return_value.write.call_args[0][0], 'dummy-crt')
+
+ # Test negative server reply
+ dumps_mock.reset_mock()
+ open_mock.return_value.write.reset_mock()
+ loads_mock.return_value = {
+ 'result': 'FAIL',
+ 'signedCa': 'fail-crt'
+ }
+
+ # If certificate signing failed, then exception must be raised
+ try:
+ man.reqSignCrt()
+ self.fail()
+ except ssl.SSLError:
+ pass
+ self.assertFalse(open_mock.return_value.write.called)
+
+ # Test connection fail
+ dumps_mock.reset_mock()
+ open_mock.return_value.write.reset_mock()
+
+ try:
+ man.reqSignCrt()
+ self.fail("Expected exception here")
+ except Exception, err:
+ # expected
+ pass
+
+ # Test malformed JSON response
+ open_mock.return_value.write.reset_mock()
+ loads_mock.side_effect = Exception()
+ try:
+ man.reqSignCrt()
+ self.fail("Expected exception here")
+ except ssl.SSLError:
+ pass
+ self.assertFalse(open_mock.return_value.write.called)
+
+ @patch("subprocess.Popen")
+ @patch("subprocess.Popen.communicate")
+ def test_genAgentCrtReq(self, communicate_mock, popen_mock):
+ man = CertificateManager(self.config)
+ p = MagicMock(spec=subprocess.Popen)
+ p.communicate = communicate_mock
+ popen_mock.return_value = p
+ man.genAgentCrtReq()
+ self.assertTrue(popen_mock.called)
+ self.assertTrue(communicate_mock.called)
+
+ @patch("ambari_agent.hostname.hostname")
+ @patch('__builtin__.open', create=True, autospec=True)
+ @patch("urllib2.OpenerDirector.open")
+ @patch.dict('os.environ', {'DUMMY_PASSPHRASE': 'dummy-passphrase'})
+ def test_reqSignCrt_malformedJson(self, urlopen_mock, open_mock, hostname_mock):
+ hostname_mock.return_value = "dummy-hostname"
+ open_mock.return_value.read.return_value = "dummy_request"
+ self.config.set('security', 'keysdir', '/dummy-keysdir')
+ self.config.set('security', 'passphrase_env_var_name', 'DUMMY_PASSPHRASE')
+ man = CertificateManager(self.config)
+
+ # test valid JSON response
+ urlopen_mock.return_value.read.return_value = '{"result": "OK", "signedCa":"dummy"}'
+ try:
+ man.reqSignCrt()
+ except ssl.SSLError:
+ self.fail("Unexpected exception!")
+ open_mock.return_value.write.assert_called_with(u'dummy')
+
+ # test malformed JSON response
+ open_mock.return_value.write.reset_mock()
+ urlopen_mock.return_value.read.return_value = '{malformed_object}'
+ try:
+ man.reqSignCrt()
+ self.fail("Expected exception!")
+ except ssl.SSLError:
+ pass
+ self.assertFalse(open_mock.return_value.write.called)
+
+ @patch.object(security.CertificateManager, "checkCertExists")
+ def test_initSecurity(self, checkCertExists_method):
+ man = CertificateManager(self.config)
+ man.initSecurity()
+ self.assertTrue(checkCertExists_method.called)
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index 1ee5b70..e0a7877 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -61,7 +61,7 @@ USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT> <LOGGING_LEV
<BASEDIR> path to service metadata dir. Ex: /var/lib/ambari-agent/cache/common-services/HDFS/2.1.0.2.0/package
<STROUTPUT> path to file with structured command output (file will be created). Ex:/tmp/my.txt
<LOGGING_LEVEL> log level for stdout. Ex:DEBUG,INFO
-<TMP_DIR> temporary directory for executable scripts. Ex: /var/lib/ambari-agent/data/tmp
+<TMP_DIR> temporary directory for executable scripts. Ex: /var/lib/ambari-agent/tmp
"""
_PASSWORD_MAP = {"/configurations/cluster-env/hadoop.user.name":"/configurations/cluster-env/hadoop.user.password"}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-server/src/main/python/bootstrap.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/bootstrap.py b/ambari-server/src/main/python/bootstrap.py
index faf873b..3eba75c 100755
--- a/ambari-server/src/main/python/bootstrap.py
+++ b/ambari-server/src/main/python/bootstrap.py
@@ -47,7 +47,7 @@ MAX_PARALLEL_BOOTSTRAPS = 20
# How many seconds to wait between polling parallel bootstraps
POLL_INTERVAL_SEC = 1
DEBUG = False
-DEFAULT_AGENT_TEMP_FOLDER = "/var/lib/ambari-agent/data/tmp"
+DEFAULT_AGENT_TEMP_FOLDER = "/var/lib/ambari-agent/tmp"
DEFAULT_AGENT_DATA_FOLDER = "/var/lib/ambari-agent/data"
DEFAULT_AGENT_LIB_FOLDER = "/var/lib/ambari-agent"
PYTHON_ENV="env PYTHONPATH=$PYTHONPATH:" + DEFAULT_AGENT_TEMP_FOLDER
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
index 1415367..a3c02a6 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
@@ -202,7 +202,7 @@ class NameNodeDefault(NameNode):
# Create the kerberos credentials cache (ccache) file and set it in the environment to use
# when executing HDFS rebalance command. Use the md5 hash of the combination of the principal and keytab file
# to generate a (relatively) unique cache filename so that we can use it as needed.
- # TODO: params.tmp_dir=/var/lib/ambari-agent/data/tmp. However hdfs user doesn't have access to this path.
+ # TODO: params.tmp_dir=/var/lib/ambari-agent/tmp. However hdfs user doesn't have access to this path.
# TODO: Hence using /tmp
ccache_file_name = "hdfs_rebalance_cc_" + _md5(format("{hdfs_principal_name}|{hdfs_user_keytab}")).hexdigest()
ccache_file_path = os.path.join(tempfile.gettempdir(), ccache_file_name)
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-server/src/test/python/TestBootstrap.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestBootstrap.py b/ambari-server/src/test/python/TestBootstrap.py
index 792d99d..e143a68 100644
--- a/ambari-server/src/test/python/TestBootstrap.py
+++ b/ambari-server/src/test/python/TestBootstrap.py
@@ -112,8 +112,8 @@ class TestBootstrap(TestCase):
utime = 1234
bootstrap_obj.getUtime = MagicMock(return_value=utime)
ret = bootstrap_obj.getRunSetupWithPasswordCommand("hostname")
- expected = "sudo -S python /var/lib/ambari-agent/data/tmp/setupAgent{0}.py hostname TEST_PASSPHRASE " \
- "ambariServer root 8440 < /var/lib/ambari-agent/data/tmp/host_pass{0}".format(utime)
+ expected = "sudo -S python /var/lib/ambari-agent/tmp/setupAgent{0}.py hostname TEST_PASSPHRASE " \
+ "ambariServer root 8440 < /var/lib/ambari-agent/tmp/host_pass{0}".format(utime)
self.assertEquals(ret, expected)
@@ -353,11 +353,11 @@ class TestBootstrap(TestCase):
self.assertEquals(res, expected)
command = str(init_mock.call_args[0][3])
self.assertEqual(command,
- "sudo mkdir -p /var/lib/ambari-agent/data/tmp ; "
- "sudo chown -R root /var/lib/ambari-agent/data/tmp ; "
+ "sudo mkdir -p /var/lib/ambari-agent/tmp ; "
+ "sudo chown -R root /var/lib/ambari-agent/tmp ; "
"sudo chmod 755 /var/lib/ambari-agent ; "
"sudo chmod 755 /var/lib/ambari-agent/data ; "
- "sudo chmod 777 /var/lib/ambari-agent/data/tmp")
+ "sudo chmod 777 /var/lib/ambari-agent/tmp")
@patch.object(BootstrapDefault, "getOsCheckScript")
@patch.object(BootstrapDefault, "getOsCheckScriptRemoteLocation")
@@ -520,7 +520,7 @@ class TestBootstrap(TestCase):
command = str(init_mock.call_args[0][3])
self.assertEqual(command,
"chmod a+x OsCheckScriptRemoteLocation && "
- "env PYTHONPATH=$PYTHONPATH:/var/lib/ambari-agent/data/tmp OsCheckScriptRemoteLocation centos6")
+ "env PYTHONPATH=$PYTHONPATH:/var/lib/ambari-agent/tmp OsCheckScriptRemoteLocation centos6")
@patch.object(SSH, "__init__")