You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2015/09/14 14:01:20 UTC
[2/2] ambari git commit: Verify if restricting acls on
/var/lib/ambari-agent/data will be OK (aonishuk)
Verify if restricting acls on /var/lib/ambari-agent/data will be OK (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b8388e12
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b8388e12
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b8388e12
Branch: refs/heads/trunk
Commit: b8388e12b348e3989490e812dd2fe993b2ca603e
Parents: e837d4f
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Mon Sep 14 15:01:01 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Mon Sep 14 15:01:01 2015 +0300
----------------------------------------------------------------------
ambari-agent/conf/unix/ambari-agent.ini | 1 -
ambari-agent/conf/unix/install-helper.sh | 5 +-
ambari-agent/pom.xml | 4 +-
.../src/main/python/ambari_agent/Constants.py | 21 +
.../ambari_agent/CustomServiceOrchestrator.py | 3 +-
.../python/ambari_agent/alerts/metric_alert.py | 3 +-
.../python/ambari_agent/alerts/script_alert.py | 3 +-
.../python/ambari_agent/alerts/web_alert.py | 3 +-
.../src/main/python/ambari_agent/security.py | 11 +-
.../python/ambari_agent/TestCertGeneration.py | 6 +-
.../ambari_agent/TestCertGeneration.py.orig | 50 ++
.../TestCustomServiceOrchestrator.py | 1 -
.../TestCustomServiceOrchestrator.py.orig | 643 ++++++++++++++
.../test/python/ambari_agent/TestSecurity.py | 6 +-
.../python/ambari_agent/TestSecurity.py.orig | 389 ++++++++
.../libraries/script/script.py | 2 +-
ambari-server/src/main/python/bootstrap.py | 2 +-
.../HDFS/2.1.0.2.0/package/scripts/namenode.py | 2 +-
ambari-server/src/test/python/TestBootstrap.py | 12 +-
.../src/test/python/TestBootstrap.py.orig | 879 +++++++++++++++++++
20 files changed, 2020 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/conf/unix/ambari-agent.ini
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/unix/ambari-agent.ini b/ambari-agent/conf/unix/ambari-agent.ini
index abfde62..3b7631c 100644
--- a/ambari-agent/conf/unix/ambari-agent.ini
+++ b/ambari-agent/conf/unix/ambari-agent.ini
@@ -19,7 +19,6 @@ secured_url_port=8441
[agent]
prefix=/var/lib/ambari-agent/data
-tmp_dir=/var/lib/ambari-agent/data/tmp
;loglevel=(DEBUG/INFO)
loglevel=INFO
data_cleanup_interval=86400
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/conf/unix/install-helper.sh
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/unix/install-helper.sh b/ambari-agent/conf/unix/install-helper.sh
index 48391d5..35c67fb 100644
--- a/ambari-agent/conf/unix/install-helper.sh
+++ b/ambari-agent/conf/unix/install-helper.sh
@@ -58,8 +58,9 @@ do_install(){
# on nano Ubuntu, when umask=027 those folders are created without 'x' bit for 'others'.
# which causes failures when hadoop users try to access tmp_dir
chmod a+x /var/lib/ambari-agent
- chmod a+x /var/lib/ambari-agent/data
- chmod 777 /var/lib/ambari-agent/data/tmp
+
+ chmod 777 /var/lib/ambari-agent/tmp
+ chmod 700 /var/lib/ambari-agent/data
}
do_remove(){
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-agent/pom.xml b/ambari-agent/pom.xml
index 24927e2..0096162 100644
--- a/ambari-agent/pom.xml
+++ b/ambari-agent/pom.xml
@@ -391,7 +391,7 @@
<groupname>root</groupname>
</mapping>
<mapping>
- <directory>/var/lib/${project.artifactId}/data/tmp</directory>
+ <directory>/var/lib/${project.artifactId}/tmp</directory>
<filemode>777</filemode>
<username>root</username>
<groupname>root</groupname>
@@ -596,7 +596,7 @@
<paths>
<path>${package.pid.dir}</path>
<path>/var/lib/${project.artifactId}/data</path>
- <path>/var/lib/${project.artifactId}/data/tmp</path>
+ <path>/var/lib/${project.artifactId}/tmp</path>
<path>/var/lib/${project.artifactId}/keys</path>
<path>${package.log.dir}</path>
<path>/var/lib/${project.artifactId}/lib</path>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/main/python/ambari_agent/Constants.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/Constants.py b/ambari-agent/src/main/python/ambari_agent/Constants.py
new file mode 100644
index 0000000..6b80f00
--- /dev/null
+++ b/ambari-agent/src/main/python/ambari_agent/Constants.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+AGENT_TMP_DIR = "/var/lib/ambari-agent/tmp"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
index 6ee929c..6c1a161 100644
--- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
+++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
@@ -29,6 +29,7 @@ from FileCache import FileCache
from AgentException import AgentException
from PythonExecutor import PythonExecutor
from PythonReflectiveExecutor import PythonReflectiveExecutor
+import Constants
import hostname
@@ -61,7 +62,7 @@ class CustomServiceOrchestrator():
def __init__(self, config, controller):
self.config = config
self.tmp_dir = config.get('agent', 'prefix')
- self.exec_tmp_dir = config.get('agent', 'tmp_dir')
+ self.exec_tmp_dir = Constants.AGENT_TMP_DIR
self.file_cache = FileCache(config)
self.status_commands_stdout = os.path.join(self.tmp_dir,
'status_command_stdout.txt')
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
index aa4ad75..8de49cd 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
@@ -30,6 +30,7 @@ from alerts.base_alert import BaseAlert
from ambari_commons.urllib_handlers import RefreshHeaderProcessor
from resource_management.libraries.functions.get_port_from_url import get_port_from_url
from resource_management.libraries.functions.curl_krb_request import curl_krb_request
+from ambari_agent import Constants
logger = logging.getLogger()
@@ -199,7 +200,7 @@ class MetricAlert(BaseAlert):
content = ''
try:
if kerberos_principal is not None and kerberos_keytab is not None and security_enabled:
- tmp_dir = self.config.get('agent', 'tmp_dir')
+ tmp_dir = Constants.AGENT_TMP_DIR
if tmp_dir is None:
tmp_dir = gettempdir()
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py
index 76afbc9..e70dc22 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py
@@ -25,6 +25,7 @@ import re
from alerts.base_alert import BaseAlert
from resource_management.core.environment import Environment
from resource_management.core.logger import Logger
+from ambari_agent import Constants
logger = logging.getLogger(__name__)
@@ -99,7 +100,7 @@ class ScriptAlert(BaseAlert):
matchObj = re.match( r'((.*)services(.*)package)', self.path_to_script)
if matchObj:
basedir = matchObj.group(1)
- with Environment(basedir, tmp_dir=self.config.get('agent', 'tmp_dir')) as env:
+ with Environment(basedir, tmp_dir=Constants.AGENT_TMP_DIR) as env:
return cmd_module.execute(configurations, self.parameters, self.host_name)
else:
return cmd_module.execute(configurations, self.parameters, self.host_name)
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
index b76d5e0..0d627c3 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
@@ -31,6 +31,7 @@ from resource_management.libraries.functions.get_port_from_url import get_port_f
from resource_management.libraries.functions.curl_krb_request import curl_krb_request
from ambari_commons import OSCheck
from ambari_commons.inet_utils import resolve_address
+from ambari_agent import Constants
# hashlib is supplied as of Python 2.5 as the replacement interface for md5
# and other secure hashes. In 2.6, md5 is deprecated. Import hashlib if
@@ -171,7 +172,7 @@ class WebAlert(BaseAlert):
# Create the kerberos credentials cache (ccache) file and set it in the environment to use
# when executing curl. Use the md5 hash of the combination of the principal and keytab file
# to generate a (relatively) unique cache filename so that we can use it as needed.
- tmp_dir = self.config.get('agent', 'tmp_dir')
+ tmp_dir = Constants.AGENT_TMP_DIR
if tmp_dir is None:
tmp_dir = gettempdir()
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/main/python/ambari_agent/security.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/security.py b/ambari-agent/src/main/python/ambari_agent/security.py
index bfaf134..dc1f141 100644
--- a/ambari-agent/src/main/python/ambari_agent/security.py
+++ b/ambari-agent/src/main/python/ambari_agent/security.py
@@ -35,6 +35,7 @@ logger = logging.getLogger(__name__)
GEN_AGENT_KEY = 'openssl req -new -newkey rsa:1024 -nodes -keyout "%(keysdir)s' \
+ os.sep + '%(hostname)s.key" -subj /OU=%(hostname)s/ ' \
'-out "%(keysdir)s' + os.sep + '%(hostname)s.csr"'
+KEY_FILENAME = '%(hostname)s.key'
class VerifiedHTTPSConnection(httplib.HTTPSConnection):
@@ -190,7 +191,7 @@ class CertificateManager():
if not agent_key_exists:
logger.info("Agent key not exists, generating request")
- self.genAgentCrtReq()
+ self.genAgentCrtReq(self.getAgentKeyName())
else:
logger.info("Agent key exists, ok")
@@ -252,10 +253,12 @@ class CertificateManager():
'\nExiting..')
raise ssl.SSLError
- def genAgentCrtReq(self):
+ def genAgentCrtReq(self, keyname):
+ keysdir = os.path.abspath(self.config.get('security', 'keysdir'))
generate_script = GEN_AGENT_KEY % {
'hostname': hostname.hostname(self.config),
- 'keysdir': os.path.abspath(self.config.get('security', 'keysdir'))}
+ 'keysdir': keysdir}
+
logger.info(generate_script)
if platform.system() == 'Windows':
p = subprocess.Popen(generate_script, stdout=subprocess.PIPE)
@@ -264,6 +267,8 @@ class CertificateManager():
p = subprocess.Popen([generate_script], shell=True,
stdout=subprocess.PIPE)
p.communicate()
+ # this is required to be 600 for security concerns.
+ os.chmod(keyname, 0600)
def initSecurity(self):
self.checkCertExists()
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py b/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py
index 2461281..1cf7866 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py
@@ -41,8 +41,10 @@ class TestCertGeneration(TestCase):
config.set('security', 'server_crt', 'ca.crt')
self.certMan = CertificateManager(config)
- def test_generation(self):
- self.certMan.genAgentCrtReq()
+ @patch.object(os, "chmod")
+ def test_generation(self, chmod_mock):
+ self.certMan.genAgentCrtReq('/dummy_dir/hostname.key')
+ self.assertTrue(chmod_mock.called)
self.assertTrue(os.path.exists(self.certMan.getAgentKeyName()))
self.assertTrue(os.path.exists(self.certMan.getAgentCrtReqName()))
def tearDown(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py.orig
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py.orig b/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py.orig
new file mode 100644
index 0000000..2461281
--- /dev/null
+++ b/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py.orig
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import os
+import tempfile
+import shutil
+from unittest import TestCase
+
+from ambari_agent.security import CertificateManager
+from ambari_agent import AmbariConfig
+from mock.mock import patch, MagicMock
+from ambari_commons import OSCheck
+from only_for_platform import os_distro_value
+
+class TestCertGeneration(TestCase):
+ @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+ def setUp(self):
+ self.tmpdir = tempfile.mkdtemp()
+ config = AmbariConfig.AmbariConfig()
+ #config.add_section('server')
+ config.set('server', 'hostname', 'example.com')
+ config.set('server', 'url_port', '777')
+ #config.add_section('security')
+ config.set('security', 'keysdir', self.tmpdir)
+ config.set('security', 'server_crt', 'ca.crt')
+ self.certMan = CertificateManager(config)
+
+ def test_generation(self):
+ self.certMan.genAgentCrtReq()
+ self.assertTrue(os.path.exists(self.certMan.getAgentKeyName()))
+ self.assertTrue(os.path.exists(self.certMan.getAgentCrtReqName()))
+ def tearDown(self):
+ shutil.rmtree(self.tmpdir)
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
index 8957b44..a4025eb 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
@@ -56,7 +56,6 @@ class TestCustomServiceOrchestrator(TestCase):
self.config = ConfigParser.RawConfigParser()
self.config.add_section('agent')
self.config.set('agent', 'prefix', tmpdir)
- self.config.set('agent', 'tmp_dir', exec_tmp_dir)
self.config.set('agent', 'cache_dir', "/cachedir")
self.config.add_section('python')
self.config.set('python', 'custom_actions_dir', tmpdir)
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py.orig
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py.orig b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py.orig
new file mode 100644
index 0000000..8957b44
--- /dev/null
+++ b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py.orig
@@ -0,0 +1,643 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import ConfigParser
+from multiprocessing.pool import ThreadPool
+import os
+
+import pprint
+from ambari_commons import shell
+
+from unittest import TestCase
+import threading
+import tempfile
+import time
+from threading import Thread
+
+from mock.mock import MagicMock, patch
+import StringIO
+import sys
+
+from ambari_agent.ActionQueue import ActionQueue
+from ambari_agent.AgentException import AgentException
+from ambari_agent.AmbariConfig import AmbariConfig
+from ambari_agent.BackgroundCommandExecutionHandle import BackgroundCommandExecutionHandle
+from ambari_agent.CustomServiceOrchestrator import CustomServiceOrchestrator
+from ambari_agent.FileCache import FileCache
+from ambari_agent.PythonExecutor import PythonExecutor
+from ambari_commons import OSCheck
+from only_for_platform import get_platform, os_distro_value, PLATFORM_WINDOWS
+
+class TestCustomServiceOrchestrator(TestCase):
+
+ def setUp(self):
+ # disable stdout
+ out = StringIO.StringIO()
+ sys.stdout = out
+ # generate sample config
+ tmpdir = tempfile.gettempdir()
+ exec_tmp_dir = os.path.join(tmpdir, 'tmp')
+ self.config = ConfigParser.RawConfigParser()
+ self.config.add_section('agent')
+ self.config.set('agent', 'prefix', tmpdir)
+ self.config.set('agent', 'tmp_dir', exec_tmp_dir)
+ self.config.set('agent', 'cache_dir', "/cachedir")
+ self.config.add_section('python')
+ self.config.set('python', 'custom_actions_dir', tmpdir)
+
+
+ @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+ @patch.object(FileCache, "__init__")
+ def test_add_reg_listener_to_controller(self, FileCache_mock):
+ FileCache_mock.return_value = None
+ dummy_controller = MagicMock()
+ config = AmbariConfig().getConfig()
+ tempdir = tempfile.gettempdir()
+ config.set('agent', 'prefix', tempdir)
+ CustomServiceOrchestrator(config, dummy_controller)
+ self.assertTrue(dummy_controller.registration_listeners.append.called)
+
+
+ @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+ @patch.object(CustomServiceOrchestrator, 'decompressClusterHostInfo')
+ @patch("ambari_agent.hostname.public_hostname")
+ @patch("os.path.isfile")
+ @patch("os.unlink")
+ @patch.object(FileCache, "__init__")
+ def test_dump_command_to_json(self, FileCache_mock, unlink_mock,
+ isfile_mock, hostname_mock,
+ decompress_cluster_host_info_mock):
+ FileCache_mock.return_value = None
+ hostname_mock.return_value = "test.hst"
+ command = {
+ 'commandType': 'EXECUTION_COMMAND',
+ 'role': u'DATANODE',
+ 'roleCommand': u'INSTALL',
+ 'commandId': '1-1',
+ 'taskId': 3,
+ 'clusterName': u'cc',
+ 'serviceName': u'HDFS',
+ 'configurations':{'global' : {}},
+ 'configurationTags':{'global' : { 'tag': 'v1' }},
+ 'clusterHostInfo':{'namenode_host' : ['1'],
+ 'slave_hosts' : ['0', '1'],
+ 'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
+ 'all_ping_ports': ['8670:0,1']},
+ 'hostLevelParams':{}
+ }
+
+ decompress_cluster_host_info_mock.return_value = {'namenode_host' : ['h2.hortonworks.com'],
+ 'slave_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
+ 'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
+ 'all_ping_ports': ['8670', '8670']}
+
+ config = AmbariConfig().getConfig()
+ tempdir = tempfile.gettempdir()
+ config.set('agent', 'prefix', tempdir)
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ isfile_mock.return_value = True
+ # Test dumping EXECUTION_COMMAND
+ json_file = orchestrator.dump_command_to_json(command)
+ self.assertTrue(os.path.exists(json_file))
+ self.assertTrue(os.path.getsize(json_file) > 0)
+ if get_platform() != PLATFORM_WINDOWS:
+ self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
+ self.assertTrue(json_file.endswith("command-3.json"))
+ self.assertTrue(decompress_cluster_host_info_mock.called)
+ os.unlink(json_file)
+ # Test dumping STATUS_COMMAND
+ command['commandType']='STATUS_COMMAND'
+ decompress_cluster_host_info_mock.reset_mock()
+ json_file = orchestrator.dump_command_to_json(command)
+ self.assertTrue(os.path.exists(json_file))
+ self.assertTrue(os.path.getsize(json_file) > 0)
+ if get_platform() != PLATFORM_WINDOWS:
+ self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
+ self.assertTrue(json_file.endswith("status_command.json"))
+ self.assertFalse(decompress_cluster_host_info_mock.called)
+ os.unlink(json_file)
+ # Testing side effect of dump_command_to_json
+ self.assertEquals(command['public_hostname'], "test.hst")
+ self.assertTrue(unlink_mock.called)
+
+
+ @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+ @patch("ambari_agent.hostname.public_hostname")
+ @patch("os.path.isfile")
+ @patch("os.unlink")
+ @patch.object(FileCache, "__init__")
+ def test_dump_command_to_json_with_retry(self, FileCache_mock, unlink_mock,
+ isfile_mock, hostname_mock):
+ FileCache_mock.return_value = None
+ hostname_mock.return_value = "test.hst"
+ command = {
+ 'commandType': 'EXECUTION_COMMAND',
+ 'role': u'DATANODE',
+ 'roleCommand': u'INSTALL',
+ 'commandId': '1-1',
+ 'taskId': 3,
+ 'clusterName': u'cc',
+ 'serviceName': u'HDFS',
+ 'configurations':{'global' : {}},
+ 'configurationTags':{'global' : { 'tag': 'v1' }},
+ 'clusterHostInfo':{'namenode_host' : ['1'],
+ 'slave_hosts' : ['0', '1'],
+ 'all_racks' : [u'/default-rack:0'],
+ 'ambari_server_host' : 'a.b.c',
+ 'all_ipv4_ips' : [u'192.168.12.101:0'],
+ 'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
+ 'all_ping_ports': ['8670:0,1']},
+ 'hostLevelParams':{}
+ }
+
+ config = AmbariConfig().getConfig()
+ tempdir = tempfile.gettempdir()
+ config.set('agent', 'prefix', tempdir)
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ isfile_mock.return_value = True
+ # Test dumping EXECUTION_COMMAND
+ json_file = orchestrator.dump_command_to_json(command)
+ self.assertTrue(os.path.exists(json_file))
+ self.assertTrue(os.path.getsize(json_file) > 0)
+ if get_platform() != PLATFORM_WINDOWS:
+ self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
+ self.assertTrue(json_file.endswith("command-3.json"))
+ os.unlink(json_file)
+ # Test dumping STATUS_COMMAND
+ json_file = orchestrator.dump_command_to_json(command, True)
+ self.assertTrue(os.path.exists(json_file))
+ self.assertTrue(os.path.getsize(json_file) > 0)
+ if get_platform() != PLATFORM_WINDOWS:
+ self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
+ self.assertTrue(json_file.endswith("command-3.json"))
+ os.unlink(json_file)
+ # Testing side effect of dump_command_to_json
+ self.assertEquals(command['public_hostname'], "test.hst")
+ self.assertTrue(unlink_mock.called)
+
+ @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+ @patch("os.path.exists")
+ @patch.object(FileCache, "__init__")
+ def test_resolve_script_path(self, FileCache_mock, exists_mock):
+ FileCache_mock.return_value = None
+ dummy_controller = MagicMock()
+ config = AmbariConfig().getConfig()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ # Testing existing path
+ exists_mock.return_value = True
+ path = orchestrator.\
+ resolve_script_path(os.path.join("HBASE", "package"), os.path.join("scripts", "hbase_master.py"))
+ self.assertEqual(os.path.join("HBASE", "package", "scripts", "hbase_master.py"), path)
+ # Testing not existing path
+ exists_mock.return_value = False
+ try:
+ orchestrator.resolve_script_path("/HBASE",
+ os.path.join("scripts", "hbase_master.py"))
+ self.fail('ExpectedException not thrown')
+ except AgentException:
+ pass # Expected
+
+
+ @patch.object(CustomServiceOrchestrator, "resolve_script_path")
+ @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
+ @patch.object(FileCache, "get_host_scripts_base_dir")
+ @patch.object(FileCache, "get_service_base_dir")
+ @patch.object(FileCache, "get_hook_base_dir")
+ @patch.object(CustomServiceOrchestrator, "dump_command_to_json")
+ @patch.object(PythonExecutor, "run_file")
+ @patch.object(FileCache, "__init__")
+ def test_runCommand(self, FileCache_mock,
+ run_file_mock, dump_command_to_json_mock,
+ get_hook_base_dir_mock, get_service_base_dir_mock,
+ get_host_scripts_base_dir_mock,
+ resolve_hook_script_path_mock,
+ resolve_script_path_mock):
+
+ FileCache_mock.return_value = None
+ command = {
+ 'role' : 'REGION_SERVER',
+ 'hostLevelParams' : {
+ 'stack_name' : 'HDP',
+ 'stack_version' : '2.0.7',
+ 'jdk_location' : 'some_location'
+ },
+ 'commandParams': {
+ 'script_type': 'PYTHON',
+ 'script': 'scripts/hbase_regionserver.py',
+ 'command_timeout': '600',
+ 'service_package_folder' : 'HBASE'
+ },
+ 'taskId' : '3',
+ 'roleCommand': 'INSTALL'
+ }
+
+ get_host_scripts_base_dir_mock.return_value = "/host_scripts"
+ get_service_base_dir_mock.return_value = "/basedir/"
+ resolve_script_path_mock.return_value = "/basedir/scriptpath"
+ resolve_hook_script_path_mock.return_value = \
+ ('/hooks_dir/prefix-command/scripts/hook.py',
+ '/hooks_dir/prefix-command')
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+ unix_process_id = 111
+ orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
+ get_hook_base_dir_mock.return_value = "/hooks/"
+ # normal run case
+ run_file_mock.return_value = {
+ 'stdout' : 'sss',
+ 'stderr' : 'eee',
+ 'exitcode': 0,
+ }
+ ret = orchestrator.runCommand(command, "out.txt", "err.txt")
+ self.assertEqual(ret['exitcode'], 0)
+ self.assertTrue(run_file_mock.called)
+ self.assertEqual(run_file_mock.call_count, 3)
+
+ run_file_mock.reset_mock()
+
+ # Case when we force another command
+ run_file_mock.return_value = {
+ 'stdout' : 'sss',
+ 'stderr' : 'eee',
+ 'exitcode': 0,
+ }
+ ret = orchestrator.runCommand(command, "out.txt", "err.txt",
+ forced_command_name=CustomServiceOrchestrator.SCRIPT_TYPE_PYTHON)
+ ## Check that override_output_files was true only during first call
+ print run_file_mock
+ self.assertEquals(run_file_mock.call_args_list[0][0][8], True)
+ self.assertEquals(run_file_mock.call_args_list[1][0][8], False)
+ self.assertEquals(run_file_mock.call_args_list[2][0][8], False)
+ ## Check that forced_command_name was taken into account
+ self.assertEqual(run_file_mock.call_args_list[0][0][1][0],
+ CustomServiceOrchestrator.SCRIPT_TYPE_PYTHON)
+
+ run_file_mock.reset_mock()
+
+ # unknown script type case
+ command['commandParams']['script_type'] = "SOME_TYPE"
+ ret = orchestrator.runCommand(command, "out.txt", "err.txt")
+ self.assertEqual(ret['exitcode'], 1)
+ self.assertFalse(run_file_mock.called)
+ self.assertTrue("Unknown script type" in ret['stdout'])
+
+ #By default returns empty dictionary
+ self.assertEqual(ret['structuredOut'], '{}')
+
+ pass
+
+ @patch("ambari_commons.shell.kill_process_with_children")
+ @patch.object(CustomServiceOrchestrator, "resolve_script_path")
+ @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
+ @patch.object(FileCache, "get_host_scripts_base_dir")
+ @patch.object(FileCache, "get_service_base_dir")
+ @patch.object(FileCache, "get_hook_base_dir")
+ @patch.object(CustomServiceOrchestrator, "dump_command_to_json")
+ @patch.object(PythonExecutor, "run_file")
+ @patch.object(FileCache, "__init__")
+ def test_cancel_command(self, FileCache_mock,
+ run_file_mock, dump_command_to_json_mock,
+ get_hook_base_dir_mock, get_service_base_dir_mock,
+ get_host_scripts_base_dir_mock,
+ resolve_hook_script_path_mock, resolve_script_path_mock,
+ kill_process_with_children_mock):
+ FileCache_mock.return_value = None
+ command = {
+ 'role' : 'REGION_SERVER',
+ 'hostLevelParams' : {
+ 'stack_name' : 'HDP',
+ 'stack_version' : '2.0.7',
+ 'jdk_location' : 'some_location'
+ },
+ 'commandParams': {
+ 'script_type': 'PYTHON',
+ 'script': 'scripts/hbase_regionserver.py',
+ 'command_timeout': '600',
+ 'service_package_folder' : 'HBASE'
+ },
+ 'taskId' : '3',
+ 'roleCommand': 'INSTALL'
+ }
+
+ get_host_scripts_base_dir_mock.return_value = "/host_scripts"
+ get_service_base_dir_mock.return_value = "/basedir/"
+ resolve_script_path_mock.return_value = "/basedir/scriptpath"
+ resolve_hook_script_path_mock.return_value = \
+ ('/hooks_dir/prefix-command/scripts/hook.py',
+ '/hooks_dir/prefix-command')
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+ unix_process_id = 111
+ orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
+ get_hook_base_dir_mock.return_value = "/hooks/"
+ run_file_mock_return_value = {
+ 'stdout' : 'killed',
+ 'stderr' : 'killed',
+ 'exitcode': 1,
+ }
+ def side_effect(*args, **kwargs):
+ time.sleep(0.2)
+ return run_file_mock_return_value
+ run_file_mock.side_effect = side_effect
+
+ _, out = tempfile.mkstemp()
+ _, err = tempfile.mkstemp()
+ pool = ThreadPool(processes=1)
+ async_result = pool.apply_async(orchestrator.runCommand, (command, out, err))
+
+ time.sleep(0.1)
+ orchestrator.cancel_command(command['taskId'], 'reason')
+
+ ret = async_result.get()
+
+ self.assertEqual(ret['exitcode'], 1)
+ self.assertEquals(ret['stdout'], 'killed\nCommand aborted. reason')
+ self.assertEquals(ret['stderr'], 'killed\nCommand aborted. reason')
+
+ self.assertTrue(kill_process_with_children_mock.called)
+ self.assertFalse(command['taskId'] in orchestrator.commands_in_progress.keys())
+ self.assertTrue(os.path.exists(out))
+ self.assertTrue(os.path.exists(err))
+ try:
+ os.remove(out)
+ os.remove(err)
+ except:
+ pass
+
+ from ambari_agent.StackVersionsFileHandler import StackVersionsFileHandler
+
+ @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+ @patch.object(CustomServiceOrchestrator, "get_py_executor")
+ @patch("ambari_commons.shell.kill_process_with_children")
+ @patch.object(FileCache, "__init__")
+ @patch.object(CustomServiceOrchestrator, "resolve_script_path")
+ @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
+ @patch.object(StackVersionsFileHandler, "read_stack_version")
+ def test_cancel_backgound_command(self, read_stack_version_mock, resolve_hook_script_path_mock,
+ resolve_script_path_mock, FileCache_mock, kill_process_with_children_mock,
+ get_py_executor_mock):
+ FileCache_mock.return_value = None
+ FileCache_mock.cache_dir = MagicMock()
+ resolve_hook_script_path_mock.return_value = None
+# shell.kill_process_with_children = MagicMock()
+ dummy_controller = MagicMock()
+ cfg = AmbariConfig()
+ cfg.set('agent', 'tolerate_download_failures', 'true')
+ cfg.set('agent', 'prefix', '.')
+ cfg.set('agent', 'cache_dir', 'background_tasks')
+
+ actionQueue = ActionQueue(cfg, dummy_controller)
+
+ dummy_controller.actionQueue = actionQueue
+ orchestrator = CustomServiceOrchestrator(cfg, dummy_controller)
+ orchestrator.file_cache = MagicMock()
+ def f (a, b):
+ return ""
+ orchestrator.file_cache.get_service_base_dir = f
+ actionQueue.customServiceOrchestrator = orchestrator
+
+ import TestActionQueue
+ import copy
+
+ pyex = PythonExecutor(actionQueue.customServiceOrchestrator.tmp_dir, actionQueue.customServiceOrchestrator.config)
+ TestActionQueue.patch_output_file(pyex)
+ pyex.prepare_process_result = MagicMock()
+ get_py_executor_mock.return_value = pyex
+ orchestrator.dump_command_to_json = MagicMock()
+
+ lock = threading.RLock()
+ complete_done = threading.Condition(lock)
+
+ complete_was_called = {}
+ def command_complete_w(process_condenced_result, handle):
+ with lock:
+ complete_was_called['visited']= ''
+ complete_done.wait(3)
+
+ actionQueue.on_background_command_complete_callback = TestActionQueue.wraped(actionQueue.on_background_command_complete_callback, command_complete_w, None)
+ execute_command = copy.deepcopy(TestActionQueue.TestActionQueue.background_command)
+ actionQueue.put([execute_command])
+ actionQueue.processBackgroundQueueSafeEmpty()
+
+ time.sleep(.1)
+
+ orchestrator.cancel_command(19,'')
+ self.assertTrue(kill_process_with_children_mock.called)
+ kill_process_with_children_mock.assert_called_with(33)
+
+ with lock:
+ complete_done.notifyAll()
+
+ with lock:
+ self.assertTrue(complete_was_called.has_key('visited'))
+
+ time.sleep(.1)
+
+ runningCommand = actionQueue.commandStatuses.get_command_status(19)
+ self.assertTrue(runningCommand is not None)
+ self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS)
+
+
+ @patch.object(CustomServiceOrchestrator, "dump_command_to_json")
+ @patch.object(PythonExecutor, "run_file")
+ @patch.object(FileCache, "__init__")
+ @patch.object(FileCache, "get_custom_actions_base_dir")
+ def test_runCommand_custom_action(self, get_custom_actions_base_dir_mock,
+ FileCache_mock,
+ run_file_mock, dump_command_to_json_mock):
+ FileCache_mock.return_value = None
+ get_custom_actions_base_dir_mock.return_value = "some path"
+ _, script = tempfile.mkstemp()
+ command = {
+ 'role' : 'any',
+ 'commandParams': {
+ 'script_type': 'PYTHON',
+ 'script': 'some_custom_action.py',
+ 'command_timeout': '600',
+ 'jdk_location' : 'some_location'
+ },
+ 'taskId' : '3',
+ 'roleCommand': 'ACTIONEXECUTE'
+ }
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+ unix_process_id = 111
+ orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
+ # normal run case
+ run_file_mock.return_value = {
+ 'stdout' : 'sss',
+ 'stderr' : 'eee',
+ 'exitcode': 0,
+ }
+ ret = orchestrator.runCommand(command, "out.txt", "err.txt")
+ self.assertEqual(ret['exitcode'], 0)
+ self.assertTrue(run_file_mock.called)
+ # Hoooks are not supported for custom actions,
+ # that's why run_file() should be called only once
+ self.assertEqual(run_file_mock.call_count, 1)
+
+
+ @patch("os.path.isfile")
+ @patch.object(FileCache, "__init__")
+ def test_resolve_hook_script_path(self, FileCache_mock, isfile_mock):
+ FileCache_mock.return_value = None
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+ # Testing None param
+ res1 = orchestrator.resolve_hook_script_path(None, "prefix", "command",
+ "script_type")
+ self.assertEqual(res1, None)
+ # Testing existing hook script
+ isfile_mock.return_value = True
+ res2 = orchestrator.resolve_hook_script_path("hooks_dir", "prefix", "command",
+ "script_type")
+ self.assertEqual(res2, (os.path.join('hooks_dir', 'prefix-command', 'scripts', 'hook.py'),
+ os.path.join('hooks_dir', 'prefix-command')))
+ # Testing not existing hook script
+ isfile_mock.return_value = False
+ res3 = orchestrator.resolve_hook_script_path("hooks_dir", "prefix", "command",
+ "script_type")
+ self.assertEqual(res3, None)
+
+
+ @patch.object(CustomServiceOrchestrator, "runCommand")
+ @patch.object(FileCache, "__init__")
+ def test_requestComponentStatus(self, FileCache_mock, runCommand_mock):
+ FileCache_mock.return_value = None
+ status_command = {
+ "serviceName" : 'HDFS',
+ "commandType" : "STATUS_COMMAND",
+ "clusterName" : "",
+ "componentName" : "DATANODE",
+ 'configurations':{}
+ }
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+ # Test alive case
+ runCommand_mock.return_value = {
+ "exitcode" : 0
+ }
+
+ status = orchestrator.requestComponentStatus(status_command)
+ self.assertEqual(runCommand_mock.return_value, status)
+
+ # Test dead case
+ runCommand_mock.return_value = {
+ "exitcode" : 1
+ }
+ status = orchestrator.requestComponentStatus(status_command)
+ self.assertEqual(runCommand_mock.return_value, status)
+
+ @patch.object(CustomServiceOrchestrator, "runCommand")
+ @patch.object(FileCache, "__init__")
+ def test_requestComponentSecurityState(self, FileCache_mock, runCommand_mock):
+ FileCache_mock.return_value = None
+ status_command = {
+ "serviceName" : 'HDFS',
+ "commandType" : "STATUS_COMMAND",
+ "clusterName" : "",
+ "componentName" : "DATANODE",
+ 'configurations':{}
+ }
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+ # Test securityState
+ runCommand_mock.return_value = {
+ 'exitcode' : 0,
+ 'structuredOut' : {'securityState': 'UNSECURED'}
+ }
+
+ status = orchestrator.requestComponentSecurityState(status_command)
+ self.assertEqual('UNSECURED', status)
+
+ # Test case where exit code indicates failure
+ runCommand_mock.return_value = {
+ "exitcode" : 1
+ }
+ status = orchestrator.requestComponentSecurityState(status_command)
+ self.assertEqual('UNKNOWN', status)
+
+ @patch.object(FileCache, "__init__")
+ def test_requestComponentSecurityState_realFailure(self, FileCache_mock):
+ '''
+ Tests the case where the CustomServiceOrchestrator attempts to call a service's security_status
+ method, but fails to do so because the script or method was not found.
+ :param FileCache_mock:
+ :return:
+ '''
+ FileCache_mock.return_value = None
+ status_command = {
+ "serviceName" : 'BOGUS_SERVICE',
+ "commandType" : "STATUS_COMMAND",
+ "clusterName" : "",
+ "componentName" : "DATANODE",
+ 'configurations':{}
+ }
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+
+ status = orchestrator.requestComponentSecurityState(status_command)
+ self.assertEqual('UNKNOWN', status)
+
+
+ @patch.object(CustomServiceOrchestrator, "get_py_executor")
+ @patch.object(CustomServiceOrchestrator, "dump_command_to_json")
+ @patch.object(FileCache, "__init__")
+ @patch.object(FileCache, "get_custom_actions_base_dir")
+ def test_runCommand_background_action(self, get_custom_actions_base_dir_mock,
+ FileCache_mock,
+ dump_command_to_json_mock,
+ get_py_executor_mock):
+ FileCache_mock.return_value = None
+ get_custom_actions_base_dir_mock.return_value = "some path"
+ _, script = tempfile.mkstemp()
+ command = {
+ 'role' : 'any',
+ 'commandParams': {
+ 'script_type': 'PYTHON',
+ 'script': 'some_custom_action.py',
+ 'command_timeout': '600',
+ 'jdk_location' : 'some_location'
+ },
+ 'taskId' : '13',
+ 'roleCommand': 'ACTIONEXECUTE',
+ 'commandType': 'BACKGROUND_EXECUTION_COMMAND',
+ '__handle': BackgroundCommandExecutionHandle({'taskId': '13'}, 13,
+ MagicMock(), MagicMock())
+ }
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+
+ import TestActionQueue
+ pyex = PythonExecutor(orchestrator.tmp_dir, orchestrator.config)
+ TestActionQueue.patch_output_file(pyex)
+ pyex.condenseOutput = MagicMock()
+ get_py_executor_mock.return_value = pyex
+ orchestrator.dump_command_to_json = MagicMock()
+
+ ret = orchestrator.runCommand(command, "out.txt", "err.txt")
+ self.assertEqual(ret['exitcode'], 777)
+
+ def tearDown(self):
+ # enable stdout
+ sys.stdout = sys.__stdout__
+
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/test/python/ambari_agent/TestSecurity.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestSecurity.py b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py
index 0e1e4ee..c1b7812 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestSecurity.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py
@@ -343,12 +343,14 @@ class TestSecurity(unittest.TestCase):
@patch("subprocess.Popen")
@patch("subprocess.Popen.communicate")
- def test_genAgentCrtReq(self, communicate_mock, popen_mock):
+ @patch.object(os, "chmod")
+ def test_genAgentCrtReq(self, chmod_mock, communicate_mock, popen_mock):
man = CertificateManager(self.config)
p = MagicMock(spec=subprocess.Popen)
p.communicate = communicate_mock
popen_mock.return_value = p
- man.genAgentCrtReq()
+ man.genAgentCrtReq('/dummy-keysdir/hostname.key')
+ self.assertTrue(chmod_mock.called)
self.assertTrue(popen_mock.called)
self.assertTrue(communicate_mock.called)
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/test/python/ambari_agent/TestSecurity.py.orig
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestSecurity.py.orig b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py.orig
new file mode 100644
index 0000000..0e1e4ee
--- /dev/null
+++ b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py.orig
@@ -0,0 +1,389 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import StringIO
+import sys, subprocess
+from mock.mock import MagicMock, patch, ANY
+import mock.mock
+import unittest
+import logging
+import signal
+import ConfigParser
+import ssl
+import os
+import tempfile
+
+from ambari_commons import OSCheck
+from only_for_platform import os_distro_value
+
+with patch("platform.linux_distribution", return_value = ('Suse','11','Final')):
+ from ambari_agent import NetUtil
+ from ambari_agent.security import CertificateManager
+ from ambari_agent import ProcessHelper, main
+ from ambari_agent.AmbariConfig import AmbariConfig
+ from ambari_agent.Controller import Controller
+ from ambari_agent import security
+
+aa = mock.mock.mock_open()
+class TestSecurity(unittest.TestCase):
+
+ @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+ def setUp(self):
+ # disable stdout
+ out = StringIO.StringIO()
+ sys.stdout = out
+ # Create config
+ self.config = AmbariConfig()
+ # Instantiate CachedHTTPSConnection (skip connect() call)
+ with patch.object(security.VerifiedHTTPSConnection, "connect"):
+ self.cachedHTTPSConnection = security.CachedHTTPSConnection(self.config)
+
+
+ def tearDown(self):
+ # enable stdout
+ sys.stdout = sys.__stdout__
+
+
+ ### VerifiedHTTPSConnection ###
+
+ @patch.object(security.CertificateManager, "initSecurity")
+ @patch("socket.create_connection")
+ @patch("ssl.wrap_socket")
+ def test_VerifiedHTTPSConnection_connect(self, wrap_socket_mock,
+ create_connection_mock,
+ init_security_mock):
+ init_security_mock.return_value = None
+ self.config.set('security', 'keysdir', '/dummy-keysdir')
+ connection = security.VerifiedHTTPSConnection("example.com",
+ self.config.get('server', 'secured_url_port'), self.config)
+ connection._tunnel_host = False
+ connection.sock = None
+ connection.connect()
+ self.assertTrue(wrap_socket_mock.called)
+
+ ### VerifiedHTTPSConnection with no certificates creation
+ @patch.object(security.CertificateManager, "initSecurity")
+ @patch("socket.create_connection")
+ @patch("ssl.wrap_socket")
+ def test_Verified_HTTPSConnection_non_secure_connect(self, wrap_socket_mock,
+ create_connection_mock,
+ init_security_mock):
+ connection = security.VerifiedHTTPSConnection("example.com",
+ self.config.get('server', 'secured_url_port'), self.config)
+ connection._tunnel_host = False
+ connection.sock = None
+ connection.connect()
+ self.assertFalse(init_security_mock.called)
+
+ ### VerifiedHTTPSConnection with two-way SSL authentication enabled
+ @patch.object(security.CertificateManager, "initSecurity")
+ @patch("socket.create_connection")
+ @patch("ssl.wrap_socket")
+ def test_Verified_HTTPSConnection_two_way_ssl_connect(self, wrap_socket_mock,
+ create_connection_mock,
+ init_security_mock):
+ wrap_socket_mock.side_effect=ssl.SSLError()
+ connection = security.VerifiedHTTPSConnection("example.com",
+ self.config.get('server', 'secured_url_port'), self.config)
+ connection._tunnel_host = False
+ connection.sock = None
+ try:
+ connection.connect()
+ except ssl.SSLError:
+ pass
+ self.assertTrue(init_security_mock.called)
+
+ ### CachedHTTPSConnection ###
+
+ @patch.object(security.VerifiedHTTPSConnection, "connect")
+ def test_CachedHTTPSConnection_connect(self, vhc_connect_mock):
+ self.config.set('server', 'hostname', 'dummy.server.hostname')
+ self.config.set('server', 'secured_url_port', '443')
+ # Testing not connected case
+ self.cachedHTTPSConnection.connected = False
+ self.cachedHTTPSConnection.connect()
+ self.assertTrue(vhc_connect_mock.called)
+ vhc_connect_mock.reset_mock()
+ # Testing already connected case
+ self.cachedHTTPSConnection.connect()
+ self.assertFalse(vhc_connect_mock.called)
+
+
+ @patch.object(security.CachedHTTPSConnection, "connect")
+ def test_forceClear(self, connect_mock):
+ # Testing if httpsconn instance changed
+ old = self.cachedHTTPSConnection.httpsconn
+ self.cachedHTTPSConnection.forceClear()
+ self.assertNotEqual(old, self.cachedHTTPSConnection.httpsconn)
+
+
+ @patch.object(security.CachedHTTPSConnection, "connect")
+ def test_request(self, connect_mock):
+ httpsconn_mock = MagicMock(create = True)
+ self.cachedHTTPSConnection.httpsconn = httpsconn_mock
+
+ dummy_request = MagicMock(create = True)
+ dummy_request.get_method.return_value = "dummy_get_method"
+ dummy_request.get_full_url.return_value = "dummy_full_url"
+ dummy_request.get_data.return_value = "dummy_get_data"
+ dummy_request.headers = "dummy_headers"
+
+ responce_mock = MagicMock(create = True)
+ responce_mock.read.return_value = "dummy responce"
+ httpsconn_mock.getresponse.return_value = responce_mock
+
+ # Testing normal case
+ responce = self.cachedHTTPSConnection.request(dummy_request)
+
+ self.assertEqual(responce, responce_mock.read.return_value)
+ httpsconn_mock.request.assert_called_once_with(
+ dummy_request.get_method.return_value,
+ dummy_request.get_full_url.return_value,
+ dummy_request.get_data.return_value,
+ dummy_request.headers)
+
+ # Testing case of exception
+ try:
+ def side_eff():
+ raise Exception("Dummy exception")
+ httpsconn_mock.read.side_effect = side_eff
+ responce = self.cachedHTTPSConnection.request(dummy_request)
+ self.fail("Should raise IOError")
+ except Exception, err:
+ # Expected
+ pass
+
+
+ ### CertificateManager ###
+
+
+ @patch("ambari_agent.hostname.hostname")
+ def test_getAgentKeyName(self, hostname_mock):
+ hostname_mock.return_value = "dummy.hostname"
+ self.config.set('security', 'keysdir', '/dummy-keysdir')
+ man = CertificateManager(self.config)
+ res = man.getAgentKeyName()
+ self.assertEquals(res, os.path.abspath("/dummy-keysdir/dummy.hostname.key"))
+
+
+ @patch("ambari_agent.hostname.hostname")
+ def test_getAgentCrtName(self, hostname_mock):
+ hostname_mock.return_value = "dummy.hostname"
+ self.config.set('security', 'keysdir', '/dummy-keysdir')
+ man = CertificateManager(self.config)
+ res = man.getAgentCrtName()
+ self.assertEquals(res, os.path.abspath("/dummy-keysdir/dummy.hostname.crt"))
+
+
+ @patch("ambari_agent.hostname.hostname")
+ def test_getAgentCrtReqName(self, hostname_mock):
+ hostname_mock.return_value = "dummy.hostname"
+ self.config.set('security', 'keysdir', '/dummy-keysdir')
+ man = CertificateManager(self.config)
+ res = man.getAgentCrtReqName()
+ self.assertEquals(res, os.path.abspath("/dummy-keysdir/dummy.hostname.csr"))
+
+
+ def test_getSrvrCrtName(self):
+ self.config.set('security', 'keysdir', '/dummy-keysdir')
+ man = CertificateManager(self.config)
+ res = man.getSrvrCrtName()
+ self.assertEquals(res, os.path.abspath("/dummy-keysdir/ca.crt"))
+
+
+ @patch("os.path.exists")
+ @patch.object(security.CertificateManager, "loadSrvrCrt")
+ @patch.object(security.CertificateManager, "getAgentKeyName")
+ @patch.object(security.CertificateManager, "genAgentCrtReq")
+ @patch.object(security.CertificateManager, "getAgentCrtName")
+ @patch.object(security.CertificateManager, "reqSignCrt")
+ def test_checkCertExists(self, reqSignCrt_mock, getAgentCrtName_mock,
+ genAgentCrtReq_mock, getAgentKeyName_mock,
+ loadSrvrCrt_mock, exists_mock):
+ self.config.set('security', 'keysdir', '/dummy-keysdir')
+ getAgentKeyName_mock.return_value = "dummy AgentKeyName"
+ getAgentCrtName_mock.return_value = "dummy AgentCrtName"
+ man = CertificateManager(self.config)
+
+ # Case when all files exist
+ exists_mock.side_effect = [True, True, True]
+ man.checkCertExists()
+ self.assertFalse(loadSrvrCrt_mock.called)
+ self.assertFalse(genAgentCrtReq_mock.called)
+ self.assertFalse(reqSignCrt_mock.called)
+
+ # Absent server cert
+ exists_mock.side_effect = [False, True, True]
+ man.checkCertExists()
+ self.assertTrue(loadSrvrCrt_mock.called)
+ self.assertFalse(genAgentCrtReq_mock.called)
+ self.assertFalse(reqSignCrt_mock.called)
+ loadSrvrCrt_mock.reset_mock()
+
+ # Absent agent key
+ exists_mock.side_effect = [True, False, True]
+ man.checkCertExists()
+ self.assertFalse(loadSrvrCrt_mock.called)
+ self.assertTrue(genAgentCrtReq_mock.called)
+ self.assertFalse(reqSignCrt_mock.called)
+ genAgentCrtReq_mock.reset_mock()
+
+ # Absent agent cert
+ exists_mock.side_effect = [True, True, False]
+ man.checkCertExists()
+ self.assertFalse(loadSrvrCrt_mock.called)
+ self.assertFalse(genAgentCrtReq_mock.called)
+ self.assertTrue(reqSignCrt_mock.called)
+ reqSignCrt_mock.reset_mock()
+
+
+
+ @patch("urllib2.OpenerDirector.open")
+ @patch.object(security.CertificateManager, "getSrvrCrtName")
+ def test_loadSrvrCrt(self, getSrvrCrtName_mock, urlopen_mock):
+ read_mock = MagicMock(create=True)
+ read_mock.read.return_value = "dummy_cert"
+ urlopen_mock.return_value = read_mock
+ _, tmpoutfile = tempfile.mkstemp()
+ getSrvrCrtName_mock.return_value = tmpoutfile
+
+ man = CertificateManager(self.config)
+ man.loadSrvrCrt()
+
+ # Checking file contents
+ saved = open(tmpoutfile, 'r').read()
+ self.assertEqual(saved, read_mock.read.return_value)
+ try:
+ os.unlink(tmpoutfile)
+ except:
+ pass
+
+
+ @patch("ambari_agent.hostname.hostname")
+ @patch('__builtin__.open', create=True, autospec=True)
+ @patch.dict('os.environ', {'DUMMY_PASSPHRASE': 'dummy-passphrase'})
+ @patch('ambari_simplejson.dumps')
+ @patch('urllib2.Request')
+ @patch("urllib2.OpenerDirector.open")
+ @patch('ambari_simplejson.loads')
+ def test_reqSignCrt(self, loads_mock, urlopen_mock, request_mock, dumps_mock, open_mock, hostname_mock):
+ self.config.set('security', 'keysdir', '/dummy-keysdir')
+ self.config.set('security', 'passphrase_env_var_name', 'DUMMY_PASSPHRASE')
+ man = CertificateManager(self.config)
+ hostname_mock.return_value = "dummy-hostname"
+
+ open_mock.return_value.read.return_value = "dummy_request"
+ urlopen_mock.return_value.read.return_value = "dummy_server_request"
+ loads_mock.return_value = {
+ 'result': 'OK',
+ 'signedCa': 'dummy-crt'
+ }
+
+ # Test normal server interaction
+ man.reqSignCrt()
+
+ self.assertEqual(dumps_mock.call_args[0][0], {
+ 'csr' : 'dummy_request',
+ 'passphrase' : 'dummy-passphrase'
+ })
+ self.assertEqual(open_mock.return_value.write.call_args[0][0], 'dummy-crt')
+
+ # Test negative server reply
+ dumps_mock.reset_mock()
+ open_mock.return_value.write.reset_mock()
+ loads_mock.return_value = {
+ 'result': 'FAIL',
+ 'signedCa': 'fail-crt'
+ }
+
+ # If certificate signing failed, then exception must be raised
+ try:
+ man.reqSignCrt()
+ self.fail()
+ except ssl.SSLError:
+ pass
+ self.assertFalse(open_mock.return_value.write.called)
+
+ # Test connection fail
+ dumps_mock.reset_mock()
+ open_mock.return_value.write.reset_mock()
+
+ try:
+ man.reqSignCrt()
+ self.fail("Expected exception here")
+ except Exception, err:
+ # expected
+ pass
+
+ # Test malformed JSON response
+ open_mock.return_value.write.reset_mock()
+ loads_mock.side_effect = Exception()
+ try:
+ man.reqSignCrt()
+ self.fail("Expected exception here")
+ except ssl.SSLError:
+ pass
+ self.assertFalse(open_mock.return_value.write.called)
+
+ @patch("subprocess.Popen")
+ @patch("subprocess.Popen.communicate")
+ def test_genAgentCrtReq(self, communicate_mock, popen_mock):
+ man = CertificateManager(self.config)
+ p = MagicMock(spec=subprocess.Popen)
+ p.communicate = communicate_mock
+ popen_mock.return_value = p
+ man.genAgentCrtReq()
+ self.assertTrue(popen_mock.called)
+ self.assertTrue(communicate_mock.called)
+
+ @patch("ambari_agent.hostname.hostname")
+ @patch('__builtin__.open', create=True, autospec=True)
+ @patch("urllib2.OpenerDirector.open")
+ @patch.dict('os.environ', {'DUMMY_PASSPHRASE': 'dummy-passphrase'})
+ def test_reqSignCrt_malformedJson(self, urlopen_mock, open_mock, hostname_mock):
+ hostname_mock.return_value = "dummy-hostname"
+ open_mock.return_value.read.return_value = "dummy_request"
+ self.config.set('security', 'keysdir', '/dummy-keysdir')
+ self.config.set('security', 'passphrase_env_var_name', 'DUMMY_PASSPHRASE')
+ man = CertificateManager(self.config)
+
+ # test valid JSON response
+ urlopen_mock.return_value.read.return_value = '{"result": "OK", "signedCa":"dummy"}'
+ try:
+ man.reqSignCrt()
+ except ssl.SSLError:
+ self.fail("Unexpected exception!")
+ open_mock.return_value.write.assert_called_with(u'dummy')
+
+ # test malformed JSON response
+ open_mock.return_value.write.reset_mock()
+ urlopen_mock.return_value.read.return_value = '{malformed_object}'
+ try:
+ man.reqSignCrt()
+ self.fail("Expected exception!")
+ except ssl.SSLError:
+ pass
+ self.assertFalse(open_mock.return_value.write.called)
+
+ @patch.object(security.CertificateManager, "checkCertExists")
+ def test_initSecurity(self, checkCertExists_method):
+ man = CertificateManager(self.config)
+ man.initSecurity()
+ self.assertTrue(checkCertExists_method.called)
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index 1ee5b70..e0a7877 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -61,7 +61,7 @@ USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT> <LOGGING_LEV
<BASEDIR> path to service metadata dir. Ex: /var/lib/ambari-agent/cache/common-services/HDFS/2.1.0.2.0/package
<STROUTPUT> path to file with structured command output (file will be created). Ex:/tmp/my.txt
<LOGGING_LEVEL> log level for stdout. Ex:DEBUG,INFO
-<TMP_DIR> temporary directory for executable scripts. Ex: /var/lib/ambari-agent/data/tmp
+<TMP_DIR> temporary directory for executable scripts. Ex: /var/lib/ambari-agent/tmp
"""
_PASSWORD_MAP = {"/configurations/cluster-env/hadoop.user.name":"/configurations/cluster-env/hadoop.user.password"}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-server/src/main/python/bootstrap.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/bootstrap.py b/ambari-server/src/main/python/bootstrap.py
index faf873b..3eba75c 100755
--- a/ambari-server/src/main/python/bootstrap.py
+++ b/ambari-server/src/main/python/bootstrap.py
@@ -47,7 +47,7 @@ MAX_PARALLEL_BOOTSTRAPS = 20
# How many seconds to wait between polling parallel bootstraps
POLL_INTERVAL_SEC = 1
DEBUG = False
-DEFAULT_AGENT_TEMP_FOLDER = "/var/lib/ambari-agent/data/tmp"
+DEFAULT_AGENT_TEMP_FOLDER = "/var/lib/ambari-agent/tmp"
DEFAULT_AGENT_DATA_FOLDER = "/var/lib/ambari-agent/data"
DEFAULT_AGENT_LIB_FOLDER = "/var/lib/ambari-agent"
PYTHON_ENV="env PYTHONPATH=$PYTHONPATH:" + DEFAULT_AGENT_TEMP_FOLDER
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
index 1415367..a3c02a6 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
@@ -202,7 +202,7 @@ class NameNodeDefault(NameNode):
# Create the kerberos credentials cache (ccache) file and set it in the environment to use
# when executing HDFS rebalance command. Use the md5 hash of the combination of the principal and keytab file
# to generate a (relatively) unique cache filename so that we can use it as needed.
- # TODO: params.tmp_dir=/var/lib/ambari-agent/data/tmp. However hdfs user doesn't have access to this path.
+ # TODO: params.tmp_dir=/var/lib/ambari-agent/tmp. However hdfs user doesn't have access to this path.
# TODO: Hence using /tmp
ccache_file_name = "hdfs_rebalance_cc_" + _md5(format("{hdfs_principal_name}|{hdfs_user_keytab}")).hexdigest()
ccache_file_path = os.path.join(tempfile.gettempdir(), ccache_file_name)
http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-server/src/test/python/TestBootstrap.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestBootstrap.py b/ambari-server/src/test/python/TestBootstrap.py
index 792d99d..e143a68 100644
--- a/ambari-server/src/test/python/TestBootstrap.py
+++ b/ambari-server/src/test/python/TestBootstrap.py
@@ -112,8 +112,8 @@ class TestBootstrap(TestCase):
utime = 1234
bootstrap_obj.getUtime = MagicMock(return_value=utime)
ret = bootstrap_obj.getRunSetupWithPasswordCommand("hostname")
- expected = "sudo -S python /var/lib/ambari-agent/data/tmp/setupAgent{0}.py hostname TEST_PASSPHRASE " \
- "ambariServer root 8440 < /var/lib/ambari-agent/data/tmp/host_pass{0}".format(utime)
+ expected = "sudo -S python /var/lib/ambari-agent/tmp/setupAgent{0}.py hostname TEST_PASSPHRASE " \
+ "ambariServer root 8440 < /var/lib/ambari-agent/tmp/host_pass{0}".format(utime)
self.assertEquals(ret, expected)
@@ -353,11 +353,11 @@ class TestBootstrap(TestCase):
self.assertEquals(res, expected)
command = str(init_mock.call_args[0][3])
self.assertEqual(command,
- "sudo mkdir -p /var/lib/ambari-agent/data/tmp ; "
- "sudo chown -R root /var/lib/ambari-agent/data/tmp ; "
+ "sudo mkdir -p /var/lib/ambari-agent/tmp ; "
+ "sudo chown -R root /var/lib/ambari-agent/tmp ; "
"sudo chmod 755 /var/lib/ambari-agent ; "
"sudo chmod 755 /var/lib/ambari-agent/data ; "
- "sudo chmod 777 /var/lib/ambari-agent/data/tmp")
+ "sudo chmod 777 /var/lib/ambari-agent/tmp")
@patch.object(BootstrapDefault, "getOsCheckScript")
@patch.object(BootstrapDefault, "getOsCheckScriptRemoteLocation")
@@ -520,7 +520,7 @@ class TestBootstrap(TestCase):
command = str(init_mock.call_args[0][3])
self.assertEqual(command,
"chmod a+x OsCheckScriptRemoteLocation && "
- "env PYTHONPATH=$PYTHONPATH:/var/lib/ambari-agent/data/tmp OsCheckScriptRemoteLocation centos6")
+ "env PYTHONPATH=$PYTHONPATH:/var/lib/ambari-agent/tmp OsCheckScriptRemoteLocation centos6")
@patch.object(SSH, "__init__")