You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2015/09/14 14:00:32 UTC
ambari git commit: Verify if restricting acls on
/var/lib/ambari-agent/data will be OK (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/branch-2.1 cba000d2d -> ccc200e33
Verify if restricting acls on /var/lib/ambari-agent/data will be OK (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ccc200e3
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ccc200e3
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ccc200e3
Branch: refs/heads/branch-2.1
Commit: ccc200e3345e06b23644e20de74b51476b4cac8b
Parents: cba000d
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Mon Sep 14 14:59:53 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Mon Sep 14 15:00:23 2015 +0300
----------------------------------------------------------------------
ambari-agent/conf/unix/ambari-agent.ini | 1 -
ambari-agent/conf/unix/install-helper.sh | 5 +++--
ambari-agent/pom.xml | 4 ++--
.../src/main/python/ambari_agent/Constants.py | 21 ++++++++++++++++++++
.../ambari_agent/CustomServiceOrchestrator.py | 3 ++-
.../python/ambari_agent/alerts/metric_alert.py | 3 ++-
.../python/ambari_agent/alerts/script_alert.py | 3 ++-
.../python/ambari_agent/alerts/web_alert.py | 3 ++-
.../src/main/python/ambari_agent/security.py | 11 +++++++---
.../python/ambari_agent/TestCertGeneration.py | 6 ++++--
.../TestCustomServiceOrchestrator.py | 1 -
.../test/python/ambari_agent/TestSecurity.py | 6 ++++--
.../libraries/script/script.py | 2 +-
ambari-server/src/main/python/bootstrap.py | 2 +-
.../HDFS/2.1.0.2.0/package/scripts/namenode.py | 2 +-
ambari-server/src/test/python/TestBootstrap.py | 12 +++++------
16 files changed, 59 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/conf/unix/ambari-agent.ini
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/unix/ambari-agent.ini b/ambari-agent/conf/unix/ambari-agent.ini
index abfde62..3b7631c 100644
--- a/ambari-agent/conf/unix/ambari-agent.ini
+++ b/ambari-agent/conf/unix/ambari-agent.ini
@@ -19,7 +19,6 @@ secured_url_port=8441
[agent]
prefix=/var/lib/ambari-agent/data
-tmp_dir=/var/lib/ambari-agent/data/tmp
;loglevel=(DEBUG/INFO)
loglevel=INFO
data_cleanup_interval=86400
http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/conf/unix/install-helper.sh
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/unix/install-helper.sh b/ambari-agent/conf/unix/install-helper.sh
index 48391d5..35c67fb 100644
--- a/ambari-agent/conf/unix/install-helper.sh
+++ b/ambari-agent/conf/unix/install-helper.sh
@@ -58,8 +58,9 @@ do_install(){
# on nano Ubuntu, when umask=027 those folders are created without 'x' bit for 'others'.
# which causes failures when hadoop users try to access tmp_dir
chmod a+x /var/lib/ambari-agent
- chmod a+x /var/lib/ambari-agent/data
- chmod 777 /var/lib/ambari-agent/data/tmp
+
+ chmod 777 /var/lib/ambari-agent/tmp
+ chmod 700 /var/lib/ambari-agent/data
}
do_remove(){
http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-agent/pom.xml b/ambari-agent/pom.xml
index c2bee4a..b544d75 100644
--- a/ambari-agent/pom.xml
+++ b/ambari-agent/pom.xml
@@ -391,7 +391,7 @@
<groupname>root</groupname>
</mapping>
<mapping>
- <directory>/var/lib/${project.artifactId}/data/tmp</directory>
+ <directory>/var/lib/${project.artifactId}/tmp</directory>
<filemode>777</filemode>
<username>root</username>
<groupname>root</groupname>
@@ -596,7 +596,7 @@
<paths>
<path>${package.pid.dir}</path>
<path>/var/lib/${project.artifactId}/data</path>
- <path>/var/lib/${project.artifactId}/data/tmp</path>
+ <path>/var/lib/${project.artifactId}/tmp</path>
<path>/var/lib/${project.artifactId}/keys</path>
<path>${package.log.dir}</path>
<path>/var/lib/${project.artifactId}/lib</path>
http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/src/main/python/ambari_agent/Constants.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/Constants.py b/ambari-agent/src/main/python/ambari_agent/Constants.py
new file mode 100644
index 0000000..6b80f00
--- /dev/null
+++ b/ambari-agent/src/main/python/ambari_agent/Constants.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+AGENT_TMP_DIR = "/var/lib/ambari-agent/tmp"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
index 6ee929c..6c1a161 100644
--- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
+++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
@@ -29,6 +29,7 @@ from FileCache import FileCache
from AgentException import AgentException
from PythonExecutor import PythonExecutor
from PythonReflectiveExecutor import PythonReflectiveExecutor
+import Constants
import hostname
@@ -61,7 +62,7 @@ class CustomServiceOrchestrator():
def __init__(self, config, controller):
self.config = config
self.tmp_dir = config.get('agent', 'prefix')
- self.exec_tmp_dir = config.get('agent', 'tmp_dir')
+ self.exec_tmp_dir = Constants.AGENT_TMP_DIR
self.file_cache = FileCache(config)
self.status_commands_stdout = os.path.join(self.tmp_dir,
'status_command_stdout.txt')
http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
index aa4ad75..8de49cd 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
@@ -30,6 +30,7 @@ from alerts.base_alert import BaseAlert
from ambari_commons.urllib_handlers import RefreshHeaderProcessor
from resource_management.libraries.functions.get_port_from_url import get_port_from_url
from resource_management.libraries.functions.curl_krb_request import curl_krb_request
+from ambari_agent import Constants
logger = logging.getLogger()
@@ -199,7 +200,7 @@ class MetricAlert(BaseAlert):
content = ''
try:
if kerberos_principal is not None and kerberos_keytab is not None and security_enabled:
- tmp_dir = self.config.get('agent', 'tmp_dir')
+ tmp_dir = Constants.AGENT_TMP_DIR
if tmp_dir is None:
tmp_dir = gettempdir()
http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py
index 76afbc9..e70dc22 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py
@@ -25,6 +25,7 @@ import re
from alerts.base_alert import BaseAlert
from resource_management.core.environment import Environment
from resource_management.core.logger import Logger
+from ambari_agent import Constants
logger = logging.getLogger(__name__)
@@ -99,7 +100,7 @@ class ScriptAlert(BaseAlert):
matchObj = re.match( r'((.*)services(.*)package)', self.path_to_script)
if matchObj:
basedir = matchObj.group(1)
- with Environment(basedir, tmp_dir=self.config.get('agent', 'tmp_dir')) as env:
+ with Environment(basedir, tmp_dir=Constants.AGENT_TMP_DIR) as env:
return cmd_module.execute(configurations, self.parameters, self.host_name)
else:
return cmd_module.execute(configurations, self.parameters, self.host_name)
http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
index b76d5e0..0d627c3 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
@@ -31,6 +31,7 @@ from resource_management.libraries.functions.get_port_from_url import get_port_f
from resource_management.libraries.functions.curl_krb_request import curl_krb_request
from ambari_commons import OSCheck
from ambari_commons.inet_utils import resolve_address
+from ambari_agent import Constants
# hashlib is supplied as of Python 2.5 as the replacement interface for md5
# and other secure hashes. In 2.6, md5 is deprecated. Import hashlib if
@@ -171,7 +172,7 @@ class WebAlert(BaseAlert):
# Create the kerberos credentials cache (ccache) file and set it in the environment to use
# when executing curl. Use the md5 hash of the combination of the principal and keytab file
# to generate a (relatively) unique cache filename so that we can use it as needed.
- tmp_dir = self.config.get('agent', 'tmp_dir')
+ tmp_dir = Constants.AGENT_TMP_DIR
if tmp_dir is None:
tmp_dir = gettempdir()
http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/src/main/python/ambari_agent/security.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/security.py b/ambari-agent/src/main/python/ambari_agent/security.py
index bfaf134..dc1f141 100644
--- a/ambari-agent/src/main/python/ambari_agent/security.py
+++ b/ambari-agent/src/main/python/ambari_agent/security.py
@@ -35,6 +35,7 @@ logger = logging.getLogger(__name__)
GEN_AGENT_KEY = 'openssl req -new -newkey rsa:1024 -nodes -keyout "%(keysdir)s' \
+ os.sep + '%(hostname)s.key" -subj /OU=%(hostname)s/ ' \
'-out "%(keysdir)s' + os.sep + '%(hostname)s.csr"'
+KEY_FILENAME = '%(hostname)s.key'
class VerifiedHTTPSConnection(httplib.HTTPSConnection):
@@ -190,7 +191,7 @@ class CertificateManager():
if not agent_key_exists:
logger.info("Agent key not exists, generating request")
- self.genAgentCrtReq()
+ self.genAgentCrtReq(self.getAgentKeyName())
else:
logger.info("Agent key exists, ok")
@@ -252,10 +253,12 @@ class CertificateManager():
'\nExiting..')
raise ssl.SSLError
- def genAgentCrtReq(self):
+ def genAgentCrtReq(self, keyname):
+ keysdir = os.path.abspath(self.config.get('security', 'keysdir'))
generate_script = GEN_AGENT_KEY % {
'hostname': hostname.hostname(self.config),
- 'keysdir': os.path.abspath(self.config.get('security', 'keysdir'))}
+ 'keysdir': keysdir}
+
logger.info(generate_script)
if platform.system() == 'Windows':
p = subprocess.Popen(generate_script, stdout=subprocess.PIPE)
@@ -264,6 +267,8 @@ class CertificateManager():
p = subprocess.Popen([generate_script], shell=True,
stdout=subprocess.PIPE)
p.communicate()
+ # this is required to be 600 for security concerns.
+ os.chmod(keyname, 0600)
def initSecurity(self):
self.checkCertExists()
http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py b/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py
index d188dbd..be72588 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py
@@ -47,8 +47,10 @@ class TestCertGeneration(TestCase):
config.set('security', 'server_crt', 'ca.crt')
self.certMan = CertificateManager(config)
- def test_generation(self):
- self.certMan.genAgentCrtReq()
+ @patch.object(os, "chmod")
+ def test_generation(self, chmod_mock):
+ self.certMan.genAgentCrtReq('/dummy_dir/hostname.key')
+ self.assertTrue(chmod_mock.called)
self.assertTrue(os.path.exists(self.certMan.getAgentKeyName()))
self.assertTrue(os.path.exists(self.certMan.getAgentCrtReqName()))
def tearDown(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
index 831ecce..e08e2f7 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
@@ -61,7 +61,6 @@ class TestCustomServiceOrchestrator(TestCase):
self.config = ConfigParser.RawConfigParser()
self.config.add_section('agent')
self.config.set('agent', 'prefix', tmpdir)
- self.config.set('agent', 'tmp_dir', exec_tmp_dir)
self.config.set('agent', 'cache_dir', "/cachedir")
self.config.add_section('python')
self.config.set('python', 'custom_actions_dir', tmpdir)
http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/src/test/python/ambari_agent/TestSecurity.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestSecurity.py b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py
index c47172a..a1d04e1 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestSecurity.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py
@@ -348,12 +348,14 @@ class TestSecurity(unittest.TestCase):
@patch("subprocess.Popen")
@patch("subprocess.Popen.communicate")
- def test_genAgentCrtReq(self, communicate_mock, popen_mock):
+ @patch.object(os, "chmod")
+ def test_genAgentCrtReq(self, chmod_mock, communicate_mock, popen_mock):
man = CertificateManager(self.config)
p = MagicMock(spec=subprocess.Popen)
p.communicate = communicate_mock
popen_mock.return_value = p
- man.genAgentCrtReq()
+ man.genAgentCrtReq('/dummy-keysdir/hostname.key')
+ self.assertTrue(chmod_mock.called)
self.assertTrue(popen_mock.called)
self.assertTrue(communicate_mock.called)
http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index a2c0c45..b4731a0 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -61,7 +61,7 @@ USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT> <LOGGING_LEV
<BASEDIR> path to service metadata dir. Ex: /var/lib/ambari-agent/cache/common-services/HDFS/2.1.0.2.0/package
<STROUTPUT> path to file with structured command output (file will be created). Ex:/tmp/my.txt
<LOGGING_LEVEL> log level for stdout. Ex:DEBUG,INFO
-<TMP_DIR> temporary directory for executable scripts. Ex: /var/lib/ambari-agent/data/tmp
+<TMP_DIR> temporary directory for executable scripts. Ex: /var/lib/ambari-agent/tmp
"""
_PASSWORD_MAP = {"/configurations/cluster-env/hadoop.user.name":"/configurations/cluster-env/hadoop.user.password"}
http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-server/src/main/python/bootstrap.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/bootstrap.py b/ambari-server/src/main/python/bootstrap.py
index 98a3a93..adde438 100755
--- a/ambari-server/src/main/python/bootstrap.py
+++ b/ambari-server/src/main/python/bootstrap.py
@@ -47,7 +47,7 @@ MAX_PARALLEL_BOOTSTRAPS = 20
# How many seconds to wait between polling parallel bootstraps
POLL_INTERVAL_SEC = 1
DEBUG = False
-DEFAULT_AGENT_TEMP_FOLDER = "/var/lib/ambari-agent/data/tmp"
+DEFAULT_AGENT_TEMP_FOLDER = "/var/lib/ambari-agent/tmp"
DEFAULT_AGENT_DATA_FOLDER = "/var/lib/ambari-agent/data"
DEFAULT_AGENT_LIB_FOLDER = "/var/lib/ambari-agent"
PYTHON_ENV="env PYTHONPATH=$PYTHONPATH:" + DEFAULT_AGENT_TEMP_FOLDER
http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
index 1415367..a3c02a6 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
@@ -202,7 +202,7 @@ class NameNodeDefault(NameNode):
# Create the kerberos credentials cache (ccache) file and set it in the environment to use
# when executing HDFS rebalance command. Use the md5 hash of the combination of the principal and keytab file
# to generate a (relatively) unique cache filename so that we can use it as needed.
- # TODO: params.tmp_dir=/var/lib/ambari-agent/data/tmp. However hdfs user doesn't have access to this path.
+ # TODO: params.tmp_dir=/var/lib/ambari-agent/tmp. However hdfs user doesn't have access to this path.
# TODO: Hence using /tmp
ccache_file_name = "hdfs_rebalance_cc_" + _md5(format("{hdfs_principal_name}|{hdfs_user_keytab}")).hexdigest()
ccache_file_path = os.path.join(tempfile.gettempdir(), ccache_file_name)
http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-server/src/test/python/TestBootstrap.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestBootstrap.py b/ambari-server/src/test/python/TestBootstrap.py
index 1fcb3ad..633ee39 100644
--- a/ambari-server/src/test/python/TestBootstrap.py
+++ b/ambari-server/src/test/python/TestBootstrap.py
@@ -112,8 +112,8 @@ class TestBootstrap(TestCase):
utime = 1234
bootstrap_obj.getUtime = MagicMock(return_value=utime)
ret = bootstrap_obj.getRunSetupWithPasswordCommand("hostname")
- expected = "sudo -S python /var/lib/ambari-agent/data/tmp/setupAgent{0}.py hostname TEST_PASSPHRASE " \
- "ambariServer root 8440 < /var/lib/ambari-agent/data/tmp/host_pass{0}".format(utime)
+ expected = "sudo -S python /var/lib/ambari-agent/tmp/setupAgent{0}.py hostname TEST_PASSPHRASE " \
+ "ambariServer root 8440 < /var/lib/ambari-agent/tmp/host_pass{0}".format(utime)
self.assertEquals(ret, expected)
@@ -353,11 +353,11 @@ class TestBootstrap(TestCase):
self.assertEquals(res, expected)
command = str(init_mock.call_args[0][3])
self.assertEqual(command,
- "sudo mkdir -p /var/lib/ambari-agent/data/tmp ; "
- "sudo chown -R root /var/lib/ambari-agent/data/tmp ; "
+ "sudo mkdir -p /var/lib/ambari-agent/tmp ; "
+ "sudo chown -R root /var/lib/ambari-agent/tmp ; "
"sudo chmod 755 /var/lib/ambari-agent ; "
"sudo chmod 755 /var/lib/ambari-agent/data ; "
- "sudo chmod 777 /var/lib/ambari-agent/data/tmp")
+ "sudo chmod 777 /var/lib/ambari-agent/tmp")
@patch.object(BootstrapDefault, "getOsCheckScript")
@patch.object(BootstrapDefault, "getOsCheckScriptRemoteLocation")
@@ -491,7 +491,7 @@ class TestBootstrap(TestCase):
command = str(init_mock.call_args[0][3])
self.assertEqual(command,
"chmod a+x OsCheckScriptRemoteLocation && "
- "env PYTHONPATH=$PYTHONPATH:/var/lib/ambari-agent/data/tmp OsCheckScriptRemoteLocation centos6")
+ "env PYTHONPATH=$PYTHONPATH:/var/lib/ambari-agent/tmp OsCheckScriptRemoteLocation centos6")
@patch.object(SSH, "__init__")