You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2015/01/06 20:00:07 UTC
ambari git commit: AMBARI-8795. MapReduce service components should
indicate security state (rlevas)
Repository: ambari
Updated Branches:
refs/heads/trunk 71def630e -> 25726ae17
AMBARI-8795. MapReduce service components should indicate security state (rlevas)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/25726ae1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/25726ae1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/25726ae1
Branch: refs/heads/trunk
Commit: 25726ae176919208a4290f7901d76c2b43720657
Parents: 71def63
Author: Robert Levas <rl...@hortonworks.com>
Authored: Tue Jan 6 13:58:53 2015 -0500
Committer: Robert Levas <rl...@hortonworks.com>
Committed: Tue Jan 6 13:58:53 2015 -0500
----------------------------------------------------------------------
.../YARN/package/scripts/historyserver.py | 57 ++++++++++++
.../YARN/package/scripts/status_params.py | 7 +-
.../stacks/2.0.6/YARN/test_historyserver.py | 93 ++++++++++++++++++++
3 files changed, 156 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/25726ae1/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/historyserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/historyserver.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/historyserver.py
index d2b6ee3..66c2143 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/historyserver.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/historyserver.py
@@ -23,6 +23,9 @@ from resource_management import *
from resource_management.libraries.functions.dynamic_variable_interpretation import copy_tarballs_to_hdfs
from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.security_commons import build_expectations, \
+ cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
+ FILE_TYPE_XML
from yarn import yarn
from service import service
@@ -62,5 +65,59 @@ class HistoryServer(Script):
env.set_params(status_params)
check_process_status(status_params.mapred_historyserver_pid_file)
+ def security_status(self, env):
+ import status_params
+ env.set_params(status_params)
+
+ expectations = {}
+ expectations.update(build_expectations('mapred-site',
+ None,
+ [
+ 'mapreduce.jobhistory.keytab',
+ 'mapreduce.jobhistory.principal',
+ 'mapreduce.jobhistory.webapp.spnego-keytab-file',
+ 'mapreduce.jobhistory.webapp.spnego-principal'
+ ],
+ None))
+
+ security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
+ {'mapred-site.xml': FILE_TYPE_XML})
+ result_issues = validate_security_config_properties(security_params, expectations)
+ if not result_issues: # If all validations passed successfully
+ try:
+ # Double check the dict before calling execute
+ if ( 'mapred-site' not in security_params or
+ 'mapreduce.jobhistory.keytab' not in security_params['mapred-site'] or
+ 'mapreduce.jobhistory.principal' not in security_params['mapred-site'] or
+ 'mapreduce.jobhistory.webapp.spnego-keytab-file' not in security_params['mapred-site'] or
+ 'mapreduce.jobhistory.webapp.spnego-principal' not in security_params['mapred-site']):
+ self.put_structured_out({"securityState": "UNSECURED"})
+ self.put_structured_out(
+ {"securityIssuesFound": "Keytab file or principal not set."})
+ return
+
+ cached_kinit_executor(status_params.kinit_path_local,
+ status_params.mapred_user,
+ security_params['mapred-site']['mapreduce.jobhistory.keytab'],
+ security_params['mapred-site']['mapreduce.jobhistory.principal'],
+ status_params.hostname,
+ status_params.tmp_dir)
+ cached_kinit_executor(status_params.kinit_path_local,
+ status_params.mapred_user,
+ security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-keytab-file'],
+ security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-principal'],
+ status_params.hostname,
+ status_params.tmp_dir)
+ self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+ except Exception as e:
+ self.put_structured_out({"securityState": "ERROR"})
+ self.put_structured_out({"securityStateErrorInfo": str(e)})
+ else:
+ issues = []
+ for cf in result_issues:
+ issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
+ self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
+ self.put_structured_out({"securityState": "UNSECURED"})
+
if __name__ == "__main__":
HistoryServer().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/25726ae1/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/status_params.py
index 498a885..bd4669b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/status_params.py
@@ -21,6 +21,7 @@ limitations under the License.
from resource_management import *
config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
mapred_user = config['configurations']['mapred-env']['mapred_user']
yarn_user = config['configurations']['yarn-env']['yarn_user']
@@ -33,4 +34,8 @@ resourcemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-resourcemanag
nodemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-nodemanager.pid")
yarn_historyserver_pid_file_old = format("{yarn_pid_dir}/yarn-{yarn_user}-historyserver.pid")
yarn_historyserver_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-timelineserver.pid") # *-historyserver.pid is deprecated
-mapred_historyserver_pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-historyserver.pid")
\ No newline at end of file
+mapred_historyserver_pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-historyserver.pid")
+
+hadoop_conf_dir = "/etc/hadoop/conf"
+hostname = config['hostname']
+kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
http://git-wip-us.apache.org/repos/asf/ambari/blob/25726ae1/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
index b759de6..b8f4b83 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
@@ -511,3 +511,96 @@ class TestHistoryServer(RMFTestCase):
owner = 'mapred',
group = 'hadoop',
)
+
+ @patch("resource_management.libraries.functions.security_commons.build_expectations")
+ @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem")
+ @patch("resource_management.libraries.functions.security_commons.validate_security_config_properties")
+ @patch("resource_management.libraries.functions.security_commons.cached_kinit_executor")
+ @patch("resource_management.libraries.script.Script.put_structured_out")
+ def test_security_status(self, put_structured_out_mock, cached_kinit_executor_mock, validate_security_config_mock, get_params_mock, build_exp_mock):
+ # Test that function works when is called with correct parameters
+
+ security_params = {
+ "mapred-site": {
+ 'mapreduce.jobhistory.keytab': "/path/to/keytab1",
+ 'mapreduce.jobhistory.principal': "principal1",
+ 'mapreduce.jobhistory.webapp.spnego-keytab-file': "/path/to/keytab2",
+ 'mapreduce.jobhistory.webapp.spnego-principal': "principal2"
+ }
+ }
+ result_issues = []
+
+ get_params_mock.return_value = security_params
+ validate_security_config_mock.return_value = result_issues
+
+ self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+ classname="HistoryServer",
+ command="security_status",
+ config_file="secured.json")
+
+ import status_params
+
+ get_params_mock.assert_called_with(status_params.hadoop_conf_dir, {'mapred-site.xml': 'XML'})
+ build_exp_mock.assert_called_with('mapred-site',
+ None,
+ [
+ 'mapreduce.jobhistory.keytab',
+ 'mapreduce.jobhistory.principal',
+ 'mapreduce.jobhistory.webapp.spnego-keytab-file',
+ 'mapreduce.jobhistory.webapp.spnego-principal'
+ ],
+ None)
+ put_structured_out_mock.assert_called_with({"securityState": "SECURED_KERBEROS"})
+ self.assertTrue(cached_kinit_executor_mock.call_count, 2)
+ cached_kinit_executor_mock.assert_called_with(status_params.kinit_path_local,
+ status_params.mapred_user,
+ security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-keytab-file'],
+ security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-principal'],
+ status_params.hostname,
+ status_params.tmp_dir)
+
+ # Testing that the exception throw by cached_executor is caught
+ cached_kinit_executor_mock.reset_mock()
+ cached_kinit_executor_mock.side_effect = Exception("Invalid command")
+
+ try:
+ self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+ classname="HistoryServer",
+ command="security_status",
+ config_file="secured.json")
+ except:
+ self.assertTrue(True)
+
+ # Testing with a security_params which doesn't contain mapred-site
+ empty_security_params = {}
+ cached_kinit_executor_mock.reset_mock()
+ get_params_mock.reset_mock()
+ put_structured_out_mock.reset_mock()
+ get_params_mock.return_value = empty_security_params
+
+ self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+ classname="HistoryServer",
+ command="security_status",
+ config_file="secured.json")
+ put_structured_out_mock.assert_called_with({"securityIssuesFound": "Keytab file or principal not set."})
+
+ # Testing with not empty result_issues
+ result_issues_with_params = {'mapred-site': "Something bad happened"}
+
+ validate_security_config_mock.reset_mock()
+ get_params_mock.reset_mock()
+ validate_security_config_mock.return_value = result_issues_with_params
+ get_params_mock.return_value = security_params
+
+ self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+ classname="HistoryServer",
+ command="security_status",
+ config_file="secured.json")
+ put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
+
+ # Testing with security_enable = false
+ self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+ classname="HistoryServer",
+ command="security_status",
+ config_file="default.json")
+ put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})