You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2015/05/14 21:40:57 UTC
ambari git commit: AMBARI-10884. Addition of security to Atlas
service definition (Jon Maron via smohanty)
Repository: ambari
Updated Branches:
refs/heads/trunk 2437a799a -> af675f47d
AMBARI-10884. Addition of security to Atlas service definition (Jon Maron via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/af675f47
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/af675f47
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/af675f47
Branch: refs/heads/trunk
Commit: af675f47df28dec850cd603ee6ac2520bf444d1c
Parents: 2437a79
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu May 14 12:40:42 2015 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu May 14 12:40:42 2015 -0700
----------------------------------------------------------------------
.../common-services/ATLAS/0.1.0.2.3/alerts.json | 9 +-
.../ATLAS/0.1.0.2.3/kerberos.json | 26 ++--
.../ATLAS/0.1.0.2.3/package/scripts/metadata.py | 9 ++
.../package/scripts/metadata_server.py | 73 +++++++++--
.../ATLAS/0.1.0.2.3/package/scripts/params.py | 19 ++-
.../0.1.0.2.3/package/scripts/service_check.py | 35 ++----
.../0.1.0.2.3/package/scripts/status_params.py | 3 +-
.../stacks/2.3/ATLAS/test_metadata_server.py | 120 ++++++++++---------
8 files changed, 184 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/af675f47/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/alerts.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/alerts.json b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/alerts.json
index 7202950..5c4c06e 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/alerts.json
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/alerts.json
@@ -37,10 +37,13 @@
"source": {
"type": "WEB",
"uri": {
- "http": "{{metadata-env/metadata_port}}",
+ "http": "{{hostname}}:{{metadata-env/metadata_port}}",
+ "https": "{{hostname}}:21443",
+ "https_property": "{{application-properties/enableTLS}}",
+ "https_property_value": "true",
"default_port": 21000,
- "kerberos_keytab": "{{metadata-runtime.properties/*.metadata.http.authentication.keytab}}",
- "kerberos_principal": "{{metadata-runtime.properties/*.metadata.http.authentication.principal}}"
+ "kerberos_keytab": "{{application-properties/http_authentication_kerberos_keytab}}",
+ "kerberos_principal": "{{application-properties/http_authentication_kerberos_principal}}"
},
"reporting": {
"ok": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/af675f47/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/kerberos.json b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/kerberos.json
index 210f414..b57f565 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/kerberos.json
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/kerberos.json
@@ -2,30 +2,28 @@
"services": [
{
"name": "ATLAS",
- "identities": [
- {
- "name": "/spnego"
- }
- ],
"configurations": [
{
- "metadata-runtime.properties": {
- "*.metadata.authentication.type": "kerberos",
- "*.metadata.http.authentication.type": "kerberos",
- "*.dfs.namenode.kerberos.principal": "nn/_HOST@${realm}"
+ "application-properties": {
+ "authentication_method": "kerberos",
+ "http_authentication_enabled": "true",
+ "http_authentication_type": "kerberos"
}
}
],
+ "auth_to_local_properties" : [
+ "application-properties/http_authentication_kerberos_name_rules"
+ ],
"components": [
{
"name": "ATLAS_SERVER",
"identities": [
{
- "name": "metadata_server",
+ "name": "atlas",
"principal": {
"value": "atlas/_HOST@${realm}",
"type" : "service",
- "configuration": "application-properties/*.metadata.authentication.principal",
+ "configuration": "application-properties/authentication_principal",
"local_username" : "${metadata-env/metadata_user}"
},
"keytab": {
@@ -38,17 +36,17 @@
"name": "${cluster-env/user_group}",
"access": ""
},
- "configuration": "application-properties/*.metadata.authentication.keytab"
+ "configuration": "application-properties/authentication_keytab"
}
},
{
"name": "/spnego",
"principal": {
"value": "HTTP/_HOST@${realm}",
- "configuration": "application-properties/*.metadata.http.authentication.principal"
+ "configuration": "application-properties/http_authentication_kerberos_principal"
},
"keytab": {
- "configuration": "application-properties/*.metadata.http.authentication.keytab"
+ "configuration": "application-properties/http_authentication_kerberos_keytab"
}
}
]
http://git-wip-us.apache.org/repos/asf/ambari/blob/af675f47/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
index 61cdec4..a4274ca 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
@@ -17,6 +17,8 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
+import os
+import shutil
from resource_management import Directory, Fail, Logger, File, \
InlineTemplate, StaticFile
from resource_management.libraries.functions import format
@@ -65,6 +67,13 @@ def metadata():
recursive=True
)
+ metadata_war_file = format('{params.metadata_home}/server/webapp/metadata.war')
+ if not os.path.isfile(metadata_war_file):
+ raise Fail("Unable to copy {0} because it does not exist".format(metadata_war_file))
+
+ Logger.info("Copying {0} to {1}".format(metadata_war_file, params.expanded_war_dir))
+ shutil.copy2(metadata_war_file, params.expanded_war_dir)
+
File(format('{conf_dir}/application.properties'),
content=InlineTemplate(params.application_properties_content),
mode=0644,
http://git-wip-us.apache.org/repos/asf/ambari/blob/af675f47/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
index 2484315..c07961d 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
@@ -16,13 +16,12 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
-import os
-import shutil
-
from metadata import metadata
-from resource_management import Execute, check_process_status, Script, Fail, \
- Logger
+from resource_management import Execute, check_process_status, Script
from resource_management.libraries.functions import format
+from resource_management.libraries.functions.security_commons import build_expectations, \
+ get_params_from_filesystem, validate_security_config_properties, \
+ FILE_TYPE_PROPERTIES
class MetadataServer(Script):
@@ -31,7 +30,6 @@ class MetadataServer(Script):
def install(self, env):
self.install_packages(env)
- self.configure(env)
def configure(self, env):
import params
@@ -46,13 +44,7 @@ class MetadataServer(Script):
def start(self, env, rolling_restart=False):
import params
env.set_params(params)
-
- metadata_war_file = format('{params.metadata_home}/server/webapp/metadata.war')
- if not os.path.isfile(metadata_war_file):
- raise Fail("Unable to copy {0} because it does not exist".format(metadata_war_file))
-
- Logger.info("Copying {0} to {1}".format(metadata_war_file, params.expanded_war_dir))
- shutil.copy2(metadata_war_file, params.expanded_war_dir)
+ self.configure(env)
daemon_cmd = format('source {params.conf_dir}/metadata-env.sh ; {params.metadata_start_script} --port {params.metadata_port}')
no_op_test = format('ls {params.pid_file} >/dev/null 2>&1 && ps -p `cat {params.pid_file}` >/dev/null 2>&1')
@@ -76,5 +68,60 @@ class MetadataServer(Script):
env.set_params(status_params)
check_process_status(status_params.pid_file)
+ def security_status(self, env):
+ import status_params
+
+ env.set_params(status_params)
+
+ props_value_check = {'metadata.authentication.method': 'kerberos',
+ 'metadata.http.authentication.enabled': 'true',
+ 'metadata.http.authentication.type': 'kerberos'}
+ props_empty_check = ['metadata.authentication.principal',
+ 'metadata.authentication.keytab',
+ 'metadata.http.authentication.kerberos.principal',
+ 'metadata.http.authentication.kerberos.keytab']
+ props_read_check = ['metadata.authentication.keytab',
+ 'metadata.http.authentication.kerberos.keytab']
+ atlas_site_expectations = build_expectations('application-properties',
+ props_value_check,
+ props_empty_check,
+ props_read_check)
+
+ atlas_expectations = {}
+ atlas_expectations.update(atlas_site_expectations)
+
+ security_params = get_params_from_filesystem(status_params.conf_dir,
+ {'application.properties': FILE_TYPE_PROPERTIES})
+ result_issues = validate_security_config_properties(security_params, atlas_expectations)
+ if not result_issues: # If all validations passed successfully
+ try:
+ # Double check the dict before calling execute
+ if ( 'application-properties' not in security_params
+ or 'metadata.authentication.keytab' not in security_params['application-properties']
+ or 'metadata.authentication.principal' not in security_params['application-properties']):
+ self.put_structured_out({"securityState": "UNSECURED"})
+ self.put_structured_out(
+ {"securityIssuesFound": "Atlas service keytab file or principal are not set property."})
+ return
+
+ if ( 'application-properties' not in security_params
+ or 'metadata.http.authentication.kerberos.keytab' not in security_params['application-properties']
+ or 'metadata.http.authentication.kerberos.principal' not in security_params['application-properties']):
+ self.put_structured_out({"securityState": "UNSECURED"})
+ self.put_structured_out(
+ {"securityIssuesFound": "HTTP Authentication keytab file or principal are not set property."})
+ return
+
+ self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+ except Exception as e:
+ self.put_structured_out({"securityState": "ERROR"})
+ self.put_structured_out({"securityStateErrorInfo": str(e)})
+ else:
+ issues = []
+ for cf in result_issues:
+ issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
+ self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
+ self.put_structured_out({"securityState": "UNSECURED"})
+
if __name__ == "__main__":
MetadataServer().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/af675f47/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
index 8401a5d..577c5ac 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
@@ -69,6 +69,9 @@ metadata_host = config['hostname']
application_properties = config['configurations']['application-properties']
for key, value in application_properties.iteritems():
+ # fix the multi-line property
+ if (key == 'http_authentication_kerberos_name_rules'):
+ value = ' \\ \n'.join(value.splitlines())
globals()[key] = value
metadata_env_content = config['configurations']['metadata-env']['content']
@@ -77,4 +80,18 @@ application_properties_content = config['configurations']['application-propertie
metadata_opts = config['configurations']['metadata-env']['metadata_opts']
metadata_classpath = config['configurations']['metadata-env']['metadata_classpath']
data_dir = config['configurations']['metadata-env']['metadata_data_dir']
-expanded_war_dir = os.environ['METADATA_EXPANDED_WEBAPP_DIR'] if 'METADATA_EXPANDED_WEBAPP_DIR' in os.environ else '/var/lib/atlas/server/webapp'
\ No newline at end of file
+expanded_war_dir = os.environ['METADATA_EXPANDED_WEBAPP_DIR'] if 'METADATA_EXPANDED_WEBAPP_DIR' in os.environ else '/var/lib/atlas/server/webapp'
+
+# smoke test
+smoke_test_user = config['configurations']['cluster-env']['smokeuser']
+smoke_test_password = 'smoke'
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+smokeuser_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+
+kinit_path_local = status_params.kinit_path_local
+
+security_check_status_file = format('{log_dir}/security_check.status')
+if security_enabled:
+ smoke_cmd = format('curl --negotiate -u : -b ~/cookiejar.txt -c ~/cookiejar.txt -s -o /dev/null -w "%{{http_code}}" http://{metadata_host}:{metadata_port}/')
+else:
+ smoke_cmd = format('curl -s -o /dev/null -w "%{{http_code}}" http://{metadata_host}:{metadata_port}/')
http://git-wip-us.apache.org/repos/asf/ambari/blob/af675f47/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/service_check.py
index 1ecb795..194a3fb 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/service_check.py
@@ -16,42 +16,29 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
-import httplib
-import socket
-import time
-from resource_management import Script, Logger, ComponentIsNotRunning, Fail
+from resource_management import Script, Logger, ComponentIsNotRunning, \
+ Execute
from resource_management.libraries.functions import format
class AtlasServiceCheck(Script):
- ATLAS_CONNECT_TRIES = 5
- ATLAS_CONNECT_TIMEOUT = 10
def service_check(self, env):
import params
env.set_params(params)
- for i in xrange(0, self.ATLAS_CONNECT_TRIES):
- try:
- conn = httplib.HTTPConnection(params.metadata_host,
- int(params.metadata_port))
- conn.request("GET", format("http://{params.metadata_host}:{params.metadata_port}/"))
- except (httplib.HTTPException, socket.error) as ex:
- if i < self.ATLAS_CONNECT_TRIES - 1:
- time.sleep(self.ATLAS_CONNECT_TIMEOUT)
- Logger.info("Connection failed. Next retry in %s seconds."
- % (self.ATLAS_CONNECT_TIMEOUT))
- continue
- else:
- raise Fail("Service check has failed.")
-
- resp = conn.getresponse()
- if resp.status == 200 :
+ if params.security_enabled:
+ Execute(format("{kinit_path_local} -kt {smokeuser_keytab} {smokeuser_principal}"),
+ user=params.metadata_user)
+
+ try:
+ Execute(params.smoke_cmd, user=params.metadata_user, tries = 5,
+ try_sleep = 10)
Logger.info('Atlas server up and running')
- else:
+ except:
Logger.debug('Atlas server not running')
- raise ComponentIsNotRunning()
+
if __name__ == "__main__":
AtlasServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/af675f47/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
index aa7f614..db295c4 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
@@ -20,6 +20,7 @@ limitations under the License.
import os
from resource_management import Script
from resource_management.libraries.functions import get_kinit_path, format
+from resource_management.libraries.functions.default import default
config = Script.get_config()
@@ -32,5 +33,5 @@ metadata_user = config['configurations']['metadata-env']['metadata_user']
# Security related/required params
hostname = config['hostname']
security_enabled = config['configurations']['cluster-env']['security_enabled']
-kinit_path_local = get_kinit_path()
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
tmp_dir = Script.get_tmp_dir()
http://git-wip-us.apache.org/repos/asf/ambari/blob/af675f47/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
index 6f1377b..9db21f8 100644
--- a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
+++ b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
@@ -28,7 +28,70 @@ class TestMetadataServer(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "ATLAS/0.1.0.2.3/package"
STACK_VERSION = "2.3"
- def test_configure_default(self):
+ def configureResourcesCalled(self):
+ self.assertResourceCalled('Directory', '/var/run/atlas',
+ owner='atlas',
+ group='hadoop',
+ recursive=True,
+ cd_access='a',
+ mode=0755
+ )
+ self.assertResourceCalled('Directory', '/etc/atlas/conf',
+ owner='atlas',
+ group='hadoop',
+ recursive=True,
+ cd_access='a',
+ mode=0755
+ )
+ self.assertResourceCalled('Directory', '/var/log/atlas',
+ owner='atlas',
+ group='hadoop',
+ recursive=True,
+ cd_access='a',
+ mode=0755
+ )
+ self.assertResourceCalled('Directory', '/var/lib/atlas/data',
+ owner='atlas',
+ group='hadoop',
+ recursive=True,
+ cd_access='a',
+ mode=0644
+ )
+ self.assertResourceCalled('Directory', '/var/lib/atlas/server/webapp',
+ owner='atlas',
+ group='hadoop',
+ recursive=True,
+ cd_access='a',
+ mode=0644
+ )
+ self.assertResourceCalled('File',
+ '/etc/atlas/conf/application.properties',
+ content=InlineTemplate(
+ self.getConfig()['configurations'][
+ 'application-properties']['content']),
+ owner='atlas',
+ group='hadoop',
+ mode=0644,
+ )
+ self.assertResourceCalled('File', '/etc/atlas/conf/metadata-env.sh',
+ content=InlineTemplate(
+ self.getConfig()['configurations'][
+ 'metadata-env']['content']),
+ owner='atlas',
+ group='hadoop',
+ mode=0755,
+ )
+ self.assertResourceCalled('File', '/etc/atlas/conf/log4j.xml',
+ content=StaticFile('log4j.xml'),
+ owner='atlas',
+ group='hadoop',
+ mode=0644,
+ )
+
+ @patch("shutil.copy2", new = MagicMock())
+ @patch("os.path.isfile")
+ def test_configure_default(self, isfile_mock):
+ isfile_mock.return_value = True
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metadata_server.py",
classname = "MetadataServer",
@@ -38,59 +101,7 @@ class TestMetadataServer(RMFTestCase):
target = RMFTestCase.TARGET_COMMON_SERVICES
)
- self.assertResourceCalled('Directory', '/var/run/atlas',
- owner = 'atlas',
- group = 'hadoop',
- recursive = True,
- cd_access = 'a',
- mode=0755
- )
- self.assertResourceCalled('Directory', '/etc/atlas/conf',
- owner = 'atlas',
- group = 'hadoop',
- recursive = True,
- cd_access = 'a',
- mode=0755
- )
- self.assertResourceCalled('Directory', '/var/log/atlas',
- owner = 'atlas',
- group = 'hadoop',
- recursive = True,
- cd_access = 'a',
- mode=0755
- )
- self.assertResourceCalled('Directory', '/var/lib/atlas/data',
- owner = 'atlas',
- group = 'hadoop',
- recursive = True,
- cd_access = 'a',
- mode=0644
- )
- self.assertResourceCalled('Directory', '/var/lib/atlas/server/webapp',
- owner = 'atlas',
- group = 'hadoop',
- recursive = True,
- cd_access = 'a',
- mode=0644
- )
- self.assertResourceCalled('File', '/etc/atlas/conf/application.properties',
- content = InlineTemplate(self.getConfig()['configurations']['application-properties']['content']),
- owner = 'atlas',
- group = 'hadoop',
- mode = 0644,
- )
- self.assertResourceCalled('File', '/etc/atlas/conf/metadata-env.sh',
- content = InlineTemplate(self.getConfig()['configurations']['metadata-env']['content']),
- owner = 'atlas',
- group = 'hadoop',
- mode = 0755,
- )
- self.assertResourceCalled('File', '/etc/atlas/conf/log4j.xml',
- content = StaticFile('log4j.xml'),
- owner = 'atlas',
- group = 'hadoop',
- mode = 0644,
- )
+ self.configureResourcesCalled()
self.assertNoMoreResources()
@patch("shutil.copy2", new = MagicMock())
@@ -105,6 +116,7 @@ class TestMetadataServer(RMFTestCase):
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
+ self.configureResourcesCalled()
self.assertResourceCalled('Execute', 'source /etc/atlas/conf/metadata-env.sh ; /usr/hdp/current/atlas-server/bin/metadata_start.py --port 21000',
not_if = 'ls /var/run/atlas/metadata.pid >/dev/null 2>&1 && ps -p `cat /var/run/atlas/metadata.pid` >/dev/null 2>&1',
user = 'atlas',