You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by le...@apache.org on 2017/04/20 20:07:50 UTC
[2/3] incubator-metron git commit: METRON-799 The MPack should
function in a kerberized cluster (dlyle via justinleet) closes
apache/incubator-metron#518
METRON-799 The MPack should function in a kerberized cluster (dlyle via justinleet) closes apache/incubator-metron#518
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/55062fb7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/55062fb7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/55062fb7
Branch: refs/heads/master
Commit: 55062fb79a85722c909906874e0772f51b18953d
Parents: 187ef37
Author: David Lyle <dl...@gmail.com>
Authored: Thu Apr 13 11:23:40 2017 -0400
Committer: leet <le...@apache.org>
Committed: Thu Apr 20 16:03:05 2017 -0400
----------------------------------------------------------------------
.../METRON/CURRENT/configuration/metron-env.xml | 19 ++++----
.../METRON/CURRENT/kerberos.json | 22 ++++++++-
.../common-services/METRON/CURRENT/metainfo.xml | 14 ++++++
.../package/scripts/enrichment_commands.py | 12 -----
.../package/scripts/enrichment_master.py | 25 +++++++++-
.../CURRENT/package/scripts/metron_client.py | 49 ++++++++++++++++++++
.../CURRENT/package/scripts/metron_security.py | 9 ++++
.../CURRENT/package/scripts/metron_service.py | 23 +++++++--
.../package/scripts/params/status_params.py | 5 +-
.../METRON/CURRENT/service_advisor.py | 5 --
10 files changed, 145 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
index 334e4f3..d09614b 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -128,21 +128,20 @@
<name>metron_principal_name</name>
<display-name>Metron Principal Name</display-name>
<description>Metron principal name</description>
- </property>
- <property>
- <name>metron_keytab</name>
- <display-name>Metron Keytab</display-name>
- <description>Metron keytab path</description>
- </property>
- <property>
- <name>metron_service_principal_name</name>
- <display-name>Metron Service Principal Name</display-name>
- <description>Metron service principal name</description>
+ <property-type>KERBEROS_PRINCIPAL</property-type>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
</property>
<property>
<name>metron_service_keytab</name>
<display-name>Metron Service User Keytab</display-name>
<description>Metron Service user keytab path</description>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
</property>
<property>
<name>global-json</name>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/kerberos.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/kerberos.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/kerberos.json
index e1a847b..cba1847 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/kerberos.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/kerberos.json
@@ -21,11 +21,29 @@
"name": "${cluster-env/user_group}",
"access": "r"
},
- "configuration": "metron-env/metron_keytab"
+ "configuration": "metron-env/metron_service_keytab"
}
}
+ ],
+ "configurations": [
+ {
+ "storm-site": {
+ "nimbus.credential.renewers.classes": "['org.apache.storm.security.auth.kerberos.AutoTGT']",
+ "supervisor.run.worker.as.user": "true"
+ }
+ }
+ ],
+ "components": [
+ {
+ "name": "METRON_PARSERS"
+ },
+ {
+ "name": "METRON_ENRICHMENT_MASTER"
+ },
+ {
+ "name": "METRON_INDEXING"
+ }
]
}
]
}
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index bbe7187..25237f5 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -174,6 +174,20 @@
</customCommand>
</customCommands>
</component>
+
+ <component>
+ <name>METRON_CLIENT</name>
+ <displayName>METRON Client</displayName>
+ <category>CLIENT</category>
+ <cardinality>1+</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <commandScript>
+ <script>scripts/metron_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ </component>
+
</components>
<themes>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
index aa53391..4fda07b 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
@@ -21,8 +21,6 @@ from resource_management.core.logger import Logger
from resource_management.core.resources.system import Execute, File
import metron_service
-import metron_security
-
# Wrap major operations and functionality in this class
class EnrichmentCommands:
@@ -143,11 +141,6 @@ class EnrichmentCommands:
-s {1} \
-z {2}"""
Logger.info('Starting ' + self.__enrichment_topology)
- if self.__params.security_enabled:
- metron_security.kinit(self.__params.kinit_path_local,
- self.__params.metron_keytab_path,
- self.__params.metron_principal_name,
- execute_user=self.__params.metron_user)
Execute(start_cmd_template.format(self.__params.metron_home, self.__enrichment_topology, self.__params.zookeeper_quorum),
user=self.__params.metron_user)
@@ -156,11 +149,6 @@ class EnrichmentCommands:
def stop_enrichment_topology(self):
Logger.info('Stopping ' + self.__enrichment_topology)
stop_cmd = 'storm kill ' + self.__enrichment_topology
- if self.__params.security_enabled:
- metron_security.kinit(self.__params.kinit_path_local,
- self.__params.metron_keytab_path,
- self.__params.metron_principal_name,
- execute_user=self.__params.metron_user)
Execute(stop_cmd, user=self.__params.metron_user)
Logger.info('Done stopping enrichment topologies')
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
index e8cb8d5..4ab81d4 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
@@ -24,14 +24,13 @@ from resource_management.core.logger import Logger
from enrichment_commands import EnrichmentCommands
from metron_security import storm_security_setup
import metron_service
-
+import metron_security
class Enrichment(Script):
def install(self, env):
from params import params
env.set_params(params)
self.install_packages(env)
- self.configure(env)
def configure(self, env, upgrade_type=None, config_dir=None):
from params import params
@@ -52,6 +51,13 @@ class Enrichment(Script):
env.set_params(params)
self.configure(env)
commands = EnrichmentCommands(params)
+
+ if params.security_enabled:
+ metron_security.kinit(params.kinit_path_local,
+ params.metron_keytab_path,
+ params.metron_principal_name,
+ execute_user=params.metron_user)
+
metron_service.load_global_config(params)
if not commands.is_kafka_configured():
@@ -69,8 +75,16 @@ class Enrichment(Script):
def stop(self, env, upgrade_type=None):
from params import params
+
env.set_params(params)
commands = EnrichmentCommands(params)
+
+ if params.security_enabled:
+ metron_security.kinit(params.kinit_path_local,
+ params.metron_keytab_path,
+ params.metron_principal_name,
+ execute_user=params.metron_user)
+
commands.stop_enrichment_topology()
def status(self, env):
@@ -78,6 +92,13 @@ class Enrichment(Script):
env.set_params(status_params)
commands = EnrichmentCommands(status_params)
+ if status_params.security_enabled:
+ metron_security.kinit(status_params.kinit_path_local,
+ status_params.metron_keytab_path,
+ status_params.metron_principal_name,
+ execute_user=status_params.metron_user)
+
+
if not commands.is_topology_active(env):
raise ComponentIsNotRunning()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_client.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_client.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_client.py
new file mode 100644
index 0000000..65831c9
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_client.py
@@ -0,0 +1,49 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script.script import Script
+from resource_management.core.exceptions import ClientComponentHasNoStatus
+from resource_management.core.resources.system import Directory
+from metron_security import storm_security_setup
+
+class MetronClient(Script):
+
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+
+ def configure(self, env):
+ from params import params
+ env.set_params(params)
+ storm_security_setup(params)
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+
+ def status(self, env):
+ raise ClientComponentHasNoStatus()
+
+if __name__ == "__main__":
+ MetronClient().execute()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py
index 057339a..4f04daf 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py
@@ -30,6 +30,15 @@ def storm_security_setup(params):
# Ambari's Directory doesn't do tilde expansion.
metron_storm_dir_tilde = '~' + params.metron_user + '/.storm'
metron_storm_dir = os.path.expanduser(metron_storm_dir_tilde)
+
+
+ Directory(params.metron_home,
+ mode=0755,
+ owner=params.metron_user,
+ group=params.metron_group,
+ create_parents=True
+ )
+
Directory(metron_storm_dir,
mode=0755,
owner=params.metron_user,
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
index b213947..f2cb88d 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
@@ -22,6 +22,7 @@ from resource_management.core.resources.system import Directory, File
from resource_management.core.resources.system import Execute
from resource_management.core.source import InlineTemplate
from resource_management.libraries.functions import format as ambari_format
+from resource_management.libraries.functions.get_user_call_output import get_user_call_output
from metron_security import kinit
def init_config():
@@ -40,8 +41,7 @@ def get_running_topologies(params):
# Want to sudo to the metron user and kinit as them so we aren't polluting root with Metron's Kerberos tickets.
# This is becuase we need to run a command with a return as the metron user. Sigh
negotiate = '--negotiate -u : ' if params.security_enabled else ''
- sudo = ambari_format('sudo -u {metron_user} ') if params.security_enabled else ''
- cmd = ambari_format(sudo + 'curl --max-time 3 ' + negotiate + '{storm_rest_addr}/api/v1/topology/summary')
+ cmd = ambari_format('curl --max-time 3 ' + negotiate + '{storm_rest_addr}/api/v1/topology/summary')
if params.security_enabled:
kinit(params.kinit_path_local,
@@ -50,8 +50,23 @@ def get_running_topologies(params):
execute_user=params.metron_user)
Logger.info('Running cmd: ' + cmd)
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
- (stdout, stderr) = proc.communicate()
+ return_code, stdout, sdterr = get_user_call_output(cmd, user=params.metron_user)
+
+ try:
+ stormjson = json.loads(stdout)
+ except ValueError, e:
+ Logger.info('Stdout: ' + str(stdout))
+ Logger.info('Stderr: ' + str(stderr))
+ Logger.exception(str(e))
+ return {}
+
+ topologiesDict = {}
+
+ for topology in stormjson['topologies']:
+ topologiesDict[topology['name']] = topology['status']
+
+ Logger.info("Topologies: " + str(topologiesDict))
+ return topologiesDict
try:
stormjson = json.loads(stdout)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index 65e1eaf..154b3de 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -73,6 +73,5 @@ tmp_dir = Script.get_tmp_dir()
metron_user = config['configurations']['metron-env']['metron_user']
-if security_enabled:
- metron_principal_name = config['configurations']['metron-env']['metron_principal_name']
- metron_keytab_path = config['configurations']['metron-env']['metron_keytab']
+metron_principal_name = config['configurations']['metron-env']['metron_principal_name']
+metron_keytab_path = config['configurations']['metron-env']['metron_service_keytab']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
index c16e3e6..23aebb4 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
@@ -156,10 +156,5 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor
storm_site_desired_values = {
"topology.classpath" : "/etc/hbase/conf:/etc/hadoop/conf"
}
- if is_secured:
- storm_site_desired_values.update({
- "nimbus.credential.renewers.classes":"['org.apache.storm.security.auth.kerberos.AutoTGT']",
- "supervisor.run.worker.as.user":"true"
- })
return storm_site_desired_values