You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2019/04/17 18:47:08 UTC
[metron] branch master updated: METRON-2074 Script to handle TGT
renewal with Storm and Kerberos enabled (mmiklavc) closes
apache/metron#1382
This is an automated email from the ASF dual-hosted git repository.
mmiklavcic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new b6d8cad METRON-2074 Script to handle TGT renewal with Storm and Kerberos enabled (mmiklavc) closes apache/metron#1382
b6d8cad is described below
commit b6d8cade6aec7fe6290d4fd5660ff205de06bd64
Author: mmiklavc <mi...@gmail.com>
AuthorDate: Wed Apr 17 12:09:14 2019 -0600
METRON-2074 Script to handle TGT renewal with Storm and Kerberos enabled (mmiklavc) closes apache/metron#1382
---
metron-deployment/Kerberos-manual-setup.md | 45 +++++++++++
.../packaging/docker/rpm-docker/SPECS/metron.spec | 1 +
.../metron-common/src/main/scripts/tgt_renew.py | 90 ++++++++++++++++++++++
3 files changed, 136 insertions(+)
diff --git a/metron-deployment/Kerberos-manual-setup.md b/metron-deployment/Kerberos-manual-setup.md
index ecb0cb7..d50da34 100644
--- a/metron-deployment/Kerberos-manual-setup.md
+++ b/metron-deployment/Kerberos-manual-setup.md
@@ -31,6 +31,7 @@ This document provides instructions for kerberizing Metron's Vagrant-based devel
* [Push Data](#push-data)
* [More Information](#more-information)
* [Elasticseach X-Pack](#X-Pack)
+* [TGT Ticket Renew](#tgt-ticket-renew)
Setup
-----
@@ -636,3 +637,47 @@ The random access indexer topology fails with the following exception. This exc
#### Solution
This can occur when an HDFS Client is not installed on the Storm worker nodes. This might occur on any Storm worker node where an HDFS Client is not installed. Installing the HDFS Client on all Storm worker nodes should resolve the problem.
+
+## TGT Ticket Renew
+
+Apache Storm doesn't handle automatic TGT ticket renewal for their running topologies. Instead, it is left up to the operations team deploying the Storm topologies
+in a Kerberized environment to manage this themselves. We've included a Python script that can be setup with a cron process to automatically manage the renewal
+process for you. The script should be run on an interval that is shorter than the renew_lifetime configured for your TGT.
+
+### Setup Instructions
+
+Run the following on a node with a Storm and Metron client installed. We need python 2.7 via virtualenv for this to work correctly.
+
+```
+# run yum commands as root
+for item in epel-release centos-release-scl "@Development tools" python27 python27-scldevel python27-python-virtualenv libselinux-python; do yum install -y $item; done
+sudo yum install -y gcc krb5-devel python-devel
+sudo yum install -y libffi libffi-devel
+sudo yum install -y python-cffi
+sudo yum install -y openssl-devel
+# setup python with metron user
+su - metron
+export PYTHON27_HOME=/opt/rh/python27/root
+export LD_LIBRARY_PATH="/opt/rh/python27/root/usr/lib64"
+mkdir project_dir
+cd project_dir
+${PYTHON27_HOME}/usr/bin/virtualenv venv
+source venv/bin/activate
+pip install --upgrade setuptools==18.5
+pip install requests-kerberos
+```
+
+The script `$METRON_HOME/bin/tgt_renew.py` takes two arguments:
+
+* arg1 = host:port for Storm UI server
+* arg2 = topology owner - typically "metron" for a kerberized cluster with metron topologies
+
+Execute it like the following example:
+
+```
+# run as the metron user
+su - metron
+python $METRON_HOME/bin/tgt_renew.py node1:8744 metron
+```
+
+
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index fbffe28..dc40967 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -136,6 +136,7 @@ This package installs the Metron common files %{metron_home}
%{metron_home}/bin/zk_load_configs.sh
%{metron_home}/bin/stellar
%{metron_home}/bin/cluster_info.py
+%{metron_home}/bin/tgt_renew.py
%{metron_home}/config/zookeeper/global.json
%attr(0644,root,root) %{metron_home}/lib/metron-common-%{full_version}.jar
diff --git a/metron-platform/metron-common/src/main/scripts/tgt_renew.py b/metron-platform/metron-common/src/main/scripts/tgt_renew.py
new file mode 100755
index 0000000..b0a8d0f
--- /dev/null
+++ b/metron-platform/metron-common/src/main/scripts/tgt_renew.py
@@ -0,0 +1,90 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from requests_kerberos import HTTPKerberosAuth
+from contextlib import closing
+import json
+import requests
+import subprocess
+import sys
+
+INDENT_SIZE = 2
+
+class ShellHandler(object):
+
+ def __init__(self):
+ pass
+
+ # returns full stdout of process call
+ def call(self, command):
+ try:
+ return subprocess.call(command, shell=True)
+ except OSError as e:
+ print >> sys.stderr, "Execution failed:", e
+
+ # partly hijacked from Python 2.7+ check_output for use in 2.6
+ def ret_output(self, cmd):
+ process = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
+ output, unused_err = process.communicate()
+ retcode = process.poll()
+ if retcode:
+ raise subprocess.CalledProcessError(retcode, cmd, output=output)
+ return output
+
+def get_topologies(storm_ui_host, requested_owner):
+ print "Connecting to Storm via " + storm_ui_host
+ response = requests.get(storm_ui_host, auth=HTTPKerberosAuth())
+ topology_ids = []
+ if response.status_code == 200:
+ print "Connected to server ok, parsing results"
+ for feature, value in response.json().iteritems():
+ if feature == 'topologies':
+ for topology in value:
+ t_id = ''
+ t_owner = ''
+ for k, v in topology.iteritems():
+ if k == 'name':
+ t_id = v
+ if k == 'owner':
+ t_owner = v
+ if t_owner == requested_owner:
+ topology_ids.append(t_id)
+ else:
+ print "Request for topologies failed with status code: " + str(response.status_code)
+ print "Finished. Found {0} topologies for user '{1}'".format(len(topology_ids), requested_owner)
+ return topology_ids
+
+def upload_credentials(topologies):
+ if len(topologies) == 0:
+ print "No topologies running for specified username/owner. Exiting"
+ return
+ storm_cmd_template = "/usr/hdp/current/storm-client/bin/storm -c topology.auto-credentials=\'[\"org.apache.storm.security.auth.kerberos.AutoTGT\"]\' upload-credentials $topology_name"
+ for t in topologies:
+ cmd = storm_cmd_template.replace("$topology_name", t)
+ print "Running renew-credentials command: " + cmd
+ ShellHandler().call(cmd)
+
+if len(sys.argv) != 3:
+ raise ValueError("Need host info 'host:port' for Storm and topology owner name")
+
+host_info = sys.argv[1]
+requested_owner = sys.argv[2]
+storm_ui_host = 'http://{0}/api/v1/topology/summary'.format(host_info)
+topologies = get_topologies(storm_ui_host, requested_owner)
+upload_credentials(topologies)
+