You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2018/07/11 01:32:35 UTC
[19/50] [abbrv] metron git commit: METRON-1587 Make collection
utility work for HDP search (merrimanr) closes apache/metron#1043
METRON-1587 Make collection utility work for HDP search (merrimanr) closes apache/metron#1043
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f241f87f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f241f87f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f241f87f
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: f241f87f87523f2c6f2203b3f316cff0fd3f20b4
Parents: aed4ffc
Author: merrimanr <me...@gmail.com>
Authored: Mon Jun 11 10:09:16 2018 -0500
Committer: merrimanr <me...@gmail.com>
Committed: Mon Jun 11 10:09:16 2018 -0500
----------------------------------------------------------------------
.../package/scripts/indexing_commands.py | 63 +++++++++++++++++---
.../CURRENT/package/scripts/indexing_master.py | 31 +---------
.../package/scripts/params/params_linux.py | 5 ++
metron-platform/metron-solr/README.md | 33 +++++++++-
.../src/main/scripts/create_collection.sh | 21 +++++--
.../src/main/scripts/delete_collection.sh | 18 ++++--
6 files changed, 120 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index eeb2127..69e980b 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -25,6 +25,7 @@ from resource_management.core.exceptions import Fail
from resource_management.core.logger import Logger
from resource_management.core.resources.system import Execute, File
from resource_management.libraries.functions import format as ambari_format
+from resource_management.libraries.functions.format import format
import metron_service
import metron_security
@@ -85,14 +86,13 @@ class IndexingCommands:
:return: Dict where key is the name of a collection and the
value is a path to file containing the schema definition.
"""
- from params import params
- return {
- "bro": params.bro_schema_path,
- "yaf": params.yaf_schema_path,
- "snort": params.snort_schema_path,
- "error": params.error_schema_path,
- "metaalert": params.meta_schema_path
- }
+ return [
+ "bro",
+ "yaf",
+ "snort",
+ "error",
+ "metaalert"
+ ]
def is_configured(self):
return self.__configured
@@ -200,6 +200,53 @@ class IndexingCommands:
user=self.__params.metron_user,
err_msg=err_msg.format(template_name))
+ def solr_schema_install(self, env):
+ from params import params
+ env.set_params(params)
+ Logger.info("Installing Solr schemas")
+ if self.__params.security_enabled:
+ metron_security.kinit(self.__params.kinit_path_local,
+ self.__params.solr_keytab_path,
+ self.__params.solr_principal_name,
+ self.__params.solr_user)
+
+ commands = IndexingCommands(params)
+ for collection_name in commands.get_solr_schemas():
+
+ # install the schema
+ cmd = format((
+ "export ZOOKEEPER={solr_zookeeper_url};"
+ "export SECURITY_ENABLED={security_enabled};"
+ ))
+ cmd += "{0}/bin/create_collection.sh {1};"
+
+ Execute(
+ cmd.format(params.metron_home, collection_name),
+ user=self.__params.solr_user)
+
+ def solr_schema_delete(self, env):
+ from params import params
+ env.set_params(params)
+ Logger.info("Deleting Solr schemas")
+ if self.__params.security_enabled:
+ metron_security.kinit(self.__params.kinit_path_local,
+ self.__params.solr_keytab_path,
+ self.__params.solr_principal_name,
+ self.__params.solr_user)
+
+ commands = IndexingCommands(params)
+ for collection_name in commands.get_solr_schemas():
+ # delete the schema
+ cmd = format((
+ "export ZOOKEEPER={solr_zookeeper_url};"
+ "export SECURITY_ENABLED={security_enabled};"
+ ))
+ cmd += "{0}/bin/delete_collection.sh {1};"
+
+ Execute(
+ cmd.format(params.metron_home, collection_name),
+ user=self.__params.solr_user)
+
def start_batch_indexing_topology(self, env):
Logger.info('Starting ' + self.__batch_indexing_topology)
http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index b5c4bb9..1629465 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -97,11 +97,10 @@ class Indexing(Script):
self.configure(env)
commands = IndexingCommands(params)
if params.ra_indexing_writer == 'Solr':
- Logger.info("Loading Solr schemas")
# Install Solr schemas
try:
if not commands.is_solr_schema_installed():
- self.solr_schema_install(env)
+ commands.solr_schema_install(env)
commands.set_solr_schema_installed()
except Exception as e:
@@ -170,34 +169,6 @@ class Indexing(Script):
cmd.format(params.es_http_url, template_name),
logoutput=True)
- def solr_schema_install(self, env):
- from params import params
- env.set_params(params)
- Logger.info("Installing Solr schemas")
-
- commands = IndexingCommands(params)
- for collection_name, config_path in commands.get_solr_schemas().iteritems():
-
- # install the schema
-
- cmd = "{0}/bin/solr create -c {1} -d {2}"
- Execute(
- cmd.format(params.solr_home, collection_name, config_path),
- logoutput=True, user="solr")
-
- def solr_schema_delete(self, env):
- from params import params
- env.set_params(params)
- Logger.info("Deleting Solr schemas")
-
- commands = IndexingCommands(params)
- for collection_name, config_path in commands.get_solr_schemas().iteritems():
- # delete the schema
- cmd = "{0}/bin/solr delete -c {1}"
- Execute(
- cmd.format(params.solr_home, collection_name),
- logoutput=True, user="solr")
-
@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
def kibana_dashboard_install(self, env):
from params import params
http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 6f4760b..667a926 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -128,6 +128,9 @@ if has_zk_host:
solr_version = '6.6.2'
solr_home = '/var/solr/solr-' + solr_version
solr_zookeeper_url = format(format(config['configurations']['metron-env']['solr_zookeeper_url']))
+solr_user = config['configurations']['solr-config-env']['solr_config_user']
+solr_principal_name = config['configurations']['solr-config-env']['solr_principal_name']
+solr_keytab_path = config['configurations']['solr-config-env']['solr_keytab_path']
# Storm
storm_rest_addr = status_params.storm_rest_addr
@@ -256,6 +259,8 @@ if security_enabled:
nimbus_seeds = config['configurations']['storm-site']['nimbus.seeds']
+ solr_principal_name = solr_principal_name.replace('_HOST', hostname_lowercase)
+
# Management UI
metron_rest_host = default("/clusterHostInfo/metron_rest_hosts", [hostname])[0]
http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-platform/metron-solr/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/README.md b/metron-platform/metron-solr/README.md
index 0d525ec..159779c 100644
--- a/metron-platform/metron-solr/README.md
+++ b/metron-platform/metron-solr/README.md
@@ -20,7 +20,10 @@ limitations under the License.
## Table of Contents
* [Introduction](#introduction)
+* [Configuration](#configuration)
* [Installing](#installing)
+* [Schemas](#schemas)
+* [Collections](#collections)
## Introduction
@@ -110,4 +113,32 @@ A PointType field should be defined as:
<dynamicField name="*_point" type="pdouble" indexed="true" stored="false" docValues="false"/>
<fieldType name="point" class="solr.PointType" subFieldSuffix="_point"/>
```
-If any copy fields are defined, stored and docValues should be set to false.
\ No newline at end of file
+If any copy fields are defined, stored and docValues should be set to false.
+
+## Collections
+
+Convenience scripts are provided with Metron to create and delete collections. Ambari uses these scripts to automatically create collections. To use them outside of Ambari, a few environment variables must be set first:
+```
+# Path to the zookeeper node used by Solr
+export ZOOKEEPER=node1:2181/solr
+# Set to true if Kerberos is enabled
+export SECURITY_ENABLED=true
+```
+The scripts can then be called directly with the collection name as the first argument . For example, to create the bro collection:
+```
+$METRON_HOME/bin/create_collection.sh bro
+```
+To delete the bro collection:
+```
+$METRON_HOME/bin/delete_collection.sh bro
+```
+The `create_collection.sh` script depends on schemas installed in `$METRON_HOME/config/schema`. There are several schemas that come with Metron:
+
+* bro
+* snort
+* yaf
+* metaalert
+* error
+
+Additional schemas should be installed in that location if using the `create_collection.sh` script. Any collection can be deleted with the `delete_collection.sh` script.
+These scripts use the [Solr Collection API](http://lucene.apache.org/solr/guide/6_6/collections-api.html).
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-platform/metron-solr/src/main/scripts/create_collection.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/scripts/create_collection.sh b/metron-platform/metron-solr/src/main/scripts/create_collection.sh
index 37fbccb..7693646 100755
--- a/metron-platform/metron-solr/src/main/scripts/create_collection.sh
+++ b/metron-platform/metron-solr/src/main/scripts/create_collection.sh
@@ -18,10 +18,19 @@
#
METRON_VERSION=${project.version}
METRON_HOME=/usr/metron/$METRON_VERSION
-SOLR_VERSION=${global_solr_version}
-SOLR_USER=solr
-SOLR_SERVICE=$SOLR_USER
-SOLR_VAR_DIR="/var/$SOLR_SERVICE"
+ZOOKEEPER=${ZOOKEEPER:-localhost:2181}
+ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/usr/hdp/current/zookeeper-client}
+SECURITY_ENABLED=${SECURITY_ENABLED:-false}
+NEGOTIATE=''
+if [ ${SECURITY_ENABLED,,} == 'true' ]; then
+ NEGOTIATE=' --negotiate -u : '
+fi
-cd $SOLR_VAR_DIR/solr-${SOLR_VERSION}
-su $SOLR_USER -c "bin/solr create -c $1 -d $METRON_HOME/config/schema/$1/"
+# Get the first Solr node from the list of live nodes in Zookeeper
+SOLR_NODE=`$ZOOKEEPER_HOME/bin/zkCli.sh -server $ZOOKEEPER ls /live_nodes | tail -n 1 | sed 's/\[\([^,]*\).*\]/\1/' | sed 's/_solr//'`
+
+# Upload the collection config set
+zip -rj - $METRON_HOME/config/schema/$1 | curl -X POST $NEGOTIATE --header "Content-Type:text/xml" --data-binary @- "http://$SOLR_NODE/solr/admin/configs?action=UPLOAD&name=$1"
+
+# Create the collection
+curl -X GET $NEGOTIATE "http://$SOLR_NODE/solr/admin/collections?action=CREATE&name=$1&numShards=1"
http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-platform/metron-solr/src/main/scripts/delete_collection.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/scripts/delete_collection.sh b/metron-platform/metron-solr/src/main/scripts/delete_collection.sh
index 68f0c20..c8b45e7 100755
--- a/metron-platform/metron-solr/src/main/scripts/delete_collection.sh
+++ b/metron-platform/metron-solr/src/main/scripts/delete_collection.sh
@@ -18,10 +18,16 @@
#
METRON_VERSION=${project.version}
METRON_HOME=/usr/metron/$METRON_VERSION
-SOLR_VERSION=${global_solr_version}
-SOLR_USER=solr
-SOLR_SERVICE=$SOLR_USER
-SOLR_VAR_DIR="/var/$SOLR_SERVICE"
+ZOOKEEPER=${ZOOKEEPER:-localhost:2181}
+ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/usr/hdp/current/zookeeper-client}
+SECURITY_ENABLED=${SECURITY_ENABLED:-false}
+NEGOTIATE=''
+if [ ${SECURITY_ENABLED,,} == 'true' ]; then
+ NEGOTIATE=' --negotiate -u : '
+fi
-cd $SOLR_VAR_DIR/solr-${SOLR_VERSION}
-su $SOLR_USER -c "bin/solr delete -c $1"
+# Get the first Solr node from the list of live nodes in Zookeeper
+SOLR_NODE=`$ZOOKEEPER_HOME/bin/zkCli.sh -server $ZOOKEEPER ls /live_nodes | tail -n 1 | sed 's/\[\([^,]*\).*\]/\1/' | sed 's/_solr//'`
+
+# Delete the collection
+curl -X GET $NEGOTIATE "http://$SOLR_NODE/solr/admin/collections?action=DELETE&name=$1"