You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2018/04/27 19:30:20 UTC
[40/50] [abbrv] metron git commit: METRON-1499 Enable Configuration
of Unified Enrichment Topology via Ambari (nickwallen) closes
apache/metron#984
METRON-1499 Enable Configuration of Unified Enrichment Topology via Ambari (nickwallen) closes apache/metron#984
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/82212ba8
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/82212ba8
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/82212ba8
Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 82212ba818a0ef3b92505e8d144487c69a8d4a44
Parents: 3fcbf8b
Author: nickwallen <ni...@nickallen.org>
Authored: Tue Apr 17 09:43:16 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Tue Apr 17 09:43:16 2018 -0400
----------------------------------------------------------------------
.../packaging/ambari/metron-mpack/pom.xml | 7 +-
.../configuration/metron-enrichment-env.xml | 130 +++-
.../package/scripts/enrichment_commands.py | 20 +-
.../package/scripts/enrichment_master.py | 12 +-
.../package/scripts/params/params_linux.py | 19 +-
.../enrichment-splitjoin.properties.j2 | 63 ++
.../templates/enrichment-unified.properties.j2 | 60 ++
.../METRON/CURRENT/themes/metron_theme.json | 151 ++++-
.../docker/rpm-docker/SPECS/metron.spec | 5 +-
.../main/config/enrichment-splitjoin.properties | 63 ++
.../config/enrichment-splitjoin.properties.j2 | 63 ++
.../main/config/enrichment-unified.properties | 69 +++
.../config/enrichment-unified.properties.j2 | 60 ++
.../src/main/config/enrichment.properties | 64 --
.../src/main/config/enrichment.properties.j2 | 63 --
.../main/flux/enrichment/remote-splitjoin.yaml | 590 +++++++++++++++++++
.../main/flux/enrichment/remote-unified.yaml | 71 ++-
.../src/main/flux/enrichment/remote.yaml | 590 -------------------
.../main/scripts/start_enrichment_topology.sh | 16 +-
.../integration/EnrichmentIntegrationTest.java | 61 +-
.../UnifiedEnrichmentIntegrationTest.java | 71 +++
21 files changed, 1433 insertions(+), 815 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
index 491e8dd..1843eb7 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
@@ -110,7 +110,8 @@
<resource>
<directory>${basedir}/../../../../metron-platform/metron-enrichment/src/main/config</directory>
<includes>
- <include>enrichment.properties.j2</include>
+ <include>enrichment-splitjoin.properties.j2</include>
+ <include>enrichment-unified.properties.j2</include>
</includes>
<filtering>false</filtering>
</resource>
@@ -171,8 +172,10 @@
<fileset>
<directory>${basedir}/src/main/resources/common-services/METRON/CURRENT/package/templates</directory>
<includes>
- <include>enrichment.properties.j2</include>
+ <include>enrichment-unified.properties.j2</include>
+ <include>enrichment-splitjoin.properties.j2</include>
<include>elasticsearch.properties.j2</include>
+ <include>hdfs.properties.j2</include>
</includes>
</fileset>
</filesets>
http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
index 9737660..81b135c 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
@@ -18,6 +18,10 @@
limitations under the License.
-->
<configuration supports_final="true">
+
+ <!--
+ enrichment adapter parameters
+ -->
<property>
<name>geoip_url</name>
<value>http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz</value>
@@ -33,6 +37,10 @@
<type>content</type>
</value-attributes>
</property>
+
+ <!--
+ kafka parameters
+ -->
<property>
<name>enrichment_kafka_start</name>
<description>Enrichment Topology Spout Offset</description>
@@ -81,6 +89,10 @@
<value>indexing</value>
<display-name>Threat Intel Error Topic</display-name>
</property>
+
+ <!--
+ hbase parameters
+ -->
<property>
<name>enrichment_hbase_table</name>
<value>enrichment</value>
@@ -105,6 +117,10 @@
<description>The HBase column family which will hold threatintel data in HBase.</description>
<display-name>HBase Table Column Family</display-name>
</property>
+
+ <!--
+ storm common parameters
+ -->
<property>
<name>enrichment_workers</name>
<description>Number of Workers for the Enrichment Topology</description>
@@ -129,70 +145,156 @@
<property>
<name>enrichment_topology_max_spout_pending</name>
<description>Spout Max Pending Tuples for the Enrichment Topology</description>
- <value/>
- <display-name>Enrichment Max Pending</display-name>
+ <value>500</value>
+ <display-name>Enrichment Max Spout Pending</display-name>
<value-attributes>
<empty-value-valid>true</empty-value-valid>
</value-attributes>
</property>
<property>
+ <name>enrichment_topology</name>
+ <description>Which Enrichment topology to execute</description>
+ <value>Split-Join</value>
+ <display-name>Enrichment Topology</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>Split-Join</value>
+ </entry>
+ <entry>
+ <value>Unified</value>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ </property>
+
+ <!--
+ split-join topology parameters
+ -->
+ <property>
<name>enrichment_join_cache_size</name>
- <description>Enrichment Join Bolt Cache Max Size</description>
+ <description>Enrichment join bolt max cache size for the Split Join Enrichment Topology</description>
<value>100000</value>
- <display-name>Enrichment Join Size</display-name>
+ <display-name>Enrichment Join Max Cache Size</display-name>
</property>
<property>
<name>threatintel_join_cache_size</name>
- <description>Threat Intel Join Bolt Cache Max Size</description>
+ <description>Threat Intel join bolt max cache size for the Split Join Enrichment Topology</description>
<value>100000</value>
- <display-name>Threat Intel Join Size</display-name>
+ <display-name>Threat Intel Join Max Cache Size</display-name>
</property>
<property>
<name>enrichment_kafka_spout_parallelism</name>
- <description>Kafka Spout Parallelism for the Enrichment Topology</description>
+ <description>Kafka spout parallelism for the Split Join Enrichment Topology</description>
<value>1</value>
<display-name>Enrichment Spout Parallelism</display-name>
</property>
<property>
<name>enrichment_split_parallelism</name>
- <description>Enrichment Split Bolt Parallelism for the Enrichment Topology</description>
+ <description>Enrichment split bolt parallelism for the Split Join Enrichment Topology</description>
<value>1</value>
<display-name>Enrichment Split Parallelism</display-name>
</property>
<property>
<name>enrichment_stellar_parallelism</name>
- <description>Stellar Enrichment Bolt Parallelism for the Enrichment Topology</description>
+ <description>Stellar enrichment bolt parallelism for the Split Join Enrichment Topology</description>
<value>1</value>
<display-name>Stellar Enrichment Parallelism</display-name>
</property>
<property>
<name>enrichment_join_parallelism</name>
- <description>Enrichment Join Bolt Parallelism for the Enrichment Topology</description>
+ <description>Enrichment join bolt parallelism for the Split Join Enrichment Topology</description>
<value>1</value>
<display-name>Enrichment Join Parallelism</display-name>
</property>
<property>
<name>threat_intel_split_parallelism</name>
- <description>Threat Intel Split Bolt Parallelism for the Enrichment Topology</description>
+ <description>Threat Intel split bolt parallelism for the Split Join Enrichment Topology</description>
<value>1</value>
<display-name>Threat Intel Split Parallelism</display-name>
</property>
<property>
<name>threat_intel_stellar_parallelism</name>
- <description>Threat Intel Stellar Bolt Parallelism for the Enrichment Topology</description>
+ <description>Threat Intel stellar bolt parallelism for the Split Join Enrichment Topology</description>
<value>1</value>
<display-name>Threat Intel Stellar Parallelism</display-name>
</property>
<property>
<name>threat_intel_join_parallelism</name>
- <description>Threat Intel Join Bolt Parallelism for the Enrichment Topology</description>
+ <description>Threat Intel join bolt parallelism for the Split Join Enrichment Topology</description>
<value>1</value>
<display-name>Threat Intel Join Parallelism</display-name>
</property>
<property>
<name>kafka_writer_parallelism</name>
- <description>Kafka Writer Parallelism for the Enrichment Topology</description>
+ <description>Kafka writer parallelism for the Split Join Enrichment Topology</description>
<value>1</value>
<display-name>Enrichment Kafka Writer Parallelism</display-name>
</property>
+
+ <!--
+ unified topology parameters
+ -->
+ <property>
+ <name>unified_kafka_spout_parallelism</name>
+ <description>Kafka spout parallelism for the Unified Enrichment Topology</description>
+ <value>1</value>
+ <display-name>Unified Enrichment Spout Parallelism</display-name>
+ </property>
+ <property>
+ <name>unified_enrichment_parallelism</name>
+ <description>Enrichment parallelism for the Unified Enrichment Topology</description>
+ <value>1</value>
+ <display-name>Unified Enrichment Parallelism</display-name>
+ </property>
+ <property>
+ <name>unified_threat_intel_parallelism</name>
+ <description>Threat Intel parallelism for the Unified Enrichment Topology</description>
+ <value>1</value>
+ <display-name>Unified Threat Intel Parallelism</display-name>
+ </property>
+ <property>
+ <name>unified_kafka_writer_parallelism</name>
+ <description>Kafka writer parallelism for the Unified Enrichment Topology</description>
+ <value>1</value>
+ <display-name>Unified Kafka Writer Parallelism</display-name>
+ </property>
+ <property>
+ <name>unified_enrichment_cache_size</name>
+ <description>Enrichment max cache size for the Unified Enrichment Topology</description>
+ <value>100000</value>
+ <display-name>Unified Enrichment Cache Size</display-name>
+ </property>
+ <property>
+ <name>unified_threat_intel_cache_size</name>
+ <description>Threat Intel Max Cache Size for the Unified Enrichment Topology</description>
+ <value>100000</value>
+ <display-name>Unified Threat Intel Cache Size</display-name>
+ </property>
+ <property>
+ <name>unified_enrichment_threadpool_size</name>
+ <description>Enrichment thread pool size for the Unified Enrichment Topology</description>
+ <value>1</value>
+ <display-name>Unified Enrichment Thread Pool Size</display-name>
+ </property>
+ <property>
+ <name>unified_enrichment_threadpool_type</name>
+ <description>Enrichment thread pool type for the Unified Enrichment Topology</description>
+ <display-name>Unified Enrichment Thread Pool Type</display-name>
+ <value>FIXED</value>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>FIXED</value>
+ </entry>
+ <entry>
+ <value>WORK_STEALING</value>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
index f9ec547..a1bdbed 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
@@ -129,13 +129,21 @@ class EnrichmentCommands:
Logger.info("Starting Metron enrichment topology: {0}".format(self.__enrichment_topology))
if not self.is_topology_active(env):
- start_cmd_template = """{0}/bin/start_enrichment_topology.sh \
- -s {1} \
- -z {2}"""
+
+ # which enrichment topology needs started?
+ if self.__params.enrichment_topology == "Unified":
+ topology_flux = "{0}/flux/enrichment/remote-unified.yaml".format(self.__params.metron_home)
+ topology_props = "{0}/config/enrichment-unified.properties".format(self.__params.metron_home)
+ elif self.__params.enrichment_topology == "Split-Join":
+ topology_flux = "{0}/flux/enrichment/remote-splitjoin.yaml".format(self.__params.metron_home)
+ topology_props = "{0}/config/enrichment-splitjoin.properties".format(self.__params.metron_home)
+ else:
+ raise Fail("Unexpected enrichment topology; name=" + self.__params.enrichment_topology)
+
+ # start the topology
+ start_cmd_template = """{0}/bin/start_enrichment_topology.sh --remote {1} --filter {2}"""
Logger.info('Starting ' + self.__enrichment_topology)
- start_cmd = start_cmd_template.format(self.__params.metron_home,
- self.__enrichment_topology,
- self.__params.zookeeper_quorum)
+ start_cmd = start_cmd_template.format(self.__params.metron_home, topology_flux, topology_props)
Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True)
else:
Logger.info('Enrichment topology already running')
http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
index 24feb81..cada1d2 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
@@ -38,11 +38,15 @@ class Enrichment(Script):
env.set_params(params)
Logger.info("Running enrichment configure")
- File(format("{metron_config_path}/enrichment.properties"),
- content=Template("enrichment.properties.j2"),
+ File(format("{metron_config_path}/enrichment-splitjoin.properties"),
+ content=Template("enrichment-splitjoin.properties.j2"),
owner=params.metron_user,
- group=params.metron_group
- )
+ group=params.metron_group)
+
+ File(format("{metron_config_path}/enrichment-unified.properties"),
+ content=Template("enrichment-unified.properties.j2"),
+ owner=params.metron_user,
+ group=params.metron_group)
if not metron_service.is_zk_configured(params):
metron_service.init_zk_config(params)
http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index ccce022..f44d05f 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -256,20 +256,27 @@ user_settings_hbase_table = status_params.user_settings_hbase_table
user_settings_hbase_cf = status_params.user_settings_hbase_cf
# Enrichment
+metron_enrichment_topology = status_params.metron_enrichment_topology
geoip_url = config['configurations']['metron-enrichment-env']['geoip_url']
enrichment_host_known_hosts = config['configurations']['metron-enrichment-env']['enrichment_host_known_hosts']
+
+# Enrichment - Kafka
enrichment_kafka_start = config['configurations']['metron-enrichment-env']['enrichment_kafka_start']
enrichment_input_topic = status_params.enrichment_input_topic
enrichment_output_topic = config['configurations']['metron-enrichment-env']['enrichment_output_topic']
enrichment_error_topic = config['configurations']['metron-enrichment-env']['enrichment_error_topic']
threatintel_error_topic = config['configurations']['metron-enrichment-env']['threatintel_error_topic']
-metron_enrichment_topology = status_params.metron_enrichment_topology
+
+# Enrichment - Storm common parameters
enrichment_workers = config['configurations']['metron-enrichment-env']['enrichment_workers']
enrichment_acker_executors = config['configurations']['metron-enrichment-env']['enrichment_acker_executors']
if not len(enrichment_topology_worker_childopts) == 0:
enrichment_topology_worker_childopts += ' '
enrichment_topology_worker_childopts += config['configurations']['metron-enrichment-env']['enrichment_topology_worker_childopts']
enrichment_topology_max_spout_pending = config['configurations']['metron-enrichment-env']['enrichment_topology_max_spout_pending']
+enrichment_topology = config['configurations']['metron-enrichment-env']['enrichment_topology']
+
+# Enrichment - Split Join topology
enrichment_join_cache_size = config['configurations']['metron-enrichment-env']['enrichment_join_cache_size']
threatintel_join_cache_size = config['configurations']['metron-enrichment-env']['threatintel_join_cache_size']
enrichment_kafka_spout_parallelism = config['configurations']['metron-enrichment-env']['enrichment_kafka_spout_parallelism']
@@ -281,6 +288,16 @@ threat_intel_stellar_parallelism = config['configurations']['metron-enrichment-e
threat_intel_join_parallelism = config['configurations']['metron-enrichment-env']['threat_intel_join_parallelism']
kafka_writer_parallelism = config['configurations']['metron-enrichment-env']['kafka_writer_parallelism']
+# Enrichment - Unified topology
+unified_kafka_spout_parallelism = config['configurations']['metron-enrichment-env']['unified_kafka_spout_parallelism']
+unified_enrichment_parallelism = config['configurations']['metron-enrichment-env']['unified_enrichment_parallelism']
+unified_threat_intel_parallelism = config['configurations']['metron-enrichment-env']['unified_threat_intel_parallelism']
+unified_kafka_writer_parallelism = config['configurations']['metron-enrichment-env']['unified_kafka_writer_parallelism']
+unified_enrichment_cache_size = config['configurations']['metron-enrichment-env']['unified_enrichment_cache_size']
+unified_threat_intel_cache_size = config['configurations']['metron-enrichment-env']['unified_threat_intel_cache_size']
+unified_enrichment_threadpool_size = config['configurations']['metron-enrichment-env']['unified_enrichment_threadpool_size']
+unified_enrichment_threadpool_type = config['configurations']['metron-enrichment-env']['unified_enrichment_threadpool_type']
+
# Profiler
metron_profiler_topology = 'profiler'
profiler_input_topic = config['configurations']['metron-enrichment-env']['enrichment_output_topic']
http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2
new file mode 100644
index 0000000..a0b21c9
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2
@@ -0,0 +1,63 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+##### Storm #####
+enrichment.workers={{enrichment_workers}}
+enrichment.acker.executors={{enrichment_acker_executors}}
+topology.worker.childopts={{enrichment_topology_worker_childopts}}
+topology.auto-credentials={{topology_auto_credentials}}
+topology.max.spout.pending={{enrichment_topology_max_spout_pending}}
+
+##### Kafka #####
+kafka.zk={{zookeeper_quorum}}
+kafka.broker={{kafka_brokers}}
+kafka.security.protocol={{kafka_security_protocol}}
+
+# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST
+kafka.start={{enrichment_kafka_start}}
+
+enrichment.input.topic={{enrichment_input_topic}}
+enrichment.output.topic={{enrichment_output_topic}}
+enrichment.error.topic={{enrichment_error_topic}}
+threat.intel.error.topic={{threatintel_error_topic}}
+
+##### JoinBolt #####
+enrichment.join.cache.size={{enrichment_join_cache_size}}
+threat.intel.join.cache.size={{threatintel_join_cache_size}}
+
+##### Enrichment #####
+hbase.provider.impl={{enrichment_hbase_provider_impl}}
+enrichment.simple.hbase.table={{enrichment_hbase_table}}
+enrichment.simple.hbase.cf={{enrichment_hbase_cf}}
+enrichment.host.known_hosts={{enrichment_host_known_hosts}}
+
+##### Threat Intel #####
+threat.intel.tracker.table={{threatintel_hbase_table}}
+threat.intel.tracker.cf={{threatintel_hbase_cf}}
+threat.intel.simple.hbase.table={{threatintel_hbase_table}}
+threat.intel.simple.hbase.cf={{threatintel_hbase_cf}}
+
+##### Parallelism #####
+kafka.spout.parallelism={{enrichment_kafka_spout_parallelism}}
+enrichment.split.parallelism={{enrichment_split_parallelism}}
+enrichment.stellar.parallelism={{enrichment_stellar_parallelism}}
+enrichment.join.parallelism={{enrichment_join_parallelism}}
+threat.intel.split.parallelism={{threat_intel_split_parallelism}}
+threat.intel.stellar.parallelism={{threat_intel_stellar_parallelism}}
+threat.intel.join.parallelism={{threat_intel_join_parallelism}}
+kafka.writer.parallelism={{kafka_writer_parallelism}}
http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-unified.properties.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-unified.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-unified.properties.j2
new file mode 100644
index 0000000..8c28c49
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-unified.properties.j2
@@ -0,0 +1,60 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+##### Storm #####
+enrichment.workers={{enrichment_workers}}
+enrichment.acker.executors={{enrichment_acker_executors}}
+topology.worker.childopts={{enrichment_topology_worker_childopts}}
+topology.auto-credentials={{topology_auto_credentials}}
+topology.max.spout.pending={{enrichment_topology_max_spout_pending}}
+
+##### Kafka #####
+kafka.zk={{zookeeper_quorum}}
+kafka.broker={{kafka_brokers}}
+kafka.security.protocol={{kafka_security_protocol}}
+kafka.start={{enrichment_kafka_start}}
+enrichment.input.topic={{enrichment_input_topic}}
+enrichment.output.topic={{enrichment_output_topic}}
+enrichment.error.topic={{enrichment_error_topic}}
+threat.intel.error.topic={{threatintel_error_topic}}
+
+##### Enrichment #####
+hbase.provider.impl={{enrichment_hbase_provider_impl}}
+enrichment.simple.hbase.table={{enrichment_hbase_table}}
+enrichment.simple.hbase.cf={{enrichment_hbase_cf}}
+enrichment.host.known_hosts={{enrichment_host_known_hosts}}
+
+##### Threat Intel #####
+threat.intel.tracker.table={{threatintel_hbase_table}}
+threat.intel.tracker.cf={{threatintel_hbase_cf}}
+threat.intel.simple.hbase.table={{threatintel_hbase_table}}
+threat.intel.simple.hbase.cf={{threatintel_hbase_cf}}
+
+##### Parallelism #####
+kafka.spout.parallelism={{unified_kafka_spout_parallelism}}
+enrichment.parallelism={{unified_enrichment_parallelism}}
+threat.intel.parallelism={{unified_threat_intel_parallelism}}
+kafka.writer.parallelism={{unified_kafka_writer_parallelism}}
+
+##### Caches #####
+enrichment.cache.size={{unified_enrichment_cache_size}}
+threat.intel.cache.size={{unified_threat_intel_cache_size}}
+
+##### Threads #####
+enrichment.threadpool.size={{unified_enrichment_threadpool_size}}
+enrichment.threadpool.type={{unified_enrichment_threadpool_type}}
http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
index 364b3ef..06bc155 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
@@ -65,7 +65,7 @@
"display-name": "Enrichment",
"layout": {
"tab-columns": "1",
- "tab-rows": "3",
+ "tab-rows": "5",
"sections": [
{
"name": "section-enrichment-adapters",
@@ -123,6 +123,44 @@
"column-span": "1"
}
]
+ },
+ {
+ "name": "section-enrichment-splitjoin",
+ "row-index": "3",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-enrichment-splitjoin",
+ "display-name": "Split Join Topology",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ },
+ {
+ "name": "section-enrichment-unified",
+ "row-index": "4",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-enrichment-unified",
+ "display-name": "Unified Topology",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
}
]
}
@@ -462,44 +500,80 @@
"subsection-name": "subsection-enrichment-storm"
},
{
- "config": "metron-enrichment-env/enrichment_join_cache_size",
+ "config": "metron-enrichment-env/enrichment_topology",
"subsection-name": "subsection-enrichment-storm"
},
{
+ "config": "metron-enrichment-env/enrichment_join_cache_size",
+ "subsection-name": "subsection-enrichment-splitjoin"
+ },
+ {
"config": "metron-enrichment-env/threatintel_join_cache_size",
- "subsection-name": "subsection-enrichment-storm"
+ "subsection-name": "subsection-enrichment-splitjoin"
},
{
"config": "metron-enrichment-env/enrichment_kafka_spout_parallelism",
- "subsection-name": "subsection-enrichment-storm"
+ "subsection-name": "subsection-enrichment-splitjoin"
},
{
"config": "metron-enrichment-env/enrichment_split_parallelism",
- "subsection-name": "subsection-enrichment-storm"
+ "subsection-name": "subsection-enrichment-splitjoin"
},
{
"config": "metron-enrichment-env/enrichment_stellar_parallelism",
- "subsection-name": "subsection-enrichment-storm"
+ "subsection-name": "subsection-enrichment-splitjoin"
},
{
"config": "metron-enrichment-env/enrichment_join_parallelism",
- "subsection-name": "subsection-enrichment-storm"
+ "subsection-name": "subsection-enrichment-splitjoin"
},
{
"config": "metron-enrichment-env/threat_intel_split_parallelism",
- "subsection-name": "subsection-enrichment-storm"
+ "subsection-name": "subsection-enrichment-splitjoin"
},
{
"config": "metron-enrichment-env/threat_intel_stellar_parallelism",
- "subsection-name": "subsection-enrichment-storm"
+ "subsection-name": "subsection-enrichment-splitjoin"
},
{
"config": "metron-enrichment-env/threat_intel_join_parallelism",
- "subsection-name": "subsection-enrichment-storm"
+ "subsection-name": "subsection-enrichment-splitjoin"
},
{
"config": "metron-enrichment-env/kafka_writer_parallelism",
- "subsection-name": "subsection-enrichment-storm"
+ "subsection-name": "subsection-enrichment-splitjoin"
+ },
+ {
+ "config": "metron-enrichment-env/unified_kafka_spout_parallelism",
+ "subsection-name": "subsection-enrichment-unified"
+ },
+ {
+ "config": "metron-enrichment-env/unified_enrichment_parallelism",
+ "subsection-name": "subsection-enrichment-unified"
+ },
+ {
+ "config": "metron-enrichment-env/unified_threat_intel_parallelism",
+ "subsection-name": "subsection-enrichment-unified"
+ },
+ {
+ "config": "metron-enrichment-env/unified_kafka_writer_parallelism",
+ "subsection-name": "subsection-enrichment-unified"
+ },
+ {
+ "config": "metron-enrichment-env/unified_enrichment_cache_size",
+ "subsection-name": "subsection-enrichment-unified"
+ },
+ {
+ "config": "metron-enrichment-env/unified_threat_intel_cache_size",
+ "subsection-name": "subsection-enrichment-unified"
+ },
+ {
+ "config": "metron-enrichment-env/unified_enrichment_threadpool_size",
+ "subsection-name": "subsection-enrichment-unified"
+ },
+ {
+ "config": "metron-enrichment-env/unified_enrichment_threadpool_type",
+ "subsection-name": "subsection-enrichment-unified"
},
{
"config": "metron-indexing-env/ra_indexing_kafka_start",
@@ -838,6 +912,12 @@
}
},
{
+ "config": "metron-enrichment-env/enrichment_topology",
+ "widget": {
+ "type": "combo"
+ }
+ },
+ {
"config": "metron-enrichment-env/enrichment_join_cache_size",
"widget": {
"type": "text-field"
@@ -897,7 +977,54 @@
"type": "text-field"
}
},
-
+ {
+ "config": "metron-enrichment-env/unified_kafka_spout_parallelism",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-enrichment-env/unified_enrichment_parallelism",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-enrichment-env/unified_threat_intel_parallelism",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-enrichment-env/unified_kafka_writer_parallelism",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-enrichment-env/unified_enrichment_cache_size",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-enrichment-env/unified_threat_intel_cache_size",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-enrichment-env/unified_enrichment_threadpool_size",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-enrichment-env/unified_enrichment_threadpool_type",
+ "widget": {
+ "type": "combo"
+ }
+ },
{
"config": "metron-indexing-env/batch_indexing_kafka_start",
"widget": {
http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 6b35dae..1f40105 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -273,13 +273,14 @@ This package installs the Metron Enrichment files
%dir %{metron_home}/flux/enrichment
%{metron_home}/bin/latency_summarizer.sh
%{metron_home}/bin/start_enrichment_topology.sh
-%{metron_home}/config/enrichment.properties
+%{metron_home}/config/enrichment-splitjoin.properties
+%{metron_home}/config/enrichment-unified.properties
%{metron_home}/config/zookeeper/enrichments/bro.json
%{metron_home}/config/zookeeper/enrichments/snort.json
%{metron_home}/config/zookeeper/enrichments/websphere.json
%{metron_home}/config/zookeeper/enrichments/yaf.json
%{metron_home}/config/zookeeper/enrichments/asa.json
-%{metron_home}/flux/enrichment/remote.yaml
+%{metron_home}/flux/enrichment/remote-splitjoin.yaml
%{metron_home}/flux/enrichment/remote-unified.yaml
%attr(0644,root,root) %{metron_home}/lib/metron-enrichment-%{full_version}-uber.jar
http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties b/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties
new file mode 100644
index 0000000..109c2ee
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##### Storm #####
+enrichment.workers=1
+enrichment.acker.executors=0
+topology.worker.childopts=
+topology.auto-credentials=
+topology.max.spout.pending=500
+
+##### Kafka #####
+kafka.zk=node1:2181
+kafka.broker=node1:6667
+kafka.security.protocol=PLAINTEXT
+
+# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST
+kafka.start=UNCOMMITTED_EARLIEST
+
+enrichment.input.topic=enrichments
+enrichment.output.topic=indexing
+enrichment.error.topic=indexing
+threat.intel.error.topic=indexing
+
+##### JoinBolt #####
+enrichment.join.cache.size=100000
+threat.intel.join.cache.size=100000
+
+##### Enrichment #####
+hbase.provider.impl=org.apache.metron.hbase.HTableProvider
+enrichment.simple.hbase.table=enrichment
+enrichment.simple.hbase.cf=t
+enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
+{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
+{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
+
+##### Threat Intel #####
+threat.intel.tracker.table=access_tracker
+threat.intel.tracker.cf=t
+threat.intel.simple.hbase.table=threatintel
+threat.intel.simple.hbase.cf=t
+
+##### Parallelism #####
+kafka.spout.parallelism=1
+enrichment.split.parallelism=1
+enrichment.stellar.parallelism=1
+enrichment.join.parallelism=1
+threat.intel.split.parallelism=1
+threat.intel.stellar.parallelism=1
+threat.intel.join.parallelism=1
+kafka.writer.parallelism=1
http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties.j2
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties.j2 b/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties.j2
new file mode 100755
index 0000000..a0b21c9
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties.j2
@@ -0,0 +1,63 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+##### Storm #####
+enrichment.workers={{enrichment_workers}}
+enrichment.acker.executors={{enrichment_acker_executors}}
+topology.worker.childopts={{enrichment_topology_worker_childopts}}
+topology.auto-credentials={{topology_auto_credentials}}
+topology.max.spout.pending={{enrichment_topology_max_spout_pending}}
+
+##### Kafka #####
+kafka.zk={{zookeeper_quorum}}
+kafka.broker={{kafka_brokers}}
+kafka.security.protocol={{kafka_security_protocol}}
+
+# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST
+kafka.start={{enrichment_kafka_start}}
+
+enrichment.input.topic={{enrichment_input_topic}}
+enrichment.output.topic={{enrichment_output_topic}}
+enrichment.error.topic={{enrichment_error_topic}}
+threat.intel.error.topic={{threatintel_error_topic}}
+
+##### JoinBolt #####
+enrichment.join.cache.size={{enrichment_join_cache_size}}
+threat.intel.join.cache.size={{threatintel_join_cache_size}}
+
+##### Enrichment #####
+hbase.provider.impl={{enrichment_hbase_provider_impl}}
+enrichment.simple.hbase.table={{enrichment_hbase_table}}
+enrichment.simple.hbase.cf={{enrichment_hbase_cf}}
+enrichment.host.known_hosts={{enrichment_host_known_hosts}}
+
+##### Threat Intel #####
+threat.intel.tracker.table={{threatintel_hbase_table}}
+threat.intel.tracker.cf={{threatintel_hbase_cf}}
+threat.intel.simple.hbase.table={{threatintel_hbase_table}}
+threat.intel.simple.hbase.cf={{threatintel_hbase_cf}}
+
+##### Parallelism #####
+kafka.spout.parallelism={{enrichment_kafka_spout_parallelism}}
+enrichment.split.parallelism={{enrichment_split_parallelism}}
+enrichment.stellar.parallelism={{enrichment_stellar_parallelism}}
+enrichment.join.parallelism={{enrichment_join_parallelism}}
+threat.intel.split.parallelism={{threat_intel_split_parallelism}}
+threat.intel.stellar.parallelism={{threat_intel_stellar_parallelism}}
+threat.intel.join.parallelism={{threat_intel_join_parallelism}}
+kafka.writer.parallelism={{kafka_writer_parallelism}}
http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties b/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties
new file mode 100644
index 0000000..5338ead
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties
@@ -0,0 +1,69 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+##### Storm #####
+enrichment.workers=1
+enrichment.acker.executors=0
+topology.worker.childopts=
+topology.auto-credentials=
+topology.max.spout.pending=500
+
+##### Kafka #####
+kafka.zk=node1:2181
+kafka.broker=node1:6667
+kafka.security.protocol=PLAINTEXT
+
+# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST
+kafka.start=UNCOMMITTED_EARLIEST
+
+enrichment.input.topic=enrichments
+enrichment.output.topic=indexing
+enrichment.error.topic=indexing
+threat.intel.error.topic=indexing
+
+##### JoinBolt #####
+enrichment.join.cache.size=100000
+threat.intel.join.cache.size=100000
+
+##### Enrichment #####
+hbase.provider.impl=org.apache.metron.hbase.HTableProvider
+enrichment.simple.hbase.table=enrichment
+enrichment.simple.hbase.cf=t
+enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
+{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
+{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
+
+##### Threat Intel #####
+threat.intel.tracker.table=access_tracker
+threat.intel.tracker.cf=t
+threat.intel.simple.hbase.table=threatintel
+threat.intel.simple.hbase.cf=t
+
+##### Parallelism #####
+kafka.spout.parallelism=1
+enrichment.parallelism=1
+threat.intel.parallelism=1
+kafka.writer.parallelism=1
+
+##### Caches #####
+enrichment.cache.size=100000
+threat.intel.cache.size=100000
+
+##### Threads #####
+enrichment.threadpool.size=1
+enrichment.threadpool.type=FIXED
http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties.j2
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties.j2 b/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties.j2
new file mode 100644
index 0000000..8c28c49
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties.j2
@@ -0,0 +1,60 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+##### Storm #####
+enrichment.workers={{enrichment_workers}}
+enrichment.acker.executors={{enrichment_acker_executors}}
+topology.worker.childopts={{enrichment_topology_worker_childopts}}
+topology.auto-credentials={{topology_auto_credentials}}
+topology.max.spout.pending={{enrichment_topology_max_spout_pending}}
+
+##### Kafka #####
+kafka.zk={{zookeeper_quorum}}
+kafka.broker={{kafka_brokers}}
+kafka.security.protocol={{kafka_security_protocol}}
+kafka.start={{enrichment_kafka_start}}
+enrichment.input.topic={{enrichment_input_topic}}
+enrichment.output.topic={{enrichment_output_topic}}
+enrichment.error.topic={{enrichment_error_topic}}
+threat.intel.error.topic={{threatintel_error_topic}}
+
+##### Enrichment #####
+hbase.provider.impl={{enrichment_hbase_provider_impl}}
+enrichment.simple.hbase.table={{enrichment_hbase_table}}
+enrichment.simple.hbase.cf={{enrichment_hbase_cf}}
+enrichment.host.known_hosts={{enrichment_host_known_hosts}}
+
+##### Threat Intel #####
+threat.intel.tracker.table={{threatintel_hbase_table}}
+threat.intel.tracker.cf={{threatintel_hbase_cf}}
+threat.intel.simple.hbase.table={{threatintel_hbase_table}}
+threat.intel.simple.hbase.cf={{threatintel_hbase_cf}}
+
+##### Parallelism #####
+kafka.spout.parallelism={{unified_kafka_spout_parallelism}}
+enrichment.parallelism={{unified_enrichment_parallelism}}
+threat.intel.parallelism={{unified_threat_intel_parallelism}}
+kafka.writer.parallelism={{unified_kafka_writer_parallelism}}
+
+##### Caches #####
+enrichment.cache.size={{unified_enrichment_cache_size}}
+threat.intel.cache.size={{unified_threat_intel_cache_size}}
+
+##### Threads #####
+enrichment.threadpool.size={{unified_enrichment_threadpool_size}}
+enrichment.threadpool.type={{unified_enrichment_threadpool_type}}
http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/config/enrichment.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment.properties b/metron-platform/metron-enrichment/src/main/config/enrichment.properties
deleted file mode 100644
index 9592968..0000000
--- a/metron-platform/metron-enrichment/src/main/config/enrichment.properties
+++ /dev/null
@@ -1,64 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-##### Storm #####
-enrichment.workers=1
-enrichment.acker.executors=0
-topology.worker.childopts=
-topology.auto-credentials=
-topology.max.spout.pending=
-
-##### Kafka #####
-kafka.zk=node1:2181
-kafka.broker=node1:6667
-kafka.security.protocol=PLAINTEXT
-
-# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST
-kafka.start=UNCOMMITTED_EARLIEST
-
-enrichment.input.topic=enrichments
-enrichment.output.topic=indexing
-enrichment.error.topic=indexing
-threat.intel.error.topic=indexing
-
-##### JoinBolt #####
-enrichment.join.cache.size=100000
-threat.intel.join.cache.size=100000
-
-##### Enrichment #####
-hbase.provider.impl=org.apache.metron.hbase.HTableProvider
-enrichment.simple.hbase.table=enrichment
-enrichment.simple.hbase.cf=t
-enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
-{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
-{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
-
-
-##### Threat Intel #####
-threat.intel.tracker.table=access_tracker
-threat.intel.tracker.cf=t
-threat.intel.simple.hbase.table=threatintel
-threat.intel.simple.hbase.cf=t
-
-##### Parallelism #####
-kafka.spout.parallelism=1
-enrichment.split.parallelism=1
-enrichment.stellar.parallelism=1
-enrichment.join.parallelism=1
-threat.intel.split.parallelism=1
-threat.intel.stellar.parallelism=1
-threat.intel.join.parallelism=1
-kafka.writer.parallelism=1
http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2 b/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2
deleted file mode 100755
index 133f9c5..0000000
--- a/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2
+++ /dev/null
@@ -1,63 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-##### Storm #####
-enrichment.workers={{enrichment_workers}}
-enrichment.acker.executors={{enrichment_acker_executors}}
-topology.worker.childopts={{enrichment_topology_worker_childopts}}
-topology.auto-credentials={{topology_auto_credentials}}
-topology.max.spout.pending={{enrichment_topology_max_spout_pending}}
-
-##### Kafka #####
-kafka.zk={{zookeeper_quorum}}
-kafka.broker={{kafka_brokers}}
-kafka.security.protocol={{kafka_security_protocol}}
-
-# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST
-kafka.start={{enrichment_kafka_start}}
-
-enrichment.input.topic={{enrichment_input_topic}}
-enrichment.output.topic={{enrichment_output_topic}}
-enrichment.error.topic={{enrichment_error_topic}}
-threat.intel.error.topic={{threatintel_error_topic}}
-
-##### JoinBolt #####
-enrichment.join.cache.size={{enrichment_join_cache_size}}
-threat.intel.join.cache.size={{threatintel_join_cache_size}}
-
-##### Enrichment #####
-hbase.provider.impl={{enrichment_hbase_provider_impl}}
-enrichment.simple.hbase.table={{enrichment_hbase_table}}
-enrichment.simple.hbase.cf={{enrichment_hbase_cf}}
-enrichment.host.known_hosts={{enrichment_host_known_hosts}}
-
-##### Threat Intel #####
-threat.intel.tracker.table={{threatintel_hbase_table}}
-threat.intel.tracker.cf={{threatintel_hbase_cf}}
-threat.intel.simple.hbase.table={{threatintel_hbase_table}}
-threat.intel.simple.hbase.cf={{threatintel_hbase_cf}}
-
-##### Parallelism #####
-kafka.spout.parallelism={{enrichment_kafka_spout_parallelism}}
-enrichment.split.parallelism={{enrichment_split_parallelism}}
-enrichment.stellar.parallelism={{enrichment_stellar_parallelism}}
-enrichment.join.parallelism={{enrichment_join_parallelism}}
-threat.intel.split.parallelism={{threat_intel_split_parallelism}}
-threat.intel.stellar.parallelism={{threat_intel_stellar_parallelism}}
-threat.intel.join.parallelism={{threat_intel_join_parallelism}}
-kafka.writer.parallelism={{kafka_writer_parallelism}}
http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-splitjoin.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-splitjoin.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-splitjoin.yaml
new file mode 100644
index 0000000..fd7ceff
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-splitjoin.yaml
@@ -0,0 +1,590 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "enrichment"
+config:
+ topology.workers: ${enrichment.workers}
+ topology.acker.executors: ${enrichment.acker.executors}
+ topology.worker.childopts: ${topology.worker.childopts}
+ topology.auto-credentials: ${topology.auto-credentials}
+ topology.max.spout.pending: ${topology.max.spout.pending}
+
+components:
+
+# Enrichment
+ - id: "stellarEnrichmentAdapter"
+ className: "org.apache.metron.enrichment.adapters.stellar.StellarAdapter"
+ configMethods:
+ - name: "ofType"
+ args:
+ - "ENRICHMENT"
+
+ # Any kafka props for the producer go here.
+ - id: "kafkaWriterProps"
+ className: "java.util.HashMap"
+ configMethods:
+ - name: "put"
+ args:
+ - "security.protocol"
+ - "${kafka.security.protocol}"
+
+ - id: "stellarEnrichment"
+ className: "org.apache.metron.enrichment.configuration.Enrichment"
+ constructorArgs:
+ - "stellar"
+ - ref: "stellarEnrichmentAdapter"
+
+ - id: "geoEnrichmentAdapter"
+ className: "org.apache.metron.enrichment.adapters.geo.GeoAdapter"
+ - id: "geoEnrichment"
+ className: "org.apache.metron.enrichment.configuration.Enrichment"
+ constructorArgs:
+ - "geo"
+ - ref: "geoEnrichmentAdapter"
+ - id: "hostEnrichmentAdapter"
+ className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter"
+ constructorArgs:
+ - '${enrichment.host.known_hosts}'
+ - id: "hostEnrichment"
+ className: "org.apache.metron.enrichment.configuration.Enrichment"
+ constructorArgs:
+ - "host"
+ - ref: "hostEnrichmentAdapter"
+
+ - id: "simpleHBaseEnrichmentConfig"
+ className: "org.apache.metron.enrichment.adapters.simplehbase.SimpleHBaseConfig"
+ configMethods:
+ - name: "withProviderImpl"
+ args:
+ - "${hbase.provider.impl}"
+ - name: "withHBaseTable"
+ args:
+ - "${enrichment.simple.hbase.table}"
+ - name: "withHBaseCF"
+ args:
+ - "${enrichment.simple.hbase.cf}"
+ - id: "simpleHBaseEnrichmentAdapter"
+ className: "org.apache.metron.enrichment.adapters.simplehbase.SimpleHBaseAdapter"
+ configMethods:
+ - name: "withConfig"
+ args:
+ - ref: "simpleHBaseEnrichmentConfig"
+ - id: "simpleHBaseEnrichment"
+ className: "org.apache.metron.enrichment.configuration.Enrichment"
+ constructorArgs:
+ - "hbaseEnrichment"
+ - ref: "simpleHBaseEnrichmentAdapter"
+ - id: "enrichments"
+ className: "java.util.ArrayList"
+ configMethods:
+ - name: "add"
+ args:
+ - ref: "geoEnrichment"
+ - name: "add"
+ args:
+ - ref: "hostEnrichment"
+ - name: "add"
+ args:
+ - ref: "simpleHBaseEnrichment"
+ - name: "add"
+ args:
+ - ref: "stellarEnrichment"
+
+ #enrichment error
+ - id: "enrichmentErrorKafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
+ configMethods:
+ - name: "withTopic"
+ args:
+ - "${enrichment.error.topic}"
+ - name: "withZkQuorum"
+ args:
+ - "${kafka.zk}"
+ - name: "withProducerConfigs"
+ args:
+ - ref: "kafkaWriterProps"
+
+# Threat Intel
+ - id: "stellarThreatIntelAdapter"
+ className: "org.apache.metron.enrichment.adapters.stellar.StellarAdapter"
+ configMethods:
+ - name: "ofType"
+ args:
+ - "THREAT_INTEL"
+ - id: "stellarThreatIntelEnrichment"
+ className: "org.apache.metron.enrichment.configuration.Enrichment"
+ constructorArgs:
+ - "stellar"
+ - ref: "stellarThreatIntelAdapter"
+ - id: "simpleHBaseThreatIntelConfig"
+ className: "org.apache.metron.enrichment.adapters.threatintel.ThreatIntelConfig"
+ configMethods:
+ - name: "withProviderImpl"
+ args:
+ - "${hbase.provider.impl}"
+ - name: "withTrackerHBaseTable"
+ args:
+ - "${threat.intel.tracker.table}"
+ - name: "withTrackerHBaseCF"
+ args:
+ - "${threat.intel.tracker.cf}"
+ - name: "withHBaseTable"
+ args:
+ - "${threat.intel.simple.hbase.table}"
+ - name: "withHBaseCF"
+ args:
+ - "${threat.intel.simple.hbase.cf}"
+ - id: "simpleHBaseThreatIntelAdapter"
+ className: "org.apache.metron.enrichment.adapters.threatintel.ThreatIntelAdapter"
+ configMethods:
+ - name: "withConfig"
+ args:
+ - ref: "simpleHBaseThreatIntelConfig"
+ - id: "simpleHBaseThreatIntelEnrichment"
+ className: "org.apache.metron.enrichment.configuration.Enrichment"
+ constructorArgs:
+ - "hbaseThreatIntel"
+ - ref: "simpleHBaseThreatIntelAdapter"
+
+ - id: "threatIntels"
+ className: "java.util.ArrayList"
+ configMethods:
+ - name: "add"
+ args:
+ - ref: "simpleHBaseThreatIntelEnrichment"
+ - name: "add"
+ args:
+ - ref: "stellarThreatIntelEnrichment"
+
+ #threatintel error
+ - id: "threatIntelErrorKafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
+ configMethods:
+ - name: "withTopic"
+ args:
+ - "${threat.intel.error.topic}"
+ - name: "withZkQuorum"
+ args:
+ - "${kafka.zk}"
+ - name: "withProducerConfigs"
+ args:
+ - ref: "kafkaWriterProps"
+#indexing
+ - id: "kafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
+ configMethods:
+ - name: "withTopic"
+ args:
+ - "${enrichment.output.topic}"
+ - name: "withZkQuorum"
+ args:
+ - "${kafka.zk}"
+ - name: "withProducerConfigs"
+ args:
+ - ref: "kafkaWriterProps"
+
+#kafka/zookeeper
+ # Any kafka props for the consumer go here.
+ - id: "kafkaProps"
+ className: "java.util.HashMap"
+ configMethods:
+ - name: "put"
+ args:
+ - "value.deserializer"
+ - "org.apache.kafka.common.serialization.ByteArrayDeserializer"
+ - name: "put"
+ args:
+ - "key.deserializer"
+ - "org.apache.kafka.common.serialization.ByteArrayDeserializer"
+ - name: "put"
+ args:
+ - "group.id"
+ - "enrichments"
+ - name: "put"
+ args:
+ - "security.protocol"
+ - "${kafka.security.protocol}"
+
+
+ # The fields to pull out of the kafka messages
+ - id: "fields"
+ className: "java.util.ArrayList"
+ configMethods:
+ - name: "add"
+ args:
+ - "value"
+
+ - id: "kafkaConfig"
+ className: "org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder"
+ constructorArgs:
+ - ref: "kafkaProps"
+ # topic name
+ - "${enrichment.input.topic}"
+ - "${kafka.zk}"
+ - ref: "fields"
+ configMethods:
+ - name: "setFirstPollOffsetStrategy"
+ args:
+ - "${kafka.start}"
+
+
+spouts:
+ - id: "kafkaSpout"
+ className: "org.apache.metron.storm.kafka.flux.StormKafkaSpout"
+ constructorArgs:
+ - ref: "kafkaConfig"
+ parallelism: ${kafka.spout.parallelism}
+
+bolts:
+# Enrichment Bolts
+ - id: "enrichmentSplitBolt"
+ className: "org.apache.metron.enrichment.bolt.EnrichmentSplitterBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichments"
+ args:
+ - ref: "enrichments"
+ parallelism: ${enrichment.split.parallelism}
+
+ - id: "geoEnrichmentBolt"
+ className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichment"
+ args:
+ - ref: "geoEnrichment"
+ - name: "withMaxCacheSize"
+ args: [10000]
+ - name: "withMaxTimeRetain"
+ args: [10]
+
+ - id: "stellarEnrichmentBolt"
+ className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichment"
+ args:
+ - ref: "stellarEnrichment"
+ - name: "withMaxCacheSize"
+ args: [10000]
+ - name: "withMaxTimeRetain"
+ args: [10]
+ parallelism: ${enrichment.stellar.parallelism}
+
+ - id: "hostEnrichmentBolt"
+ className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichment"
+ args:
+ - ref: "hostEnrichment"
+ - name: "withMaxCacheSize"
+ args: [10000]
+ - name: "withMaxTimeRetain"
+ args: [10]
+
+ - id: "simpleHBaseEnrichmentBolt"
+ className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichment"
+ args:
+ - ref: "simpleHBaseEnrichment"
+ - name: "withMaxCacheSize"
+ args: [10000]
+ - name: "withMaxTimeRetain"
+ args: [10]
+
+ - id: "enrichmentJoinBolt"
+ className: "org.apache.metron.enrichment.bolt.EnrichmentJoinBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withMaxCacheSize"
+ args: [${enrichment.join.cache.size}]
+ - name: "withMaxTimeRetain"
+ args: [10]
+ parallelism: ${enrichment.join.parallelism}
+
+ - id: "enrichmentErrorOutputBolt"
+ className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withMessageWriter"
+ args:
+ - ref: "enrichmentErrorKafkaWriter"
+
+
+# Threat Intel Bolts
+ - id: "threatIntelSplitBolt"
+ className: "org.apache.metron.enrichment.bolt.ThreatIntelSplitterBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichments"
+ args:
+ - ref: "threatIntels"
+ - name: "withMessageFieldName"
+ args: ["message"]
+ parallelism: ${threat.intel.split.parallelism}
+
+ - id: "simpleHBaseThreatIntelBolt"
+ className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichment"
+ args:
+ - ref: "simpleHBaseThreatIntelEnrichment"
+ - name: "withMaxCacheSize"
+ args: [10000]
+ - name: "withMaxTimeRetain"
+ args: [10]
+ - id: "stellarThreatIntelBolt"
+ className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichment"
+ args:
+ - ref: "stellarThreatIntelEnrichment"
+ - name: "withMaxCacheSize"
+ args: [10000]
+ - name: "withMaxTimeRetain"
+ args: [10]
+ parallelism: ${threat.intel.stellar.parallelism}
+
+ - id: "threatIntelJoinBolt"
+ className: "org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withMaxCacheSize"
+ args: [${threat.intel.join.cache.size}]
+ - name: "withMaxTimeRetain"
+ args: [10]
+ parallelism: ${threat.intel.join.parallelism}
+
+ - id: "threatIntelErrorOutputBolt"
+ className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withMessageWriter"
+ args:
+ - ref: "threatIntelErrorKafkaWriter"
+
+# Indexing Bolts
+ - id: "outputBolt"
+ className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withMessageWriter"
+ args:
+ - ref: "kafkaWriter"
+ parallelism: ${kafka.writer.parallelism}
+
+
+streams:
+#parser
+ - name: "spout -> enrichmentSplit"
+ from: "kafkaSpout"
+ to: "enrichmentSplitBolt"
+ grouping:
+ type: LOCAL_OR_SHUFFLE
+
+#enrichment
+ - name: "enrichmentSplit -> host"
+ from: "enrichmentSplitBolt"
+ to: "hostEnrichmentBolt"
+ grouping:
+ streamId: "host"
+ type: FIELDS
+ args: ["message"]
+
+ - name: "enrichmentSplit -> geo"
+ from: "enrichmentSplitBolt"
+ to: "geoEnrichmentBolt"
+ grouping:
+ streamId: "geo"
+ type: FIELDS
+ args: ["message"]
+
+ - name: "enrichmentSplit -> stellar"
+ from: "enrichmentSplitBolt"
+ to: "stellarEnrichmentBolt"
+ grouping:
+ streamId: "stellar"
+ type: FIELDS
+ args: ["message"]
+
+
+ - name: "enrichmentSplit -> simpleHBaseEnrichmentBolt"
+ from: "enrichmentSplitBolt"
+ to: "simpleHBaseEnrichmentBolt"
+ grouping:
+ streamId: "hbaseEnrichment"
+ type: FIELDS
+ args: ["message"]
+
+ - name: "splitter -> join"
+ from: "enrichmentSplitBolt"
+ to: "enrichmentJoinBolt"
+ grouping:
+ streamId: "message"
+ type: FIELDS
+ args: ["key"]
+
+ - name: "geo -> join"
+ from: "geoEnrichmentBolt"
+ to: "enrichmentJoinBolt"
+ grouping:
+ streamId: "geo"
+ type: FIELDS
+ args: ["key"]
+
+ - name: "stellar -> join"
+ from: "stellarEnrichmentBolt"
+ to: "enrichmentJoinBolt"
+ grouping:
+ streamId: "stellar"
+ type: FIELDS
+ args: ["key"]
+
+ - name: "simpleHBaseEnrichmentBolt -> join"
+ from: "simpleHBaseEnrichmentBolt"
+ to: "enrichmentJoinBolt"
+ grouping:
+ streamId: "hbaseEnrichment"
+ type: FIELDS
+ args: ["key"]
+
+ - name: "host -> join"
+ from: "hostEnrichmentBolt"
+ to: "enrichmentJoinBolt"
+ grouping:
+ streamId: "host"
+ type: FIELDS
+ args: ["key"]
+
+ # Error output
+ - name: "geoEnrichmentBolt -> enrichmentErrorOutputBolt"
+ from: "geoEnrichmentBolt"
+ to: "enrichmentErrorOutputBolt"
+ grouping:
+ streamId: "error"
+ type: LOCAL_OR_SHUFFLE
+
+ - name: "stellarEnrichmentBolt -> enrichmentErrorOutputBolt"
+ from: "stellarEnrichmentBolt"
+ to: "enrichmentErrorOutputBolt"
+ grouping:
+ streamId: "error"
+ type: LOCAL_OR_SHUFFLE
+
+ - name: "hostEnrichmentBolt -> enrichmentErrorOutputBolt"
+ from: "hostEnrichmentBolt"
+ to: "enrichmentErrorOutputBolt"
+ grouping:
+ streamId: "error"
+ type: LOCAL_OR_SHUFFLE
+
+ - name: "simpleHBaseEnrichmentBolt -> enrichmentErrorOutputBolt"
+ from: "simpleHBaseEnrichmentBolt"
+ to: "enrichmentErrorOutputBolt"
+ grouping:
+ streamId: "error"
+ type: LOCAL_OR_SHUFFLE
+
+#threat intel
+ - name: "enrichmentJoin -> threatSplit"
+ from: "enrichmentJoinBolt"
+ to: "threatIntelSplitBolt"
+ grouping:
+ streamId: "message"
+ type: FIELDS
+ args: ["key"]
+
+ - name: "threatSplit -> simpleHBaseThreatIntel"
+ from: "threatIntelSplitBolt"
+ to: "simpleHBaseThreatIntelBolt"
+ grouping:
+ streamId: "hbaseThreatIntel"
+ type: FIELDS
+ args: ["message"]
+
+ - name: "threatSplit -> stellarThreatIntel"
+ from: "threatIntelSplitBolt"
+ to: "stellarThreatIntelBolt"
+ grouping:
+ streamId: "stellar"
+ type: FIELDS
+ args: ["message"]
+
+
+ - name: "simpleHBaseThreatIntel -> join"
+ from: "simpleHBaseThreatIntelBolt"
+ to: "threatIntelJoinBolt"
+ grouping:
+ streamId: "hbaseThreatIntel"
+ type: FIELDS
+ args: ["key"]
+
+ - name: "stellarThreatIntel -> join"
+ from: "stellarThreatIntelBolt"
+ to: "threatIntelJoinBolt"
+ grouping:
+ streamId: "stellar"
+ type: FIELDS
+ args: ["key"]
+
+ - name: "threatIntelSplit -> threatIntelJoin"
+ from: "threatIntelSplitBolt"
+ to: "threatIntelJoinBolt"
+ grouping:
+ streamId: "message"
+ type: FIELDS
+ args: ["key"]
+#output
+ - name: "threatIntelJoin -> output"
+ from: "threatIntelJoinBolt"
+ to: "outputBolt"
+ grouping:
+ streamId: "message"
+ type: LOCAL_OR_SHUFFLE
+
+ # Error output
+ - name: "simpleHBaseThreatIntelBolt -> threatIntelErrorOutputBolt"
+ from: "simpleHBaseThreatIntelBolt"
+ to: "threatIntelErrorOutputBolt"
+ grouping:
+ streamId: "error"
+ type: LOCAL_OR_SHUFFLE
+
+ - name: "stellarThreatIntelBolt -> threatIntelErrorOutputBolt"
+ from: "stellarThreatIntelBolt"
+ to: "threatIntelErrorOutputBolt"
+ grouping:
+ streamId: "error"
+ type: LOCAL_OR_SHUFFLE
+
http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-unified.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-unified.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-unified.yaml
index ddc5ffc..d7107d9 100644
--- a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-unified.yaml
+++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-unified.yaml
@@ -27,13 +27,12 @@ config:
topology.worker.childopts: ${topology.worker.childopts}
topology.auto-credentials: ${topology.auto-credentials}
topology.max.spout.pending: ${topology.max.spout.pending}
- # Change this if you want to adjust the threadpool size
- metron.threadpool.size: "2C" # Either a number (e.g. 5) or multiple of cores (e.g. 5C = 5 times the number of cores)
- # Change this if you want to adjust the threadpool type
- metron.threadpool.type: "FIXED" # FIXED or WORK_STEALING
+ metron.threadpool.size: ${enrichment.threadpool.size} # Either a number (e.g. 5) or multiple of cores (e.g. 5C = 5 times the number of cores)
+ metron.threadpool.type: ${enrichment.threadpool.type} # FIXED or WORK_STEALING
+
components:
-# Enrichment
+ # enrichment
- id: "stellarEnrichmentAdapter"
className: "org.apache.metron.enrichment.adapters.stellar.StellarAdapter"
configMethods:
@@ -41,7 +40,7 @@ components:
args:
- "ENRICHMENT"
- # Any kafka props for the producer go here.
+ # any kafka props for the producer go here.
- id: "kafkaWriterProps"
className: "java.util.HashMap"
configMethods:
@@ -58,15 +57,18 @@ components:
- id: "geoEnrichmentAdapter"
className: "org.apache.metron.enrichment.adapters.geo.GeoAdapter"
+
- id: "geoEnrichment"
className: "org.apache.metron.enrichment.configuration.Enrichment"
constructorArgs:
- "geo"
- ref: "geoEnrichmentAdapter"
+
- id: "hostEnrichmentAdapter"
className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter"
constructorArgs:
- '${enrichment.host.known_hosts}'
+
- id: "hostEnrichment"
className: "org.apache.metron.enrichment.configuration.Enrichment"
constructorArgs:
@@ -85,17 +87,20 @@ components:
- name: "withHBaseCF"
args:
- "${enrichment.simple.hbase.cf}"
+
- id: "simpleHBaseEnrichmentAdapter"
className: "org.apache.metron.enrichment.adapters.simplehbase.SimpleHBaseAdapter"
configMethods:
- name: "withConfig"
args:
- ref: "simpleHBaseEnrichmentConfig"
+
- id: "simpleHBaseEnrichment"
className: "org.apache.metron.enrichment.configuration.Enrichment"
constructorArgs:
- "hbaseEnrichment"
- ref: "simpleHBaseEnrichmentAdapter"
+
- id: "enrichments"
className: "java.util.ArrayList"
configMethods:
@@ -112,7 +117,7 @@ components:
args:
- ref: "stellarEnrichment"
- #enrichment error
+ # enrichment error
- id: "enrichmentErrorKafkaWriter"
className: "org.apache.metron.writer.kafka.KafkaWriter"
configMethods:
@@ -123,21 +128,23 @@ components:
args:
- "${kafka.zk}"
- name: "withProducerConfigs"
- args:
+ args:
- ref: "kafkaWriterProps"
-# Threat Intel
+ # threat Intel
- id: "stellarThreatIntelAdapter"
className: "org.apache.metron.enrichment.adapters.stellar.StellarAdapter"
configMethods:
- name: "ofType"
args:
- "THREAT_INTEL"
+
- id: "stellarThreatIntelEnrichment"
className: "org.apache.metron.enrichment.configuration.Enrichment"
constructorArgs:
- "stellar"
- ref: "stellarThreatIntelAdapter"
+
- id: "simpleHBaseThreatIntelConfig"
className: "org.apache.metron.enrichment.adapters.threatintel.ThreatIntelConfig"
configMethods:
@@ -156,12 +163,14 @@ components:
- name: "withHBaseCF"
args:
- "${threat.intel.simple.hbase.cf}"
+
- id: "simpleHBaseThreatIntelAdapter"
className: "org.apache.metron.enrichment.adapters.threatintel.ThreatIntelAdapter"
configMethods:
- name: "withConfig"
args:
- ref: "simpleHBaseThreatIntelConfig"
+
- id: "simpleHBaseThreatIntelEnrichment"
className: "org.apache.metron.enrichment.configuration.Enrichment"
constructorArgs:
@@ -178,7 +187,7 @@ components:
args:
- ref: "stellarThreatIntelEnrichment"
- #threatintel error
+ # threatintel error
- id: "threatIntelErrorKafkaWriter"
className: "org.apache.metron.writer.kafka.KafkaWriter"
configMethods:
@@ -189,9 +198,9 @@ components:
args:
- "${kafka.zk}"
- name: "withProducerConfigs"
- args:
+ args:
- ref: "kafkaWriterProps"
-#indexing
+ # indexing
- id: "kafkaWriter"
className: "org.apache.metron.writer.kafka.KafkaWriter"
configMethods:
@@ -202,11 +211,11 @@ components:
args:
- "${kafka.zk}"
- name: "withProducerConfigs"
- args:
+ args:
- ref: "kafkaWriterProps"
-#kafka/zookeeper
- # Any kafka props for the consumer go here.
+ # kafka/zookeeper
+ # any kafka props for the consumer go here.
- id: "kafkaProps"
className: "java.util.HashMap"
configMethods:
@@ -228,7 +237,7 @@ components:
- "${kafka.security.protocol}"
- # The fields to pull out of the kafka messages
+ # the fields to pull out of the kafka messages
- id: "fields"
className: "java.util.ArrayList"
configMethods:
@@ -251,6 +260,7 @@ components:
spouts:
+
- id: "kafkaSpout"
className: "org.apache.metron.storm.kafka.flux.StormKafkaSpout"
constructorArgs:
@@ -258,7 +268,8 @@ spouts:
parallelism: ${kafka.spout.parallelism}
bolts:
-# Enrichment Bolts
+
+ # enrichment bolt
- id: "enrichmentBolt"
className: "org.apache.metron.enrichment.bolt.UnifiedEnrichmentBolt"
constructorArgs:
@@ -268,7 +279,7 @@ bolts:
args:
- ref: "enrichments"
- name: "withMaxCacheSize"
- args: [${enrichment.join.cache.size}]
+ args: [${enrichment.cache.size}]
- name: "withMaxTimeRetain"
args: [10]
- name: "withCaptureCacheStats"
@@ -278,7 +289,7 @@ bolts:
- "ENRICHMENT"
- name: "withMessageGetter"
args: ["JSON_FROM_POSITION"]
- parallelism: ${enrichment.join.parallelism}
+ parallelism: ${enrichment.parallelism}
- id: "enrichmentErrorOutputBolt"
className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
@@ -290,7 +301,7 @@ bolts:
- ref: "enrichmentErrorKafkaWriter"
-# Threat Intel Bolts
+ # threat intel bolts
- id: "threatIntelBolt"
className: "org.apache.metron.enrichment.bolt.UnifiedEnrichmentBolt"
constructorArgs:
@@ -300,7 +311,7 @@ bolts:
args:
- ref: "threatIntels"
- name: "withMaxCacheSize"
- args: [${enrichment.join.cache.size}]
+ args: [${threat.intel.cache.size}]
- name: "withMaxTimeRetain"
args: [10]
- name: "withCaptureCacheStats"
@@ -312,7 +323,7 @@ bolts:
args: ["message"]
- name: "withMessageGetter"
args: ["JSON_FROM_FIELD_BY_REFERENCE"]
- parallelism: ${threat.intel.join.parallelism}
+ parallelism: ${threat.intel.parallelism}
- id: "threatIntelErrorOutputBolt"
className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
@@ -323,7 +334,7 @@ bolts:
args:
- ref: "threatIntelErrorKafkaWriter"
-# Indexing Bolts
+ # output bolt
- id: "outputBolt"
className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
constructorArgs:
@@ -334,16 +345,16 @@ bolts:
- ref: "kafkaWriter"
parallelism: ${kafka.writer.parallelism}
-
streams:
-#parser
+
+ # parser
- name: "spout -> enrichmentBolt"
from: "kafkaSpout"
to: "enrichmentBolt"
grouping:
type: LOCAL_OR_SHUFFLE
- # Error output
+ # error output
- name: "enrichmentBolt -> enrichmentErrorOutputBolt"
from: "enrichmentBolt"
to: "enrichmentErrorOutputBolt"
@@ -351,7 +362,7 @@ streams:
streamId: "error"
type: LOCAL_OR_SHUFFLE
-#threat intel
+ # threat intel
- name: "enrichmentBolt -> threatIntelBolt"
from: "enrichmentBolt"
to: "threatIntelBolt"
@@ -359,7 +370,7 @@ streams:
streamId: "message"
type: LOCAL_OR_SHUFFLE
-#output
+ # output
- name: "threatIntelBolt -> output"
from: "threatIntelBolt"
to: "outputBolt"
@@ -367,12 +378,10 @@ streams:
streamId: "message"
type: LOCAL_OR_SHUFFLE
- # Error output
+ # error output
- name: "threatIntelBolt -> threatIntelErrorOutputBolt"
from: "threatIntelBolt"
to: "threatIntelErrorOutputBolt"
grouping:
streamId: "error"
type: LOCAL_OR_SHUFFLE
-
-