You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2019/04/08 21:20:22 UTC
[metron] branch master updated: METRON-2053 Refactor
metron-enrichment to decouple Storm dependencies (mmiklavc) closes
apache/metron#1368
This is an automated email from the ASF dual-hosted git repository.
mmiklavcic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 14efe83 METRON-2053 Refactor metron-enrichment to decouple Storm dependencies (mmiklavc) closes apache/metron#1368
14efe83 is described below
commit 14efe83e3a8d9ba7c2bef35061fc9829ee0e61f8
Author: mmiklavc <mi...@gmail.com>
AuthorDate: Mon Apr 8 15:19:47 2019 -0600
METRON-2053 Refactor metron-enrichment to decouple Storm dependencies (mmiklavc) closes apache/metron#1368
---
Upgrading.md | 6 +
.../packaging/ambari/metron-mpack/pom.xml | 2 +-
.../common-services/METRON/CURRENT/metainfo.xml | 5 +-
.../packaging/docker/deb-docker/pom.xml | 8 +-
.../packaging/docker/rpm-docker/SPECS/metron.spec | 73 ++--
.../packaging/docker/rpm-docker/pom.xml | 8 +-
.../apache/metron/rest/config/HBaseConfigTest.java | 18 +-
.../java/org/apache/metron/common/Constants.java | 1 +
.../components/ConfigUploadComponent.java | 21 +-
metron-platform/metron-data-management/pom.xml | 2 +-
metron-platform/metron-elasticsearch/pom.xml | 7 +-
metron-platform/metron-enrichment/README.md | 400 +-------------------
.../{ => metron-enrichment-common}/README.md | 134 +------
.../metron-enrichment-common}/pom.xml | 229 +++++-------
.../src/main/assembly/assembly.xml | 23 --
.../src/main/config/zookeeper/enrichments/asa.json | 0
.../src/main/config/zookeeper/enrichments/bro.json | 0
.../main/config/zookeeper/enrichments/snort.json | 0
.../config/zookeeper/enrichments/websphere.json | 0
.../src/main/config/zookeeper/enrichments/yaf.json | 0
.../adapters/cif/AbstractCIFAdapter.java | 2 +-
.../enrichment/adapters/cif/CIFHbaseAdapter.java | 2 +-
.../metron/enrichment/adapters/geo/GeoAdapter.java | 2 +-
.../adapters/host/AbstractHostAdapter.java | 2 +-
.../adapters/host/HostFromJSONListAdapter.java | 2 +-
.../host/HostFromPropertiesFileAdapter.java | 2 +-
.../enrichment/adapters/jdbc/BaseJdbcConfig.java | 0
.../enrichment/adapters/jdbc/JdbcAdapter.java | 2 +-
.../enrichment/adapters/jdbc/JdbcConfig.java | 0
.../enrichment/adapters/jdbc/MySqlConfig.java | 0
.../adapters/maxmind/MaxMindDatabase.java | 0
.../adapters/maxmind/MaxMindDbUtilities.java | 0
.../adapters/maxmind/asn/GeoLiteAsnDatabase.java | 0
.../adapters/maxmind/geo/GeoLiteCityDatabase.java | 0
.../maxmind/geo/hash/DistanceStrategies.java | 0
.../maxmind/geo/hash/DistanceStrategy.java | 0
.../adapters/maxmind/geo/hash/GeoHashUtil.java | 0
.../adapters/simplehbase/SimpleHBaseAdapter.java | 2 +-
.../adapters/simplehbase/SimpleHBaseConfig.java | 0
.../adapters/stellar/StellarAdapter.java | 8 +-
.../adapters/threatintel/ThreatIntelAdapter.java | 2 +-
.../adapters/threatintel/ThreatIntelConfig.java | 0
.../apache/metron/enrichment/cache}/CacheKey.java | 2 +-
.../metron/enrichment/cli/LatencySummarizer.java | 0
.../enrichment/configuration/Enrichment.java | 0
.../enrichment/converter/AbstractConverter.java | 0
.../enrichment/converter/EnrichmentConverter.java | 0
.../enrichment/converter/EnrichmentHelper.java | 0
.../metron/enrichment/converter/EnrichmentKey.java | 0
.../enrichment/converter/EnrichmentValue.java | 0
.../enrichment/converter/HbaseConverter.java | 0
.../enrichment/interfaces/EnrichmentAdapter.java | 0
.../metron/enrichment/lookup/EnrichmentLookup.java | 0
.../apache/metron/enrichment/lookup/Lookup.java | 0
.../apache/metron/enrichment/lookup/LookupKV.java | 0
.../apache/metron/enrichment/lookup/LookupKey.java | 0
.../metron/enrichment/lookup/LookupValue.java | 0
.../lookup/accesstracker/AccessTracker.java | 0
.../lookup/accesstracker/AccessTrackerCreator.java | 0
.../lookup/accesstracker/AccessTrackerUtil.java | 0
.../lookup/accesstracker/AccessTrackers.java | 0
.../lookup/accesstracker/BloomAccessTracker.java | 0
.../lookup/accesstracker/NoopAccessTracker.java | 0
.../accesstracker/PersistentAccessTracker.java | 0
.../PersistentBloomTrackerCreator.java | 0
.../metron/enrichment/lookup/handler/Handler.java | 0
.../enrichment/lookup/handler/KeyWithContext.java | 0
.../enrichment/parallel/ConcurrencyContext.java | 2 +-
.../enrichment/parallel/EnrichmentCallable.java | 2 +-
.../enrichment/parallel/EnrichmentContext.java | 0
.../enrichment/parallel/EnrichmentStrategies.java | 0
.../enrichment/parallel/EnrichmentStrategy.java | 0
.../enrichment/parallel/ParallelEnricher.java | 2 +-
.../enrichment/parallel/WorkerPoolStrategies.java | 0
.../enrichment/stellar/AsnEnrichmentFunctions.java | 0
.../enrichment/stellar/GeoEnrichmentFunctions.java | 0
.../enrichment/stellar/GeoHashFunctions.java | 0
.../metron/enrichment/stellar/ObjectGet.java | 0
.../stellar/SimpleHBaseEnrichmentFunctions.java | 0
.../metron/enrichment/utils/EnrichmentUtils.java | 0
.../metron/enrichment/utils/ThreatIntelUtils.java | 0
.../threatintel/triage/ThreatTriageProcessor.java | 0
.../src/main/resources/META-INF/LICENSE | 0
.../src/main/resources/META-INF/NOTICE | 0
.../src/main/scripts/latency_summarizer.sh | 0
.../adapters/host/HostFromJSONListAdapterTest.java | 2 +-
.../host/HostFromPropertiesFileAdapterTest.java | 2 +-
.../enrichment/adapters/jdbc/MySqlConfigTest.java | 0
.../maxmind/asn/GeoLiteAsnDatabaseTest.java | 0
.../adapters/maxmind/geo/GeoAdapterTest.java | 2 +-
.../maxmind/geo/GeoLiteCityDatabaseTest.java | 0
.../simplehbase/SimpleHBaseAdapterTest.java | 2 +-
.../simplehbase/SimpleHBaseConfigTest.java | 0
.../adapters/stellar/StellarAdapterTest.java | 0
.../threatintel/ThreatIntelAdapterTest.java | 2 +-
.../threatintel/ThreatIntelConfigTest.java | 0
.../converter/EnrichmentConverterTest.java | 0
.../integration/mock/MockHBaseConnector.java | 0
.../enrichment/parallel/ParallelEnricherTest.java | 2 +-
.../stellar/AsnEnrichmentFunctionsTest.java | 0
.../enrichment/stellar/DocumentationGenerator.java | 0
.../stellar/GeoEnrichmentFunctionsTest.java | 0
.../enrichment/stellar/GeoHashFunctionsTest.java | 0
.../metron/enrichment/stellar/ObjectGetTest.java | 0
.../SimpleHBaseEnrichmentFunctionsTest.java | 0
.../threatintel/triage/ThreatTriageTest.java | 0
.../test/resources/CIFHbaseAdapterTest.properties | 0
.../src/test/resources/GeoLite/GeoLite2-ASN.tar.gz | Bin
.../test/resources/GeoLite/GeoLite2-City.mmdb.gz | Bin
.../test/resources/GeoLite/GeoLite2-City.tar.gz | Bin
.../test/resources/TestSchemas/CIFHbaseSchema.json | 0
.../resources/TestSchemas/WhoisHbaseSchema.json | 0
.../resources/WhoisHbaseAdapterTest.properties | 0
.../src/test/resources/log4j.properties | 0
.../{ => metron-enrichment-storm}/Performance.md | 0
.../metron-enrichment-storm/README.md | 143 ++++++++
.../enrichment_arch.png | Bin
.../{ => metron-enrichment-storm}/pom.xml | 220 +++++------
.../src/main/assembly/assembly.xml | 0
.../main/config/enrichment-splitjoin.properties | 0
.../main/config/enrichment-splitjoin.properties.j2 | 0
.../src/main/config/enrichment-unified.properties | 0
.../main/config/enrichment-unified.properties.j2 | 0
.../src/main/flux/enrichment/remote-splitjoin.yaml | 0
.../src/main/flux/enrichment/remote-unified.yaml | 0
.../metron/enrichment/bolt/EnrichmentJoinBolt.java | 0
.../enrichment/bolt/EnrichmentSplitterBolt.java | 0
.../enrichment/bolt/GenericEnrichmentBolt.java | 4 +-
.../apache/metron/enrichment/bolt/JoinBolt.java | 0
.../apache/metron/enrichment/bolt/SplitBolt.java | 0
.../enrichment/bolt/ThreatIntelJoinBolt.java | 0
.../enrichment/bolt/ThreatIntelSplitterBolt.java | 0
.../enrichment/bolt/UnifiedEnrichmentBolt.java | 5 +-
.../src/main/resources/META-INF/LICENSE | 0
.../src/main/resources/META-INF/NOTICE | 0
.../src/main/scripts/start_enrichment_topology.sh | 0
.../enrichment/bolt/EnrichmentJoinBoltTest.java | 21 +-
.../bolt/EnrichmentSplitterBoltTest.java | 21 +-
.../enrichment/bolt/GenericEnrichmentBoltTest.java | 42 ++-
.../metron/enrichment/bolt/JoinBoltTest.java | 1 -
.../metron/enrichment/bolt/SplitBoltTest.java | 0
.../enrichment/bolt/ThreatIntelJoinBoltTest.java | 27 +-
.../bolt/ThreatIntelSplitterBoltTest.java | 4 +-
.../integration/EnrichmentIntegrationTest.java | 15 +-
.../UnifiedEnrichmentIntegrationTest.java | 4 +-
.../unified_enrichment_arch.svg | 0
.../unified_enrichment_arch_diagram.xml | 0
metron-platform/metron-enrichment/pom.xml | 407 ++-------------------
.../apache/metron/hbase/client/HBaseClient.java | 6 +-
metron-platform/metron-indexing/pom.xml | 52 ++-
.../integration/IndexingIntegrationTest.java | 18 +-
metron-platform/metron-management/pom.xml | 9 +
.../metron-parsing/metron-parsers-common/pom.xml | 21 +-
.../metron-parsing/metron-parsing-storm/pom.xml | 8 +-
...SimpleHbaseEnrichmentWriterIntegrationTest.java | 4 +-
.../integration/WriterBoltIntegrationTest.java | 2 +-
metron-platform/metron-pcap/pom.xml | 4 +
metron-platform/metron-solr/pom.xml | 141 ++++---
.../enrichment/integration/utils/SampleUtil.java | 0
.../main/java/org/apache/metron/TestConstants.java | 2 +-
metron-platform/metron-writer/pom.xml | 19 +-
.../writer/hbase}/SimpleHbaseEnrichmentWriter.java | 5 +-
.../hbase}/SimpleHBaseEnrichmentWriterTest.java | 18 +-
.../apache/metron/writer/hdfs/HdfsWriterTest.java | 4 +
site-book/bin/generate-md.sh | 8 +-
165 files changed, 737 insertions(+), 1481 deletions(-)
diff --git a/Upgrading.md b/Upgrading.md
index fcd5f27..b3ab8d2 100644
--- a/Upgrading.md
+++ b/Upgrading.md
@@ -19,7 +19,13 @@ limitations under the License.
This document constitutes a per-version listing of changes of
configuration which are non-backwards compatible.
+## 0.7.1 to 0.7.2
+
+### [METRON-2053: Refactor metron-enrichment to decouple Storm dependencies](https://issues.apache.org/jira/browse/METRON-2053)
+`org.apache.metron.enrichment.writer.SimpleHbaseEnrichmentWriter` has had its packaged changed to `org.apache.metron.writer.hbase.SimpleHbaseEnrichmentWriter`. It has also been moved from the `metron-platform/metron-enrichment` module to a more appropriate home in `metron-platform/metron-writer`.
+
## 0.7.0 to 0.7.1
+
### [METRON-1929: Build GET_ASN Stellar function](https://issues.apache.org/jira/browse/METRON-1929)
The script for `geo_enrichment_load.sh` has been renamed, and now is `maxmind_enrichment_load.sh`. A couple changes should happen for users who are upgrading.
diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
index 25aa7ad..2568bf2 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
@@ -108,7 +108,7 @@
<outputDirectory>${basedir}/src/main/resources/common-services/METRON/CURRENT/package/templates</outputDirectory>
<resources>
<resource>
- <directory>${basedir}/../../../../metron-platform/metron-enrichment/src/main/config</directory>
+ <directory>${basedir}/../../../../metron-platform/metron-enrichment/metron-enrichment-storm/src/main/config</directory>
<includes>
<include>enrichment-splitjoin.properties.j2</include>
<include>enrichment-unified.properties.j2</include>
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index 99e9325..5112868 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -453,7 +453,10 @@
<name>metron-parsing-storm</name>
</package>
<package>
- <name>metron-enrichment</name>
+ <name>metron-enrichment-common</name>
+ </package>
+ <package>
+ <name>metron-enrichment-storm</name>
</package>
<package>
<name>metron-profiler-storm</name>
diff --git a/metron-deployment/packaging/docker/deb-docker/pom.xml b/metron-deployment/packaging/docker/deb-docker/pom.xml
index 5062a51..98ccd8c 100644
--- a/metron-deployment/packaging/docker/deb-docker/pom.xml
+++ b/metron-deployment/packaging/docker/deb-docker/pom.xml
@@ -90,7 +90,13 @@
</includes>
</resource>
<resource>
- <directory>${metron_dir}/metron-platform/metron-enrichment/target/</directory>
+ <directory>${metron_dir}/metron-platform/metron-enrichment/metron-enrichment-common/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-enrichment/metron-enrichment-storm/target/</directory>
<includes>
<include>*.tar.gz</include>
</includes>
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index ddd8170..1c40085 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -48,20 +48,21 @@ Source1: metron-parsers-common-%{full_version}-archive.tar.gz
Source2: metron-elasticsearch-%{full_version}-archive.tar.gz
Source3: metron-data-management-%{full_version}-archive.tar.gz
Source4: metron-solr-%{full_version}-archive.tar.gz
-Source5: metron-enrichment-%{full_version}-archive.tar.gz
-Source6: metron-indexing-%{full_version}-archive.tar.gz
-Source7: metron-pcap-backend-%{full_version}-archive.tar.gz
-Source8: metron-profiler-storm-%{full_version}-archive.tar.gz
-Source9: metron-rest-%{full_version}-archive.tar.gz
-Source10: metron-config-%{full_version}-archive.tar.gz
-Source11: metron-management-%{full_version}-archive.tar.gz
-Source12: metron-maas-service-%{full_version}-archive.tar.gz
-Source13: metron-alerts-%{full_version}-archive.tar.gz
-Source14: metron-performance-%{full_version}-archive.tar.gz
-Source15: metron-profiler-spark-%{full_version}-archive.tar.gz
-Source16: metron-profiler-repl-%{full_version}-archive.tar.gz
-Source17: metron-parsing-storm-%{full_version}-archive.tar.gz
-Source18: metron-parsers-%{full_version}-archive.tar.gz
+Source5: metron-enrichment-common-%{full_version}-archive.tar.gz
+Source6: metron-enrichment-storm-%{full_version}-archive.tar.gz
+Source7: metron-indexing-%{full_version}-archive.tar.gz
+Source8: metron-pcap-backend-%{full_version}-archive.tar.gz
+Source9: metron-profiler-storm-%{full_version}-archive.tar.gz
+Source10: metron-rest-%{full_version}-archive.tar.gz
+Source11: metron-config-%{full_version}-archive.tar.gz
+Source12: metron-management-%{full_version}-archive.tar.gz
+Source13: metron-maas-service-%{full_version}-archive.tar.gz
+Source14: metron-alerts-%{full_version}-archive.tar.gz
+Source15: metron-performance-%{full_version}-archive.tar.gz
+Source16: metron-profiler-spark-%{full_version}-archive.tar.gz
+Source17: metron-profiler-repl-%{full_version}-archive.tar.gz
+Source18: metron-parsing-storm-%{full_version}-archive.tar.gz
+Source19: metron-parsers-%{full_version}-archive.tar.gz
%description
Apache Metron provides a scalable advanced security analytics framework
@@ -103,6 +104,7 @@ tar -xzf %{SOURCE15} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE16} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE17} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE18} -C %{buildroot}%{metron_home}
+tar -xzf %{SOURCE19} -C %{buildroot}%{metron_home}
install %{buildroot}%{metron_home}/bin/metron-management-ui %{buildroot}/etc/init.d/
install %{buildroot}%{metron_home}/bin/metron-alerts-ui %{buildroot}/etc/init.d/
@@ -318,15 +320,15 @@ This package installs the Metron Solr files
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-%package enrichment
-Summary: Metron Enrichment Files
+%package enrichment-common
+Summary: Metron Enrichment Common Files
Group: Applications/Internet
-Provides: enrichment = %{version}
+Provides: enrichment-common = %{version}
-%description enrichment
-This package installs the Metron Enrichment files
+%description enrichment-common
+This package installs the Metron Enrichment Common files
-%files enrichment
+%files enrichment-common
%defattr(-,root,root,755)
%dir %{metron_root}
%dir %{metron_home}
@@ -334,20 +336,37 @@ This package installs the Metron Enrichment files
%dir %{metron_home}/config
%dir %{metron_home}/config/zookeeper
%dir %{metron_home}/config/zookeeper/enrichments
-%dir %{metron_home}/flux
-%dir %{metron_home}/flux/enrichment
%{metron_home}/bin/latency_summarizer.sh
-%{metron_home}/bin/start_enrichment_topology.sh
-%{metron_home}/config/enrichment-splitjoin.properties
-%{metron_home}/config/enrichment-unified.properties
%{metron_home}/config/zookeeper/enrichments/bro.json
%{metron_home}/config/zookeeper/enrichments/snort.json
%{metron_home}/config/zookeeper/enrichments/websphere.json
%{metron_home}/config/zookeeper/enrichments/yaf.json
%{metron_home}/config/zookeeper/enrichments/asa.json
+%attr(0644,root,root) %{metron_home}/lib/metron-enrichment-common-%{full_version}-uber.jar
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+%package enrichment-storm
+Summary: Metron Enrichment Storm Files
+Group: Applications/Internet
+Provides: enrichment-storm = %{version}
+
+%description enrichment-storm
+This package installs the Metron Enrichment Storm files
+
+%files enrichment-storm
+%defattr(-,root,root,755)
+%dir %{metron_root}
+%dir %{metron_home}
+%dir %{metron_home}/bin
+%dir %{metron_home}/flux
+%dir %{metron_home}/flux/enrichment
+%{metron_home}/bin/start_enrichment_topology.sh
+%{metron_home}/config/enrichment-splitjoin.properties
+%{metron_home}/config/enrichment-unified.properties
%{metron_home}/flux/enrichment/remote-splitjoin.yaml
%{metron_home}/flux/enrichment/remote-unified.yaml
-%attr(0644,root,root) %{metron_home}/lib/metron-enrichment-%{full_version}-uber.jar
+%attr(0644,root,root) %{metron_home}/lib/metron-enrichment-storm-%{full_version}-uber.jar
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -644,6 +663,8 @@ chkconfig --del metron-management-ui
chkconfig --del metron-alerts-ui
%changelog
+* Tue Mar 12 2019 Apache Metron <de...@metron.apache.og> - 0.7.1
+- Split metron-enrichment into submodules
* Thu Dec 27 2018 Apache Metron <de...@metron.apache.og> - 0.7.1
- Updat metron SPEC to move syslog configurations to right place
* Wed Dec 26 2018 Apache Metron <de...@metron.apache.org> - 0.7.1
diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml
index 8e70686..d8d117a 100644
--- a/metron-deployment/packaging/docker/rpm-docker/pom.xml
+++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml
@@ -126,7 +126,13 @@
</includes>
</resource>
<resource>
- <directory>${metron_dir}/metron-platform/metron-enrichment/target/</directory>
+ <directory>${metron_dir}/metron-platform/metron-enrichment/metron-enrichment-common/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-enrichment/metron-enrichment-storm/target/</directory>
<includes>
<include>*.tar.gz</include>
</includes>
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HBaseConfigTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HBaseConfigTest.java
index 835aa8d..a527e6d 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HBaseConfigTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HBaseConfigTest.java
@@ -17,6 +17,16 @@
*/
package org.apache.metron.rest.config;
+import static org.apache.metron.hbase.client.UserSettingsClient.USER_SETTINGS_HBASE_CF;
+import static org.apache.metron.hbase.client.UserSettingsClient.USER_SETTINGS_HBASE_TABLE;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.whenNew;
+
+import java.util.HashMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.metron.hbase.HTableProvider;
@@ -27,14 +37,6 @@ import org.junit.runner.RunWith;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
-import java.util.HashMap;
-
-import static org.apache.metron.hbase.client.UserSettingsClient.USER_SETTINGS_HBASE_CF;
-import static org.apache.metron.hbase.client.UserSettingsClient.USER_SETTINGS_HBASE_TABLE;
-import static org.mockito.Mockito.*;
-import static org.powermock.api.mockito.PowerMockito.mockStatic;
-import static org.powermock.api.mockito.PowerMockito.whenNew;
-
@RunWith(PowerMockRunner.class)
@PrepareForTest({HTableProvider.class, HBaseConfiguration.class, HBaseConfig.class})
public class HBaseConfigTest {
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
index a0b5bce..75d7caf 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
@@ -34,6 +34,7 @@ public class Constants {
public static final String ERROR_TYPE = "error";
public static final String SIMPLE_HBASE_ENRICHMENT = "hbaseEnrichment";
public static final String SIMPLE_HBASE_THREAT_INTEL = "hbaseThreatIntel";
+ public static final String STELLAR_CONTEXT_CONF = "stellarContext";
public static final String GUID = "guid";
/**
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/components/ConfigUploadComponent.java b/metron-platform/metron-common/src/test/java/org/apache/metron/integration/components/ConfigUploadComponent.java
similarity index 92%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/components/ConfigUploadComponent.java
rename to metron-platform/metron-common/src/test/java/org/apache/metron/integration/components/ConfigUploadComponent.java
index e4625ac..c9f9f70 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/components/ConfigUploadComponent.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/integration/components/ConfigUploadComponent.java
@@ -15,24 +15,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.metron.enrichment.integration.components;
+package org.apache.metron.integration.components;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
-import org.apache.metron.common.configuration.SensorParserConfig;
-import org.apache.metron.integration.InMemoryComponent;
-import org.apache.metron.integration.UnableToStartException;
-import org.apache.metron.integration.components.ZKServerComponent;
-import org.apache.zookeeper.KeeperException;
+import static org.apache.metron.common.configuration.ConfigurationsUtils.getClient;
+import static org.apache.metron.common.configuration.ConfigurationsUtils.readSensorParserConfigFromZookeeper;
+import static org.apache.metron.common.configuration.ConfigurationsUtils.uploadConfigsToZookeeper;
+import static org.apache.metron.common.configuration.ConfigurationsUtils.writeGlobalConfigToZookeeper;
+import static org.apache.metron.common.configuration.ConfigurationsUtils.writeSensorParserConfigToZookeeper;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.function.Consumer;
-import java.util.function.Function;
-
-import static org.apache.metron.common.configuration.ConfigurationsUtils.*;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.integration.InMemoryComponent;
+import org.apache.metron.integration.UnableToStartException;
public class ConfigUploadComponent implements InMemoryComponent {
diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml
index 22e3947..c4b2bf6 100644
--- a/metron-platform/metron-data-management/pom.xml
+++ b/metron-platform/metron-data-management/pom.xml
@@ -88,7 +88,7 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-enrichment</artifactId>
+ <artifactId>metron-enrichment-common</artifactId>
<version>${project.parent.version}</version>
<exclusions>
<exclusion>
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index c387d1e..31c160c 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -182,6 +182,11 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${global_httpclient_version}</version>
+ </dependency>
+ <dependency>
<groupId>org.elasticsearch.plugin</groupId>
<artifactId>transport-netty4-client</artifactId>
<version>${global_elasticsearch_version}</version>
@@ -246,7 +251,7 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-enrichment</artifactId>
+ <artifactId>metron-common</artifactId>
<version>${project.parent.version}</version>
<type>test-jar</type>
<scope>test</scope>
diff --git a/metron-platform/metron-enrichment/README.md b/metron-platform/metron-enrichment/README.md
index 9e35f48..1e2d055 100644
--- a/metron-platform/metron-enrichment/README.md
+++ b/metron-platform/metron-enrichment/README.md
@@ -19,8 +19,8 @@ limitations under the License.
## Introduction
-The `enrichment` topology is a topology dedicated to taking the data
-from the parsing topologies that have been normalized into the Metron
+The `enrichment` module is a module dedicated to taking the data
+from the parsers that have been normalized into the Metron
data format (e.g. a JSON Map structure with `original_message` and
`timestamp`) and
* Enriching messages with external data from data stores (e.g. hbase) by
@@ -29,398 +29,12 @@ data format (e.g. a JSON Map structure with `original_message` and
* Marking threat alerts with a numeric triage level based on a set of
Stellar rules.
-## Enrichment Architecture
+## Deployment Options
-
+There is currently one option for running enrichments in Metron, which is as a Storm topology.
-### Unified Enrichment Topology
+## Submodules
-The unified enrichment topology uses data parallelism as opposed to the deprecated
-split/join topology's task parallelism. This architecture uses a worker pool to fully
-enrich any message within a worker. This results in
-* Fewer bolts in the topology
-* Each bolt fully operates on a message.
-* Fewer network hops
+* metron-enrichment-common - this module houses the prepackaged enrichment configuration by sensor. It also contains the core enrichment and threat intelligence processing functionality.
+* metron-common-storm - this module is home to Storm-specific code such as Flux files and Storm Bolts.
-This architecture is fully backwards compatible with the old split-join
-topology; the only difference is how the enrichment will operate on each
-message (in one bolt where the split/join is done in a threadpool as opposed
-to split across multiple bolts).
-
-#### Configuring It
-
-There are two parameters which you might want to tune in this topology.
-Both of them are topology configuration adjustable in the flux file
-`$METRON_HOME/config/flux/enrichment/remote-unified.yaml`:
-* `metron.threadpool.size` : The size of the threadpool. This can take a number or a multiple of the number of cores (e.g. `5C` to 5 times the number of cores). The default is `2C`.
-* `metron.threadpool.type` : The type of threadpool. (note: descriptions taken from [here](https://zeroturnaround.com/rebellabs/fixedthreadpool-cachedthreadpool-or-forkjoinpool-picking-correct-java-executors-for-background-tasks/)).
- * `FIXED` is a fixed threadpool of size `n`. `n` threads will process tasks at the time, when the pool is saturated, new tasks will get added to a queue without a limit on size. Good for CPU intensive tasks. This is the default.
- * `WORK_STEALING` is a work stealing threadpool. This will create and shut down threads dynamically to accommodate the required parallelism level. It also tries to reduce the contention on the task queue, so can be really good in heavily loaded environments. Also good when your tasks create more tasks for the executor, like recursive tasks.
-
-In order to configure the parallelism for the enrichment bolt and threat
-intel bolt, the configurations will be taken from the respective join bolt
-parallelism. When proper ambari support for this is added, we will add
-its own property.
-
-### Split-Join Enrichment Topology
-
-The now-deprecated split/join topology is also available and performs enrichments in parallel.
-This poses some issues in terms of ease of tuning and reasoning about performance.
-
-
-
-#### Using It
-
-In order to use the older, deprecated topology, you will need to
-* Edit `$METRON_HOME/bin/start_enrichment_topology.sh` and adjust it to use `remote-splitjoin.yaml` instead of `remote-unified.yaml`
-* Restart the enrichment topology.
-
-## Enrichment Configuration
-
-The configuration for the `enrichment` topology, the topology primarily
-responsible for enrichment and threat intelligence enrichment, is
-defined by JSON documents stored in zookeeper.
-
-There are two types of configurations at the moment, `global` and
-`sensor` specific.
-
-## Global Configuration
-
-There are a few enrichments which have independent configurations, such
-as from the global config. You can also configure the enrichment topology's
-writer batching settings.
-
-Also, see the "[Global Configuration](../metron-common)" section for
-more discussion of the global config.
-
-### GeoIP
-Metron supports enrichment of IP information using
-[GeoLite2](https://dev.maxmind.com/geoip/geoip2/geolite2/). The
-location of the file is managed in the global config.
-
-#### `geo.hdfs.file`
-
-The location on HDFS of the GeoLite2 database file to use for GeoIP
-lookups. This file will be localized on the storm supervisors running
-the topology and used from there. This is lazy, so if this property
-changes in a running topology, the file will be localized from HDFS upon first
-time the file is used via the geo enrichment.
-
-### Writer Batching
-
-#### `enrichment.writer.batchSize`
-
-The size of the batch that is written to Kafka at once. Defaults to `15` (size of 1 disables batching).
-
-#### `enrichment.writer.batchTimeout`
-
-The timeout after which a batch will be flushed even if batchSize has not been met. Optional.
-If unspecified, or set to `0`, it defaults to a system-determined duration which is a fraction of the Storm
-parameter `topology.message.timeout.secs`. Ignored if batchSize is `1`, since this disables batching.
-
-## Sensor Enrichment Configuration
-
-The sensor specific configuration is intended to configure the
-individual enrichments and threat intelligence enrichments for a given
-sensor type (e.g. `snort`).
-
-Just like the global config, the format is a JSON stored in zookeeper.
-The configuration is a complex JSON object with the following top level fields:
-
-* `enrichment` : A complex JSON object representing the configuration of the enrichments
-* `threatIntel` : A complex JSON object representing the configuration of the threat intelligence enrichments
-
-### The `enrichment` Configuration
-
-| Field | Description | Example |
-|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------|
-| `fieldToTypeMap` | In the case of a simple HBase enrichment (i.e. a key/value lookup), the mapping between fields and the enrichment types associated with those fields must be known. This enrichment type is used as part of the HBase key. Note: applies to hbaseEnrichment only. | `"fieldToTypeMap" : { "ip_src_addr" : [ "asset_enrichment" ] }` |
-| `fieldMap` | The map of enrichment bolts names to configuration handlers which know how to split the message up. The simplest of which is just a list of fields. More complex examples would be the stellar enrichment which provides stellar statements. Each field listed in the array arg is sent to the enrichment referenced in the key. Cardinality of fields to enrichments is many-to-many. | `"fieldMap": {"hbaseEnrichment": ["ip_src_addr","ip_dst_addr"]}` |
-| `config` | The general configuration for the enrichment | `"config": {"typeToColumnFamily": { "asset_enrichment" : "cf" } }` |
-
-The `config` map is intended to house enrichment specific configuration.
-For instance, for the `hbaseEnrichment`, the mappings between the
-enrichment types to the column families is specified.
-
-The `fieldMap`contents are of interest because they contain the routing and configuration information for the enrichments.
-When we say 'routing', we mean how the messages get split up and sent to the enrichment adapter bolts.
-The simplest, by far, is just providing a simple list as in
-```
- "fieldMap": {
- "geo": [
- "ip_src_addr",
- "ip_dst_addr"
- ],
- "host": [
- "ip_src_addr",
- "ip_dst_addr"
- ],
- "hbaseEnrichment": [
- "ip_src_addr",
- "ip_dst_addr"
- ]
- }
-```
-Based on this sample config, both `ip_src_addr` and `ip_dst_addr` will go to the `geo`, `host`, and
-`hbaseEnrichment` adapter bolts.
-
-#### Stellar Enrichment Configuration
-For the `geo`, `host` and `hbaseEnrichment`, this is sufficient. However, more complex enrichments
-may contain their own configuration. Currently, the `stellar` enrichment is more adaptable and thus
-requires a more nuanced configuration.
-
-At its most basic, we want to take a message and apply a couple of enrichments, such as converting the
-`hostname` field to lowercase. We do this by specifying the transformation inside of the
-`config` for the `stellar` fieldMap. There are two syntaxes that are supported, specifying the transformations
-as a map with the key as the field and the value the stellar expression:
-```
- "fieldMap": {
- ...
- "stellar" : {
- "config" : {
- "hostname" : "TO_LOWER(hostname)"
- }
- }
- }
-```
-
-Another approach is to make the transformations as a list with the same `var := expr` syntax as is used
-in the Stellar REPL, such as:
-```
- "fieldMap": {
- ...
- "stellar" : {
- "config" : [
- "hostname := TO_LOWER(hostname)"
- ]
- }
- }
-```
-
-Sometimes arbitrary stellar enrichments may take enough time that you would prefer to split some of them
-into groups and execute the groups of stellar enrichments in parallel. Take, for instance, if you wanted
-to do an HBase enrichment and a profiler call which were independent of one another. This usecase is
-supported by splitting the enrichments up as groups.
-
-Consider the following example:
-```
- "fieldMap": {
- ...
- "stellar" : {
- "config" : {
- "malicious_domain_enrichment" : {
- "is_bad_domain" : "ENRICHMENT_EXISTS('malicious_domains', ip_dst_addr, 'enrichments', 'cf')"
- },
- "login_profile" : [
- "profile_window := PROFILE_WINDOW('from 6 months ago')",
- "global_login_profile := PROFILE_GET('distinct_login_attempts', 'global', profile_window)",
- "stats := STATS_MERGE(global_login_profile)",
- "auth_attempts_median := STATS_PERCENTILE(stats, 0.5)",
- "auth_attempts_sd := STATS_SD(stats)",
- "profile_window := null",
- "global_login_profile := null",
- "stats := null"
- ]
- }
- }
- }
-```
-
-Here we want to perform two enrichments that hit HBase and we would rather not run in sequence. These
-enrichments are entirely independent of one another (i.e. neither relies on the output of the other). In
-this case, we've created a group called `malicious_domain_enrichment` to inquire about whether the destination
-address exists in the HBase enrichment table in the `malicious_domains` enrichment type. This is a simple
-enrichment, so we can express the enrichment group as a map with the new field `is_bad_domain` being a key
-and the stellar expression associated with that operation being the associated value.
-
-In contrast, the stellar enrichment group `login_profile` is interacting with the profiler, has multiple temporary
-expressions (i.e. `profile_window`, `global_login_profile`, and `stats`) that are useful only within the context
-of this group of stellar expressions. In this case, we would need to ensure that we use the list construct
-when specifying the group and remember to set the temporary variables to `null` so they are not passed along.
-
-In general, things to note from this section are as follows:
-* The stellar enrichments for the `stellar` enrichment adapter are specified in the `config` for the `stellar` enrichment
-adapter in the `fieldMap`
-* Groups of independent (i.e. no expression in any group depend on the output of an expression from an other group) may be executed in parallel
-* If you have the need to use temporary variables, you may use the list construct. Ensure that you assign the variables to `null` before the end of the group.
-* **Ensure that you do not assign a field to a stellar expression which returns an object which JSON cannot represent.**
-* Fields assigned to Maps as part of stellar enrichments have the maps unfolded, similar to the HBase Enrichment
- * For example the stellar enrichment for field `foo` which assigns a map such as `foo := { 'grok' : 1, 'bar' : 'baz'}`
- would yield the following fields:
- * `foo.grok` == `1`
- * `foo.bar` == `'baz'`
-
-### The `threatIntel` Configuration
-
-| Field | Description | Example |
-|------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------|
-| `fieldToTypeMap` | In the case of a simple HBase threat intel enrichment (i.e. a key/value lookup), the mapping between fields and the enrichment types associated with those fields must be known. This enrichment type is used as part of the HBase key. Note: applies to hbaseThreatIntel only. | `"fieldToTypeMap" : { "ip_src_addr" : [ "malicious_ips" ] }` |
-| `fieldMap` | The map of threat intel enrichment bolts names to fields in the JSON messages. Each field is sent to the threat intel enrichment bolt referenced in the key. Each field listed in the array arg is sent to the enrichment referenced in the key. Cardinality of fields to enrichments is many-to-many. | `"fieldMap": {"hbaseThreatIntel": ["ip_src_addr","ip_dst_addr"]}` |
-| `triageConfig` | The configuration of the threat triage scorer. In the situation where a threat is detected, a score is assigned to the message and embedded in the indexed message. | `"riskLevelRules" : { "IN_SUBNET(ip_dst_addr, '192.168.0.0/24')" : 10 }` |
-| `config` | The general configuration for the Threat Intel | `"config": {"typeToColumnFamily": { "malicious_ips","cf" } }` |
-
-The `config` map is intended to house threat intel specific configuration.
-For instance, for the `hbaseThreatIntel` threat intel adapter, the mappings between the
-enrichment types to the column families is specified. The `fieldMap` configuration is similar to the `enrichment`
-configuration in that the adapters available are the same.
-
-The `triageConfig` field is also a complex field and it bears some description:
-
-| Field | Description | Example |
-|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------|
-| `riskLevelRules` | This is a list of rules (represented as Stellar expressions) associated with scores with optional names and comments | see below|
-| `aggregator` | An aggregation function that takes all non-zero scores representing the matching queries from `riskLevelRules` and aggregates them into a single score. | `"MAX"` |
-
-A risk level rule is of the following format:
-* `name` : The name of the threat triage rule
-* `comment` : A comment describing the rule
-* `rule` : The rule, represented as a Stellar statement
-* `score` : The score attributed to the rule. Can be either numeric or a Stellar expression. The expression has access to all fields with the message being triaged.
-* `reason` : Reason the rule tripped. Can be represented as a Stellar statement
-
-An example of a rule is as follows:
-```
- "riskLevelRules" : [
- {
- "name" : "is internal",
- "comment" : "determines if the destination is internal.",
- "rule" : "IN_SUBNET(ip_dst_addr, '192.168.0.0/24')",
- "score" : 10,
- "reason" : "FORMAT('%s is internal', ip_dst_addr)"
- }
- ]
-```
-
-The supported aggregation functions are:
-* `MAX` : The max of all of the associated values for matching queries
-* `MIN` : The min of all of the associated values for matching queries
-* `MEAN` : The mean of all of the associated values for matching queries
-* `SUM` : The sum of all the associated values for matching queries
-* `POSITIVE_MEAN` : The mean of the positive associated values for the matching queries.
-
-### Example Configuration
-
-An example configuration for the YAF sensor is as follows:
-```json
-{
- "enrichment": {
- "fieldMap": {
- "geo": [
- "ip_src_addr",
- "ip_dst_addr"
- ],
- "host": [
- "ip_src_addr",
- "ip_dst_addr"
- ],
- "hbaseEnrichment": [
- "ip_src_addr",
- "ip_dst_addr"
- ]
- }
- ,"fieldToTypeMap": {
- "ip_src_addr": [
- "playful_classification"
- ],
- "ip_dst_addr": [
- "playful_classification"
- ]
- }
- },
- "threatIntel": {
- "fieldMap": {
- "hbaseThreatIntel": [
- "ip_src_addr",
- "ip_dst_addr"
- ]
- },
- "fieldToTypeMap": {
- "ip_src_addr": [
- "malicious_ip"
- ],
- "ip_dst_addr": [
- "malicious_ip"
- ]
- },
- "triageConfig" : {
- "riskLevelRules" : [
- {
- "rule" : "ip_src_addr == '10.0.2.3' or ip_dst_addr == '10.0.2.3'",
- "score" : 10
- }
- ],
- "aggregator" : "MAX"
- }
- }
-}
-```
-
-ThreatIntel alert levels are emitted as a new field "threat.triage.level." So for the example above, an incoming message that trips the `ip_src_addr` rule will have a new field threat.triage.level=10.
-
-
-# Example Enrichment via Stellar
-
-Let's walk through doing a simple enrichment using Stellar on your cluster using the Squid topology.
-
-## Install Prerequisites
-Now let's install some prerequisites:
-* Squid client via `yum install squid`
-* ES Head plugin via `/usr/share/elasticsearch/bin/plugin install mobz/elasticsearch-head`
-
-Start Squid via `service squid start`
-
-## Adjust Enrichment Configurations for Squid to Call Stellar
-Let's adjust the configurations for the Squid topology to annotate the messages using some Stellar functions.
-
-* Edit the squid enrichment configuration at `$METRON_HOME/config/zookeeper/enrichments/squid.json` (this file will not exist, so create a new one) to add some new fields based on stellar queries:
-
-```
-{
- "enrichment" : {
- "fieldMap": {
- "stellar" : {
- "config" : {
- "numeric" : {
- "foo": "1 + 1"
- }
- ,"ALL_CAPS" : "TO_UPPER(source.type)"
- }
- }
- }
- },
- "threatIntel" : {
- "fieldMap":{
- "stellar" : {
- "config" : {
- "bar" : "TO_UPPER(source.type)"
- }
- }
- },
- "triageConfig" : {
- }
- }
-}
-```
-We have added the following fields as part of the enrichment phase of the enrichment topology:
-* `foo` == 2
-* `ALL_CAPS` == SQUID
-
-We have added the following as part of the threat intel:
-* ` bar` == SQUID
-
-Please note that foo and ALL_CAPS will be applied in separate workers due to them being in separate groups.
-
-* Upload new configs via `$METRON_HOME/bin/zk_load_configs.sh --mode PUSH -i $METRON_HOME/config/zookeeper -z node1:2181`
-* Make the Squid topic in kafka via `/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper node1:2181 --create --topic squid --partitions 1 --replication-factor 1`
-
-## Start Topologies and Send Data
-Now we need to start the topologies and send some data:
-* Start the squid topology via `$METRON_HOME/bin/start_parser_topology.sh -k node1:6667 -z node1:2181 -s squid`
-* Generate some data via the squid client:
- * `squidclient http://yahoo.com`
- * `squidclient http://cnn.com`
-* Send the data to kafka via `cat /var/log/squid/access.log | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list node1:6667 --topic squid`
-* Browse the data in elasticsearch via the ES Head plugin @ [http://node1:9200/_plugin/head/](http://node1:9200/_plugin/head/) and verify that in the squid index you have two documents
-* Ensure that the documents have new fields `foo`, `bar` and `ALL_CAPS` with values as described above.
-
-Note that we could have used any Stellar statements here, including calling out to HBase via `ENRICHMENT_GET` and `ENRICHMENT_EXISTS` or even calling a machine learning model via [Model as a Service](../../metron-analytics/metron-maas-service).
diff --git a/metron-platform/metron-enrichment/README.md b/metron-platform/metron-enrichment/metron-enrichment-common/README.md
similarity index 71%
copy from metron-platform/metron-enrichment/README.md
copy to metron-platform/metron-enrichment/metron-enrichment-common/README.md
index 9e35f48..c6e6efc 100644
--- a/metron-platform/metron-enrichment/README.md
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/README.md
@@ -15,69 +15,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
-# Enrichment
+# Enrichment Common
## Introduction
-The `enrichment` topology is a topology dedicated to taking the data
-from the parsing topologies that have been normalized into the Metron
-data format (e.g. a JSON Map structure with `original_message` and
-`timestamp`) and
-* Enriching messages with external data from data stores (e.g. hbase) by
- adding new fields based on existing fields in the messages.
-* Marking messages as threats based on data in external data stores
-* Marking threat alerts with a numeric triage level based on a set of
- Stellar rules.
-
-## Enrichment Architecture
-
-
-
-### Unified Enrichment Topology
-
-The unified enrichment topology uses data parallelism as opposed to the deprecated
-split/join topology's task parallelism. This architecture uses a worker pool to fully
-enrich any message within a worker. This results in
-* Fewer bolts in the topology
-* Each bolt fully operates on a message.
-* Fewer network hops
-
-This architecture is fully backwards compatible with the old split-join
-topology; the only difference is how the enrichment will operate on each
-message (in one bolt where the split/join is done in a threadpool as opposed
-to split across multiple bolts).
-
-#### Configuring It
-
-There are two parameters which you might want to tune in this topology.
-Both of them are topology configuration adjustable in the flux file
-`$METRON_HOME/config/flux/enrichment/remote-unified.yaml`:
-* `metron.threadpool.size` : The size of the threadpool. This can take a number or a multiple of the number of cores (e.g. `5C` to 5 times the number of cores). The default is `2C`.
-* `metron.threadpool.type` : The type of threadpool. (note: descriptions taken from [here](https://zeroturnaround.com/rebellabs/fixedthreadpool-cachedthreadpool-or-forkjoinpool-picking-correct-java-executors-for-background-tasks/)).
- * `FIXED` is a fixed threadpool of size `n`. `n` threads will process tasks at the time, when the pool is saturated, new tasks will get added to a queue without a limit on size. Good for CPU intensive tasks. This is the default.
- * `WORK_STEALING` is a work stealing threadpool. This will create and shut down threads dynamically to accommodate the required parallelism level. It also tries to reduce the contention on the task queue, so can be really good in heavily loaded environments. Also good when your tasks create more tasks for the executor, like recursive tasks.
-
-In order to configure the parallelism for the enrichment bolt and threat
-intel bolt, the configurations will be taken from the respective join bolt
-parallelism. When proper ambari support for this is added, we will add
-its own property.
-
-### Split-Join Enrichment Topology
-
-The now-deprecated split/join topology is also available and performs enrichments in parallel.
-This poses some issues in terms of ease of tuning and reasoning about performance.
-
-
-
-#### Using It
-
-In order to use the older, deprecated topology, you will need to
-* Edit `$METRON_HOME/bin/start_enrichment_topology.sh` and adjust it to use `remote-splitjoin.yaml` instead of `remote-unified.yaml`
-* Restart the enrichment topology.
+This module holds shared code for enrichment activities.
## Enrichment Configuration
-The configuration for the `enrichment` topology, the topology primarily
+The configuration for the `enrichment` module, the module primarily
responsible for enrichment and threat intelligence enrichment, is
defined by JSON documents stored in zookeeper.
@@ -90,7 +36,7 @@ There are a few enrichments which have independent configurations, such
as from the global config. You can also configure the enrichment topology's
writer batching settings.
-Also, see the "[Global Configuration](../metron-common)" section for
+Also, see the "[Global Configuration](../../metron-common)" section for
more discussion of the global config.
### GeoIP
@@ -143,7 +89,7 @@ For instance, for the `hbaseEnrichment`, the mappings between the
enrichment types to the column families is specified.
The `fieldMap`contents are of interest because they contain the routing and configuration information for the enrichments.
-When we say 'routing', we mean how the messages get split up and sent to the enrichment adapter bolts.
+When we say 'routing', we mean how the messages get split up and sent to the enrichment adapters.
The simplest, by far, is just providing a simple list as in
```
"fieldMap": {
@@ -162,7 +108,7 @@ The simplest, by far, is just providing a simple list as in
}
```
Based on this sample config, both `ip_src_addr` and `ip_dst_addr` will go to the `geo`, `host`, and
-`hbaseEnrichment` adapter bolts.
+`hbaseEnrichment` adapters.
#### Stellar Enrichment Configuration
For the `geo`, `host` and `hbaseEnrichment`, this is sufficient. However, more complex enrichments
@@ -356,71 +302,3 @@ An example configuration for the YAF sensor is as follows:
```
ThreatIntel alert levels are emitted as a new field "threat.triage.level." So for the example above, an incoming message that trips the `ip_src_addr` rule will have a new field threat.triage.level=10.
-
-
-# Example Enrichment via Stellar
-
-Let's walk through doing a simple enrichment using Stellar on your cluster using the Squid topology.
-
-## Install Prerequisites
-Now let's install some prerequisites:
-* Squid client via `yum install squid`
-* ES Head plugin via `/usr/share/elasticsearch/bin/plugin install mobz/elasticsearch-head`
-
-Start Squid via `service squid start`
-
-## Adjust Enrichment Configurations for Squid to Call Stellar
-Let's adjust the configurations for the Squid topology to annotate the messages using some Stellar functions.
-
-* Edit the squid enrichment configuration at `$METRON_HOME/config/zookeeper/enrichments/squid.json` (this file will not exist, so create a new one) to add some new fields based on stellar queries:
-
-```
-{
- "enrichment" : {
- "fieldMap": {
- "stellar" : {
- "config" : {
- "numeric" : {
- "foo": "1 + 1"
- }
- ,"ALL_CAPS" : "TO_UPPER(source.type)"
- }
- }
- }
- },
- "threatIntel" : {
- "fieldMap":{
- "stellar" : {
- "config" : {
- "bar" : "TO_UPPER(source.type)"
- }
- }
- },
- "triageConfig" : {
- }
- }
-}
-```
-We have added the following fields as part of the enrichment phase of the enrichment topology:
-* `foo` == 2
-* `ALL_CAPS` == SQUID
-
-We have added the following as part of the threat intel:
-* ` bar` == SQUID
-
-Please note that foo and ALL_CAPS will be applied in separate workers due to them being in separate groups.
-
-* Upload new configs via `$METRON_HOME/bin/zk_load_configs.sh --mode PUSH -i $METRON_HOME/config/zookeeper -z node1:2181`
-* Make the Squid topic in kafka via `/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper node1:2181 --create --topic squid --partitions 1 --replication-factor 1`
-
-## Start Topologies and Send Data
-Now we need to start the topologies and send some data:
-* Start the squid topology via `$METRON_HOME/bin/start_parser_topology.sh -k node1:6667 -z node1:2181 -s squid`
-* Generate some data via the squid client:
- * `squidclient http://yahoo.com`
- * `squidclient http://cnn.com`
-* Send the data to kafka via `cat /var/log/squid/access.log | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list node1:6667 --topic squid`
-* Browse the data in elasticsearch via the ES Head plugin @ [http://node1:9200/_plugin/head/](http://node1:9200/_plugin/head/) and verify that in the squid index you have two documents
-* Ensure that the documents have new fields `foo`, `bar` and `ALL_CAPS` with values as described above.
-
-Note that we could have used any Stellar statements here, including calling out to HBase via `ENRICHMENT_GET` and `ENRICHMENT_EXISTS` or even calling a machine learning model via [Model as a Service](../../metron-analytics/metron-maas-service).
diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-enrichment/metron-enrichment-common/pom.xml
similarity index 51%
copy from metron-platform/metron-indexing/pom.xml
copy to metron-platform/metron-enrichment/metron-enrichment-common/pom.xml
index d2c0f6d..c34566f 100644
--- a/metron-platform/metron-indexing/pom.xml
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/pom.xml
@@ -11,47 +11,49 @@
OR CONDITIONS OF ANY KIND, either express or implied. See the License for
the specific language governing permissions and limitations under the License.
-->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-platform</artifactId>
+ <artifactId>metron-enrichment</artifactId>
<version>0.7.1</version>
</parent>
- <artifactId>metron-indexing</artifactId>
- <name>metron-indexing</name>
- <url>https://metron.apache.org/</url>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>metron-enrichment-common</artifactId>
+
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<slf4j.version>1.7.7</slf4j.version>
<storm.hdfs.version>0.1.2</storm.hdfs.version>
+ <commons-compress.version>1.13</commons-compress.version>
+ <geoip.version>2.12.0</geoip.version>
<guava_version>${global_hbase_guava_version}</guava_version>
</properties>
+
<dependencies>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <version>2.8.2</version>
- <scope>test</scope>
- </dependency>
+
+ <!-- Metron -->
+
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-common</artifactId>
+ <artifactId>stellar-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
+
+ <!-- Other -->
+
<dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-hbase</artifactId>
- <version>${project.parent.version}</version>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ <artifactId>caffeine</artifactId>
+ <version>${global_caffeine_version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${global_hbase_version}</version>
- <scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -62,146 +64,57 @@
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${global_hadoop_version}</version>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- </exclusions>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- <version>${global_storm_version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-over-slf4j</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-slf4j-impl</artifactId>
- <groupId>org.apache.logging.log4j</groupId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
</exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-hdfs</artifactId>
- <version>${global_storm_version}</version>
- <exclusions>
<exclusion>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
+ <artifactId>hadoop-common</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava_version}</version>
+ <groupId>ch.hsr</groupId>
+ <artifactId>geohash</artifactId>
+ <version>1.3.0</version>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${global_hadoop_version}</version>
- <scope>provided</scope>
+ <groupId>com.maxmind.geoip2</groupId>
+ <artifactId>geoip2</artifactId>
+ <version>${geoip.version}</version>
<exclusions>
<exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
+ <artifactId>jackson-core</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <version>${global_mockito_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>commons-validator</groupId>
- <artifactId>commons-validator</artifactId>
- <version>1.4.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-test-utilities</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- <exclusions>
<exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
+ <artifactId>jackson-databind</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-integration-test</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-enrichment</artifactId>
- <version>${project.parent.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- <version>${global_kafka_version}</version>
- <classifier>test</classifier>
- <scope>test</scope>
+ <groupId>org.locationtech.spatial4j</groupId>
+ <artifactId>spatial4j</artifactId>
+ <version>0.6</version>
<exclusions>
<exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
+ <groupId>com.vividsolutions</groupId>
+ <artifactId>jts-core</artifactId>
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-hbase</artifactId>
- <version>${project.parent.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-core</artifactId>
- <version>1.3</version>
- <scope>test</scope>
- </dependency>
+
+ <!-- Test -->
+
+ <!-- All test deps currently inherited from parent -->
+
</dependencies>
+
<build>
<plugins>
<plugin>
@@ -238,30 +151,45 @@
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
<filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
</filters>
<relocations>
<relocation>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>org.apache.metron.jackson</shadedPattern>
+ </relocation>
+ <relocation>
<pattern>com.google.common</pattern>
- <shadedPattern>org.apache.metron.guava.${guava_version}</shadedPattern>
+ <shadedPattern>org.apache.metron.guava.enrichment
+ </shadedPattern>
</relocation>
</relocations>
+ <artifactSet>
+ <excludes>
+ <exclude>storm:storm-core:*</exclude>
+ <exclude>storm:storm-lib:*</exclude>
+ <exclude>org.slf4j.impl*</exclude>
+ <exclude>org.slf4j:slf4j-log4j*</exclude>
+ </excludes>
+ </artifactSet>
<transformers>
<transformer
- implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
- <resources>
+ implementation="org.atteo.classindex.ClassIndexTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resources>
<resource>.yaml</resource>
<resource>LICENSE.txt</resource>
<resource>ASL2.0</resource>
<resource>NOTICE.txt</resource>
- </resources>
+ </resources>
</transformer>
<!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE -->
<!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
@@ -269,15 +197,22 @@
<projectName>${project.name}</projectName>
</transformer-->
<transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass></mainClass>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.atteo.classindex</groupId>
+ <artifactId>classindex-transformer</artifactId>
+ <version>${global_classindex_version}</version>
+ </dependency>
+ </dependencies>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
diff --git a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/assembly/assembly.xml
similarity index 72%
copy from metron-platform/metron-enrichment/src/main/assembly/assembly.xml
copy to metron-platform/metron-enrichment/metron-enrichment-common/src/main/assembly/assembly.xml
index c681468..aabc537 100644
--- a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/assembly/assembly.xml
@@ -44,29 +44,6 @@
<filtered>true</filtered>
</fileSet>
<fileSet>
- <directory>${project.basedir}/src/main/resources/ddl</directory>
- <outputDirectory>ddl</outputDirectory>
- <useDefaultExcludes>true</useDefaultExcludes>
- <excludes>
- <exclude>**/*.formatted</exclude>
- <exclude>**/*.filtered</exclude>
- </excludes>
- <fileMode>0644</fileMode>
- <lineEnding>unix</lineEnding>
- <filtered>true</filtered>
- </fileSet>
- <fileSet>
- <directory>${project.basedir}/src/main/flux</directory>
- <outputDirectory>flux</outputDirectory>
- <useDefaultExcludes>true</useDefaultExcludes>
- <excludes>
- <exclude>**/*.formatted</exclude>
- <exclude>**/*.filtered</exclude>
- </excludes>
- <fileMode>0644</fileMode>
- <lineEnding>unix</lineEnding>
- </fileSet>
- <fileSet>
<directory>${project.basedir}/target</directory>
<includes>
<include>${project.artifactId}-${project.version}-uber.jar</include>
diff --git a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/asa.json b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/config/zookeeper/enrichments/asa.json
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/asa.json
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/config/zookeeper/enrichments/asa.json
diff --git a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/bro.json b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/config/zookeeper/enrichments/bro.json
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/bro.json
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/config/zookeeper/enrichments/bro.json
diff --git a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/snort.json b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/config/zookeeper/enrichments/snort.json
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/snort.json
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/config/zookeeper/enrichments/snort.json
diff --git a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/websphere.json b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/config/zookeeper/enrichments/websphere.json
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/websphere.json
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/config/zookeeper/enrichments/websphere.json
diff --git a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/yaf.json b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/config/zookeeper/enrichments/yaf.json
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/yaf.json
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/config/zookeeper/enrichments/yaf.json
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/cif/AbstractCIFAdapter.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/cif/AbstractCIFAdapter.java
similarity index 96%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/cif/AbstractCIFAdapter.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/cif/AbstractCIFAdapter.java
index e300e59..485f87e 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/cif/AbstractCIFAdapter.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/cif/AbstractCIFAdapter.java
@@ -21,7 +21,7 @@ package org.apache.metron.enrichment.adapters.cif;
import java.io.Serializable;
import java.util.Map;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java
similarity index 98%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java
index e9220db..b5bc20b 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java
similarity index 97%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java
index 830395f..243d1f6 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java
@@ -22,7 +22,7 @@ import java.lang.invoke.MethodHandles;
import java.util.Map;
import java.util.Optional;
import org.apache.metron.enrichment.adapters.maxmind.geo.GeoLiteCityDatabase;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/host/AbstractHostAdapter.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/AbstractHostAdapter.java
similarity index 96%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/host/AbstractHostAdapter.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/AbstractHostAdapter.java
index 2b076ee..0efc1fe 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/host/AbstractHostAdapter.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/AbstractHostAdapter.java
@@ -21,7 +21,7 @@ package org.apache.metron.enrichment.adapters.host;
import java.io.Serializable;
import java.util.Map;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapter.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapter.java
similarity index 97%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapter.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapter.java
index 779f66b..9631fa4 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapter.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapter.java
@@ -17,7 +17,7 @@
*/
package org.apache.metron.enrichment.adapters.host;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapter.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapter.java
similarity index 97%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapter.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapter.java
index 83d92e1..5b3c2d8 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapter.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapter.java
@@ -20,7 +20,7 @@ package org.apache.metron.enrichment.adapters.host;
import java.util.Map;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.json.simple.JSONObject;
@SuppressWarnings("serial")
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/jdbc/BaseJdbcConfig.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/BaseJdbcConfig.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/jdbc/BaseJdbcConfig.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/BaseJdbcConfig.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java
similarity index 98%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java
index 6cd45d2..edbe891 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java
@@ -17,7 +17,7 @@
*/
package org.apache.metron.enrichment.adapters.jdbc;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcConfig.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcConfig.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcConfig.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcConfig.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfig.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfig.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfig.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfig.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/maxmind/MaxMindDatabase.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/MaxMindDatabase.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/maxmind/MaxMindDatabase.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/MaxMindDatabase.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/maxmind/MaxMindDbUtilities.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/MaxMindDbUtilities.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/maxmind/MaxMindDbUtilities.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/MaxMindDbUtilities.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/maxmind/asn/GeoLiteAsnDatabase.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/asn/GeoLiteAsnDatabase.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/maxmind/asn/GeoLiteAsnDatabase.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/asn/GeoLiteAsnDatabase.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoLiteCityDatabase.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoLiteCityDatabase.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoLiteCityDatabase.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoLiteCityDatabase.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/DistanceStrategies.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/DistanceStrategies.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/DistanceStrategies.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/DistanceStrategies.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/DistanceStrategy.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/DistanceStrategy.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/DistanceStrategy.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/DistanceStrategy.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/GeoHashUtil.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/GeoHashUtil.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/GeoHashUtil.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/GeoHashUtil.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
similarity index 98%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
index bc8b758..c84c494 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
@@ -27,7 +27,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.apache.metron.enrichment.converter.EnrichmentKey;
import org.apache.metron.enrichment.converter.EnrichmentValue;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfig.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfig.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfig.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfig.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
similarity index 97%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
index a5feac0..04b759b 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
@@ -17,22 +17,18 @@
*/
package org.apache.metron.enrichment.adapters.stellar;
-import static org.apache.metron.enrichment.bolt.GenericEnrichmentBolt.STELLAR_CONTEXT_CONF;
+import static org.apache.metron.common.Constants.STELLAR_CONTEXT_CONF;
import java.io.Serializable;
import java.lang.invoke.MethodHandles;
-import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.function.Function;
-
-import com.google.common.base.Joiner;
import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import org.apache.metron.stellar.common.StellarProcessor;
import org.apache.metron.stellar.common.utils.ConversionUtils;
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
similarity index 99%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
index 216e237..8515c68 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
@@ -26,7 +26,7 @@ import java.util.Map;
import java.util.UUID;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.apache.metron.enrichment.converter.EnrichmentKey;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import org.apache.metron.enrichment.lookup.EnrichmentLookup;
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfig.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfig.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfig.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfig.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/CacheKey.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cache/CacheKey.java
similarity index 98%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/CacheKey.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cache/CacheKey.java
index b8e4d37..6be8790 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/CacheKey.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cache/CacheKey.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.metron.enrichment.bolt;
+package org.apache.metron.enrichment.cache;
import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.stellar.common.utils.ConversionUtils;
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/cli/LatencySummarizer.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cli/LatencySummarizer.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/cli/LatencySummarizer.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cli/LatencySummarizer.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/configuration/Enrichment.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/configuration/Enrichment.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/configuration/Enrichment.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/configuration/Enrichment.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/AbstractConverter.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/AbstractConverter.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/AbstractConverter.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/AbstractConverter.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/EnrichmentConverter.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentConverter.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/EnrichmentConverter.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentConverter.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/EnrichmentHelper.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentHelper.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/EnrichmentHelper.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentHelper.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/EnrichmentKey.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentKey.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/EnrichmentKey.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentKey.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/EnrichmentValue.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentValue.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/EnrichmentValue.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentValue.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/HbaseConverter.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/HbaseConverter.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/HbaseConverter.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/HbaseConverter.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/interfaces/EnrichmentAdapter.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/interfaces/EnrichmentAdapter.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/interfaces/EnrichmentAdapter.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/interfaces/EnrichmentAdapter.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/EnrichmentLookup.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/EnrichmentLookup.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/EnrichmentLookup.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/EnrichmentLookup.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/Lookup.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/Lookup.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/Lookup.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/Lookup.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/LookupKV.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/LookupKV.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/LookupKV.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/LookupKV.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/LookupKey.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/LookupKey.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/LookupKey.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/LookupKey.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/LookupValue.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/LookupValue.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/LookupValue.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/LookupValue.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTracker.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTracker.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTracker.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTracker.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackerCreator.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackerCreator.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackerCreator.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackerCreator.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackerUtil.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackerUtil.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackerUtil.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackerUtil.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackers.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackers.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackers.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackers.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/BloomAccessTracker.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/BloomAccessTracker.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/BloomAccessTracker.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/BloomAccessTracker.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/NoopAccessTracker.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/NoopAccessTracker.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/NoopAccessTracker.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/NoopAccessTracker.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/PersistentAccessTracker.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/PersistentAccessTracker.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/PersistentAccessTracker.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/PersistentAccessTracker.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/PersistentBloomTrackerCreator.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/PersistentBloomTrackerCreator.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/PersistentBloomTrackerCreator.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/PersistentBloomTrackerCreator.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/handler/Handler.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/handler/Handler.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/handler/Handler.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/handler/Handler.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/handler/KeyWithContext.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/handler/KeyWithContext.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/handler/KeyWithContext.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/handler/KeyWithContext.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ConcurrencyContext.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/ConcurrencyContext.java
similarity index 98%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ConcurrencyContext.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/ConcurrencyContext.java
index ed5985a..3adce0b 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ConcurrencyContext.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/ConcurrencyContext.java
@@ -19,7 +19,7 @@ package org.apache.metron.enrichment.parallel;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentCallable.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentCallable.java
similarity index 97%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentCallable.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentCallable.java
index da4d574..41c0595 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentCallable.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentCallable.java
@@ -17,7 +17,7 @@
*/
package org.apache.metron.enrichment.parallel;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import org.json.simple.JSONObject;
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentContext.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentContext.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentContext.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentContext.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentStrategies.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentStrategies.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentStrategies.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentStrategies.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentStrategy.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentStrategy.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentStrategy.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentStrategy.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
similarity index 99%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
index 1de8945..aa77ecb 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
@@ -24,7 +24,7 @@ import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler;
import org.apache.metron.common.performance.PerformanceLogger;
import org.apache.metron.common.utils.MessageUtils;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import org.apache.metron.enrichment.utils.EnrichmentUtils;
import org.json.simple.JSONObject;
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/WorkerPoolStrategies.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/WorkerPoolStrategies.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/WorkerPoolStrategies.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/WorkerPoolStrategies.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/AsnEnrichmentFunctions.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/AsnEnrichmentFunctions.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/AsnEnrichmentFunctions.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/AsnEnrichmentFunctions.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctions.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctions.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctions.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctions.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/ObjectGet.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/ObjectGet.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/ObjectGet.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/ObjectGet.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctions.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctions.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctions.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctions.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java
diff --git a/metron-platform/metron-enrichment/src/main/resources/META-INF/LICENSE b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/resources/META-INF/LICENSE
similarity index 100%
copy from metron-platform/metron-enrichment/src/main/resources/META-INF/LICENSE
copy to metron-platform/metron-enrichment/metron-enrichment-common/src/main/resources/META-INF/LICENSE
diff --git a/metron-platform/metron-enrichment/src/main/resources/META-INF/NOTICE b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/resources/META-INF/NOTICE
similarity index 100%
copy from metron-platform/metron-enrichment/src/main/resources/META-INF/NOTICE
copy to metron-platform/metron-enrichment/metron-enrichment-common/src/main/resources/META-INF/NOTICE
diff --git a/metron-platform/metron-enrichment/src/main/scripts/latency_summarizer.sh b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/scripts/latency_summarizer.sh
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/scripts/latency_summarizer.sh
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/main/scripts/latency_summarizer.sh
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapterTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapterTest.java
similarity index 98%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapterTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapterTest.java
index 767b17d..c8cf4b1 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapterTest.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapterTest.java
@@ -18,7 +18,7 @@
package org.apache.metron.enrichment.adapters.host;
import org.adrianwalker.multilinestring.Multiline;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapterTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapterTest.java
similarity index 98%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapterTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapterTest.java
index 8ce93c1..90f5dfe 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapterTest.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapterTest.java
@@ -18,7 +18,7 @@
package org.apache.metron.enrichment.adapters.host;
import org.adrianwalker.multilinestring.Multiline;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfigTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfigTest.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfigTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfigTest.java
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/maxmind/asn/GeoLiteAsnDatabaseTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/maxmind/asn/GeoLiteAsnDatabaseTest.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/maxmind/asn/GeoLiteAsnDatabaseTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/maxmind/asn/GeoLiteAsnDatabaseTest.java
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoAdapterTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoAdapterTest.java
similarity index 98%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoAdapterTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoAdapterTest.java
index 4355f91..aa2b1cc 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoAdapterTest.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoAdapterTest.java
@@ -20,7 +20,7 @@ package org.apache.metron.enrichment.adapters.maxmind.geo;
import com.google.common.collect.ImmutableMap;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.enrichment.adapters.geo.GeoAdapter;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.apache.metron.test.utils.UnitTestHelper;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoLiteCityDatabaseTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoLiteCityDatabaseTest.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoLiteCityDatabaseTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoLiteCityDatabaseTest.java
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
similarity index 99%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
index 974f8ab..ef04370 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
@@ -21,7 +21,7 @@ package org.apache.metron.enrichment.adapters.simplehbase;
import com.google.common.collect.ImmutableMap;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.apache.metron.enrichment.converter.EnrichmentKey;
import org.apache.metron.enrichment.converter.EnrichmentValue;
import org.apache.metron.enrichment.lookup.EnrichmentLookup;
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfigTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfigTest.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfigTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfigTest.java
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
similarity index 99%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
index e421edc..bf51b4c 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.log4j.Level;
import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.apache.metron.hbase.TableProvider;
import org.apache.metron.enrichment.converter.EnrichmentKey;
import org.apache.metron.enrichment.converter.EnrichmentValue;
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfigTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfigTest.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfigTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfigTest.java
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/converter/EnrichmentConverterTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/converter/EnrichmentConverterTest.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/converter/EnrichmentConverterTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/converter/EnrichmentConverterTest.java
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockHBaseConnector.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/integration/mock/MockHBaseConnector.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockHBaseConnector.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/integration/mock/MockHBaseConnector.java
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
similarity index 99%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
index a6832d6..35e6000 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
@@ -24,7 +24,7 @@ import org.apache.metron.common.Constants;
import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.enrichment.adapters.stellar.StellarAdapter;
-import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.stellar.dsl.StellarFunctions;
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/AsnEnrichmentFunctionsTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/AsnEnrichmentFunctionsTest.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/AsnEnrichmentFunctionsTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/AsnEnrichmentFunctionsTest.java
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/DocumentationGenerator.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/DocumentationGenerator.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/DocumentationGenerator.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/DocumentationGenerator.java
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctionsTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctionsTest.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctionsTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctionsTest.java
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoHashFunctionsTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/GeoHashFunctionsTest.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoHashFunctionsTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/GeoHashFunctionsTest.java
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/ObjectGetTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/ObjectGetTest.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/ObjectGetTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/ObjectGetTest.java
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java
diff --git a/metron-platform/metron-enrichment/src/test/resources/CIFHbaseAdapterTest.properties b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/resources/CIFHbaseAdapterTest.properties
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/resources/CIFHbaseAdapterTest.properties
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/resources/CIFHbaseAdapterTest.properties
diff --git a/metron-platform/metron-enrichment/src/test/resources/GeoLite/GeoLite2-ASN.tar.gz b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/resources/GeoLite/GeoLite2-ASN.tar.gz
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/resources/GeoLite/GeoLite2-ASN.tar.gz
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/resources/GeoLite/GeoLite2-ASN.tar.gz
diff --git a/metron-platform/metron-enrichment/src/test/resources/GeoLite/GeoLite2-City.mmdb.gz b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/resources/GeoLite/GeoLite2-City.mmdb.gz
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/resources/GeoLite/GeoLite2-City.mmdb.gz
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/resources/GeoLite/GeoLite2-City.mmdb.gz
diff --git a/metron-platform/metron-enrichment/src/test/resources/GeoLite/GeoLite2-City.tar.gz b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/resources/GeoLite/GeoLite2-City.tar.gz
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/resources/GeoLite/GeoLite2-City.tar.gz
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/resources/GeoLite/GeoLite2-City.tar.gz
diff --git a/metron-platform/metron-enrichment/src/test/resources/TestSchemas/CIFHbaseSchema.json b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/resources/TestSchemas/CIFHbaseSchema.json
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/resources/TestSchemas/CIFHbaseSchema.json
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/resources/TestSchemas/CIFHbaseSchema.json
diff --git a/metron-platform/metron-enrichment/src/test/resources/TestSchemas/WhoisHbaseSchema.json b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/resources/TestSchemas/WhoisHbaseSchema.json
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/resources/TestSchemas/WhoisHbaseSchema.json
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/resources/TestSchemas/WhoisHbaseSchema.json
diff --git a/metron-platform/metron-enrichment/src/test/resources/WhoisHbaseAdapterTest.properties b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/resources/WhoisHbaseAdapterTest.properties
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/resources/WhoisHbaseAdapterTest.properties
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/resources/WhoisHbaseAdapterTest.properties
diff --git a/metron-platform/metron-enrichment/src/test/resources/log4j.properties b/metron-platform/metron-enrichment/metron-enrichment-common/src/test/resources/log4j.properties
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/resources/log4j.properties
rename to metron-platform/metron-enrichment/metron-enrichment-common/src/test/resources/log4j.properties
diff --git a/metron-platform/metron-enrichment/Performance.md b/metron-platform/metron-enrichment/metron-enrichment-storm/Performance.md
similarity index 100%
rename from metron-platform/metron-enrichment/Performance.md
rename to metron-platform/metron-enrichment/metron-enrichment-storm/Performance.md
diff --git a/metron-platform/metron-enrichment/metron-enrichment-storm/README.md b/metron-platform/metron-enrichment/metron-enrichment-storm/README.md
new file mode 100644
index 0000000..0766a29
--- /dev/null
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/README.md
@@ -0,0 +1,143 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# Enrichment
+
+## Introduction
+
+This module holds code for enrichments running in a Storm topology.
+
+## Enrichment Architecture
+
+
+
+### Unified Enrichment Topology
+
+The unified enrichment topology uses data parallelism as opposed to the deprecated
+split/join topology's task parallelism. This architecture uses a worker pool to fully
+enrich any message within a worker. This results in
+* Fewer bolts in the topology
+* Each bolt fully operates on a message.
+* Fewer network hops
+
+This architecture is fully backwards compatible with the old split-join
+topology; the only difference is how the enrichment will operate on each
+message (in one bolt where the split/join is done in a threadpool as opposed
+to split across multiple bolts).
+
+#### Configuring It
+
+There are two parameters which you might want to tune in this topology.
+Both of them are topology configuration adjustable in the flux file
+`$METRON_HOME/config/flux/enrichment/remote-unified.yaml`:
+* `metron.threadpool.size` : The size of the threadpool. This can take a number or a multiple of the number of cores (e.g. `5C` to 5 times the number of cores). The default is `2C`.
+* `metron.threadpool.type` : The type of threadpool. (note: descriptions taken from [here](https://zeroturnaround.com/rebellabs/fixedthreadpool-cachedthreadpool-or-forkjoinpool-picking-correct-java-executors-for-background-tasks/)).
+ * `FIXED` is a fixed threadpool of size `n`. `n` threads will process tasks at the time, when the pool is saturated, new tasks will get added to a queue without a limit on size. Good for CPU intensive tasks. This is the default.
+ * `WORK_STEALING` is a work stealing threadpool. This will create and shut down threads dynamically to accommodate the required parallelism level. It also tries to reduce the contention on the task queue, so can be really good in heavily loaded environments. Also good when your tasks create more tasks for the executor, like recursive tasks.
+
+In order to configure the parallelism for the enrichment bolt and threat
+intel bolt, the configurations will be taken from the respective join bolt
+parallelism. When proper ambari support for this is added, we will add
+its own property.
+
+### Split-Join Enrichment Topology
+
+The now-deprecated split/join topology is also available and performs enrichments in parallel.
+This poses some issues in terms of ease of tuning and reasoning about performance.
+
+
+
+#### Using It
+
+In order to use the older, deprecated topology, you will need to
+* Edit `$METRON_HOME/bin/start_enrichment_topology.sh` and adjust it to use `remote-splitjoin.yaml` instead of `remote-unified.yaml`
+* Restart the enrichment topology.
+
+## Enrichment Configuration
+
+The configuration for the `enrichment` topology, the topology primarily
+responsible for enrichment and threat intelligence enrichment, is
+defined by JSON documents stored in zookeeper.
+
+See [Enrichment Configuration](../metron-enrichment-common#enrichment-configuration)
+
+# Example Enrichment via Stellar
+
+Let's walk through doing a simple enrichment using Stellar on your cluster using the Squid topology.
+
+## Install Prerequisites
+Now let's install some prerequisites:
+* Squid client via `yum install squid`
+* ES Head plugin via `/usr/share/elasticsearch/bin/plugin install mobz/elasticsearch-head`
+
+Start Squid via `service squid start`
+
+## Adjust Enrichment Configurations for Squid to Call Stellar
+Let's adjust the configurations for the Squid topology to annotate the messages using some Stellar functions.
+
+* Edit the squid enrichment configuration at `$METRON_HOME/config/zookeeper/enrichments/squid.json` (this file will not exist, so create a new one) to add some new fields based on stellar queries:
+
+```
+{
+ "enrichment" : {
+ "fieldMap": {
+ "stellar" : {
+ "config" : {
+ "numeric" : {
+ "foo": "1 + 1"
+ }
+ ,"ALL_CAPS" : "TO_UPPER(source.type)"
+ }
+ }
+ }
+ },
+ "threatIntel" : {
+ "fieldMap":{
+ "stellar" : {
+ "config" : {
+ "bar" : "TO_UPPER(source.type)"
+ }
+ }
+ },
+ "triageConfig" : {
+ }
+ }
+}
+```
+We have added the following fields as part of the enrichment phase of the enrichment topology:
+* `foo` == 2
+* `ALL_CAPS` == SQUID
+
+We have added the following as part of the threat intel:
+* ` bar` == SQUID
+
+Please note that foo and ALL_CAPS will be applied in separate workers due to them being in separate groups.
+
+* Upload new configs via `$METRON_HOME/bin/zk_load_configs.sh --mode PUSH -i $METRON_HOME/config/zookeeper -z node1:2181`
+* Make the Squid topic in kafka via `/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper node1:2181 --create --topic squid --partitions 1 --replication-factor 1`
+
+## Start Topologies and Send Data
+Now we need to start the topologies and send some data:
+* Start the squid topology via `$METRON_HOME/bin/start_parser_topology.sh -k node1:6667 -z node1:2181 -s squid`
+* Generate some data via the squid client:
+ * `squidclient http://yahoo.com`
+ * `squidclient http://cnn.com`
+* Send the data to kafka via `cat /var/log/squid/access.log | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list node1:6667 --topic squid`
+* Browse the data in elasticsearch via the ES Head plugin @ [http://node1:9200/_plugin/head/](http://node1:9200/_plugin/head/) and verify that in the squid index you have two documents
+* Ensure that the documents have new fields `foo`, `bar` and `ALL_CAPS` with values as described above.
+
+Note that we could have used any Stellar statements here, including calling out to HBase via `ENRICHMENT_GET` and `ENRICHMENT_EXISTS` or even calling a machine learning model via [Model as a Service](../../../metron-analytics/metron-maas-service).
diff --git a/metron-platform/metron-enrichment/enrichment_arch.png b/metron-platform/metron-enrichment/metron-enrichment-storm/enrichment_arch.png
similarity index 100%
rename from metron-platform/metron-enrichment/enrichment_arch.png
rename to metron-platform/metron-enrichment/metron-enrichment-storm/enrichment_arch.png
diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml
similarity index 80%
copy from metron-platform/metron-enrichment/pom.xml
copy to metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml
index 74eb4e4..08ae7ac 100644
--- a/metron-platform/metron-enrichment/pom.xml
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml
@@ -11,39 +11,35 @@
OR CONDITIONS OF ANY KIND, either express or implied. See the License for
the specific language governing permissions and limitations under the License.
-->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-platform</artifactId>
+ <artifactId>metron-enrichment</artifactId>
<version>0.7.1</version>
</parent>
- <artifactId>metron-enrichment</artifactId>
- <name>metron-enrichment</name>
- <url>https://metron.apache.org/</url>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>metron-enrichment-storm</artifactId>
+
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<slf4j.version>1.7.7</slf4j.version>
<storm.hdfs.version>0.1.2</storm.hdfs.version>
<commons-compress.version>1.13</commons-compress.version>
- <geoip.version>2.12.0</geoip.version>
- <guava_version>${global_hbase_guava_version}</guava_version>
+ <geoip.version>2.8.0</geoip.version>
+ <guava.version>${global_hbase_guava_version}</guava.version>
</properties>
<dependencies>
+
+ <!-- Metron -->
+
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-common</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-common</artifactId>
+ <artifactId>metron-enrichment-common</artifactId>
<version>${project.parent.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
@@ -57,24 +53,6 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-hbase</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-hbase</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>com.github.ben-manes.caffeine</groupId>
- <artifactId>caffeine</artifactId>
- <version>${global_caffeine_version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.metron</groupId>
<artifactId>metron-profiler-client</artifactId>
<version>${project.parent.version}</version>
</dependency>
@@ -83,54 +61,9 @@
<artifactId>metron-statistics</artifactId>
<version>${project.parent.version}</version>
</dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${global_jackson_version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- <version>${global_jackson_version}</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>${slf4j.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>ch.hsr</groupId>
- <artifactId>geohash</artifactId>
- <version>1.3.0</version>
- </dependency>
- <dependency>
- <groupId>org.locationtech.spatial4j</groupId>
- <artifactId>spatial4j</artifactId>
- <version>0.6</version>
- <exclusions>
- <exclusion>
- <groupId>com.vividsolutions</groupId>
- <artifactId>jts-core</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.maxmind.geoip2</groupId>
- <artifactId>geoip2</artifactId>
- <version>${geoip.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>jackson-core</artifactId>
- <groupId>com.fasterxml.jackson.core</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jackson-databind</artifactId>
- <groupId>com.fasterxml.jackson.core</groupId>
- </exclusion>
- </exclusions>
- </dependency>
+ <!-- Hadoop -->
+
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
@@ -228,25 +161,8 @@
</exclusions>
</dependency>
<dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- <version>${global_log4j_core_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <version>${global_log4j_core_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava_version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
+ <artifactId>kafka_2.11</artifactId>
<version>${global_kafka_version}</version>
<exclusions>
<exclusion>
@@ -260,19 +176,6 @@
</exclusions>
</dependency>
<dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- <version>${global_kafka_version}</version>
- <classifier>test</classifier>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${global_hadoop_version}</version>
@@ -291,11 +194,34 @@
</exclusion>
</exclusions>
</dependency>
+
+ <!-- Other -->
+
<dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>${global_mockito_version}</version>
- <scope>test</scope>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ <artifactId>caffeine</artifactId>
+ <version>${global_caffeine_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${global_jackson_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${global_jackson_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${global_hbase_guava_version}</version>
</dependency>
<dependency>
<groupId>commons-validator</groupId>
@@ -303,23 +229,53 @@
<version>1.4.0</version>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ <version>${commons-compress.version}</version>
+ </dependency>
+
+ <!-- Test -->
+
+ <dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-test-utilities</artifactId>
+ <artifactId>metron-enrichment-common</artifactId>
<version>${project.parent.version}</version>
<scope>test</scope>
+ <type>test-jar</type>
</dependency>
<dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-integration-test</artifactId>
- <version>${project.parent.version}</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>${global_log4j_core_version}</version>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- <version>${commons-compress.version}</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>${global_log4j_core_version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${global_kafka_version}</version>
+ <classifier>test</classifier>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${global_httpclient_version}</version>
+ <scope>test</scope>
</dependency>
</dependencies>
+
<build>
<plugins>
<plugin>
@@ -372,7 +328,8 @@
</relocation>
<relocation>
<pattern>com.google.common</pattern>
- <shadedPattern>org.apache.metron.guava.${guava_version}</shadedPattern>
+ <shadedPattern>org.apache.metron.guava.enrichment
+ </shadedPattern>
</relocation>
</relocations>
<artifactSet>
@@ -384,15 +341,16 @@
</excludes>
</artifactSet>
<transformers>
- <transformer implementation="org.atteo.classindex.ClassIndexTransformer"/>
<transformer
- implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
- <resources>
+ implementation="org.atteo.classindex.ClassIndexTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resources>
<resource>.yaml</resource>
<resource>LICENSE.txt</resource>
<resource>ASL2.0</resource>
<resource>NOTICE.txt</resource>
- </resources>
+ </resources>
</transformer>
<!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE -->
<!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
@@ -400,9 +358,9 @@
<projectName>${project.name}</projectName>
</transformer-->
<transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass></mainClass>
</transformer>
</transformers>
diff --git a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/assembly/assembly.xml
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/assembly/assembly.xml
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/main/assembly/assembly.xml
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/config/enrichment-splitjoin.properties
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/main/config/enrichment-splitjoin.properties
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties.j2 b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/config/enrichment-splitjoin.properties.j2
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties.j2
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/main/config/enrichment-splitjoin.properties.j2
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/config/enrichment-unified.properties
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/main/config/enrichment-unified.properties
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties.j2 b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/config/enrichment-unified.properties.j2
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties.j2
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/main/config/enrichment-unified.properties.j2
diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-splitjoin.yaml b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/flux/enrichment/remote-splitjoin.yaml
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/flux/enrichment/remote-splitjoin.yaml
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/main/flux/enrichment/remote-splitjoin.yaml
diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-unified.yaml b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/flux/enrichment/remote-unified.yaml
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/flux/enrichment/remote-unified.yaml
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/main/flux/enrichment/remote-unified.yaml
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
similarity index 98%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
index e0f1f0c..55c7b66 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
@@ -18,6 +18,8 @@
package org.apache.metron.enrichment.bolt;
+import static org.apache.metron.common.Constants.STELLAR_CONTEXT_CONF;
+
import java.util.HashSet;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@@ -32,6 +34,7 @@ import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.common.error.MetronError;
import org.apache.metron.common.performance.PerformanceLogger;
import org.apache.metron.common.utils.ErrorUtils;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.apache.metron.enrichment.configuration.Enrichment;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import org.apache.metron.enrichment.utils.EnrichmentUtils;
@@ -70,7 +73,6 @@ public class GenericEnrichmentBolt extends ConfiguredEnrichmentBolt {
private PerformanceLogger perfLog; // not static bc multiple bolts may exist in same worker
private static final Logger LOG = LoggerFactory
.getLogger(GenericEnrichmentBolt.class);
- public static final String STELLAR_CONTEXT_CONF = "stellarContext";
private static final String ERROR_STREAM = "error";
private OutputCollector collector;
private Context stellarContext;
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/SplitBolt.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/java/org/apache/metron/enrichment/bolt/SplitBolt.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/SplitBolt.java
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/main/java/org/apache/metron/enrichment/bolt/SplitBolt.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/UnifiedEnrichmentBolt.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/java/org/apache/metron/enrichment/bolt/UnifiedEnrichmentBolt.java
similarity index 99%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/UnifiedEnrichmentBolt.java
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/main/java/org/apache/metron/enrichment/bolt/UnifiedEnrichmentBolt.java
index c81251e..d9f57eb 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/UnifiedEnrichmentBolt.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/java/org/apache/metron/enrichment/bolt/UnifiedEnrichmentBolt.java
@@ -17,6 +17,8 @@
*/
package org.apache.metron.enrichment.bolt;
+import static org.apache.metron.common.Constants.STELLAR_CONTEXT_CONF;
+
import org.apache.metron.common.Constants;
import org.apache.metron.common.bolt.ConfiguredEnrichmentBolt;
import org.apache.metron.common.configuration.ConfigurationType;
@@ -29,6 +31,7 @@ import org.apache.metron.common.utils.ErrorUtils;
import org.apache.metron.common.utils.MessageUtils;
import org.apache.metron.enrichment.adapters.maxmind.asn.GeoLiteAsnDatabase;
import org.apache.metron.enrichment.adapters.maxmind.geo.GeoLiteCityDatabase;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.apache.metron.enrichment.configuration.Enrichment;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import org.apache.metron.enrichment.parallel.EnrichmentContext;
@@ -71,8 +74,6 @@ public class UnifiedEnrichmentBolt extends ConfiguredEnrichmentBolt {
protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- public static final String STELLAR_CONTEXT_CONF = "stellarContext";
-
/**
* The number of threads in the threadpool. One threadpool is created per process.
* This is a topology-level configuration
diff --git a/metron-platform/metron-enrichment/src/main/resources/META-INF/LICENSE b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/resources/META-INF/LICENSE
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/resources/META-INF/LICENSE
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/main/resources/META-INF/LICENSE
diff --git a/metron-platform/metron-enrichment/src/main/resources/META-INF/NOTICE b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/resources/META-INF/NOTICE
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/resources/META-INF/NOTICE
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/main/resources/META-INF/NOTICE
diff --git a/metron-platform/metron-enrichment/src/main/scripts/start_enrichment_topology.sh b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/scripts/start_enrichment_topology.sh
similarity index 100%
rename from metron-platform/metron-enrichment/src/main/scripts/start_enrichment_topology.sh
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/main/scripts/start_enrichment_topology.sh
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBoltTest.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBoltTest.java
similarity index 95%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBoltTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBoltTest.java
index 52135e3..9fc8947 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBoltTest.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBoltTest.java
@@ -17,6 +17,14 @@
*/
package org.apache.metron.enrichment.bolt;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.message.MessageGetStrategy;
import org.apache.metron.test.bolt.BaseEnrichmentBoltTest;
@@ -28,17 +36,10 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
public class EnrichmentJoinBoltTest extends BaseEnrichmentBoltTest {
+ private static final String enrichmentConfigPath = "../" + sampleSensorEnrichmentConfigPath;
+
/**
* {
* "enrichedField": "enrichedValue",
@@ -74,7 +75,7 @@ public class EnrichmentJoinBoltTest extends BaseEnrichmentBoltTest {
EnrichmentJoinBolt enrichmentJoinBolt = new EnrichmentJoinBolt("zookeeperUrl");
enrichmentJoinBolt.setCuratorFramework(client);
enrichmentJoinBolt.setZKCache(cache);
- enrichmentJoinBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, new FileInputStream(sampleSensorEnrichmentConfigPath));
+ enrichmentJoinBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, new FileInputStream(enrichmentConfigPath));
enrichmentJoinBolt.withMaxCacheSize(100);
enrichmentJoinBolt.withMaxTimeRetain(10000);
enrichmentJoinBolt.prepare(new HashMap<>(), topologyContext, outputCollector);
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBoltTest.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBoltTest.java
similarity index 96%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBoltTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBoltTest.java
index cbe7ed6..1e9177b 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBoltTest.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBoltTest.java
@@ -17,15 +17,10 @@
*/
package org.apache.metron.enrichment.bolt;
+import static org.mockito.Mockito.when;
+
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
-import org.apache.metron.test.bolt.BaseEnrichmentBoltTest;
-import org.apache.metron.enrichment.configuration.Enrichment;
-import org.json.simple.JSONObject;
-import org.json.simple.parser.ParseException;
-import org.junit.Assert;
-import org.junit.Test;
-
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
@@ -33,12 +28,18 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
-
-import static org.mockito.Mockito.when;
+import org.apache.metron.enrichment.configuration.Enrichment;
+import org.apache.metron.test.bolt.BaseEnrichmentBoltTest;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.ParseException;
+import org.junit.Assert;
+import org.junit.Test;
public class EnrichmentSplitterBoltTest extends BaseEnrichmentBoltTest {
+ private static final String enrichmentConfigPath = "../" + sampleSensorEnrichmentConfigPath;
+
@Test
public void test() throws ParseException, IOException {
final Enrichment geo = new Enrichment();
@@ -59,7 +60,7 @@ public class EnrichmentSplitterBoltTest extends BaseEnrichmentBoltTest {
EnrichmentSplitterBolt enrichmentSplitterBolt = new EnrichmentSplitterBolt("zookeeperUrl").withEnrichments(enrichments);
enrichmentSplitterBolt.setCuratorFramework(client);
enrichmentSplitterBolt.setZKCache(cache);
- enrichmentSplitterBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, new FileInputStream(sampleSensorEnrichmentConfigPath));
+ enrichmentSplitterBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, new FileInputStream(enrichmentConfigPath));
enrichmentSplitterBolt.prepare(new HashMap<>(), topologyContext, outputCollector);
String key = enrichmentSplitterBolt.getKey(tuple, sampleMessage);
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java
similarity index 94%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java
index aa9768c..874d701 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java
@@ -17,7 +17,22 @@
*/
package org.apache.metron.enrichment.bolt;
+import static org.apache.metron.common.Constants.STELLAR_CONTEXT_CONF;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
import com.google.common.collect.ImmutableMap;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.log4j.Level;
import org.apache.metron.TestConstants;
@@ -26,6 +41,7 @@ import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.common.error.MetronError;
import org.apache.metron.enrichment.adapters.maxmind.geo.GeoLiteCityDatabase;
+import org.apache.metron.enrichment.cache.CacheKey;
import org.apache.metron.enrichment.configuration.Enrichment;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import org.apache.metron.test.bolt.BaseEnrichmentBoltTest;
@@ -42,23 +58,11 @@ import org.mockito.ArgumentMatcher;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.reset;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
public class GenericEnrichmentBoltTest extends BaseEnrichmentBoltTest {
+ private static final String sampleConfigPath = "../" + TestConstants.SAMPLE_CONFIG_PATH;
+ private static final String enrichmentConfigPath = "../" + sampleSensorEnrichmentConfigPath;
+
protected class EnrichedMessageMatcher extends ArgumentMatcher<Values> {
private String expectedKey;
@@ -161,10 +165,10 @@ public class GenericEnrichmentBoltTest extends BaseEnrichmentBoltTest {
};
genericEnrichmentBolt.setCuratorFramework(client);
genericEnrichmentBolt.setZKCache(cache);
- genericEnrichmentBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, new FileInputStream(sampleSensorEnrichmentConfigPath));
+ genericEnrichmentBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, new FileInputStream(enrichmentConfigPath));
HashMap<String, Object> globalConfig = new HashMap<>();
- String baseDir = UnitTestHelper.findDir("GeoLite");
+ String baseDir = UnitTestHelper.findDir(new File("../metron-enrichment-common"), "GeoLite");
File geoHdfsFile = new File(new File(baseDir), "GeoLite2-City.mmdb.gz");
globalConfig.put(GeoLiteCityDatabase.GEO_HDFS_FILE, geoHdfsFile.getAbsolutePath());
genericEnrichmentBolt.getConfigurations().updateGlobalConfig(globalConfig);
@@ -213,8 +217,8 @@ public class GenericEnrichmentBoltTest extends BaseEnrichmentBoltTest {
reset(enrichmentAdapter);
SensorEnrichmentConfig sensorEnrichmentConfig = SensorEnrichmentConfig.
- fromBytes(ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.SAMPLE_CONFIG_PATH).get(sensorType));
- sensorEnrichmentConfig.getConfiguration().put(GenericEnrichmentBolt.STELLAR_CONTEXT_CONF, genericEnrichmentBolt.getStellarContext());
+ fromBytes(ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(sampleConfigPath).get(sensorType));
+ sensorEnrichmentConfig.getConfiguration().put(STELLAR_CONTEXT_CONF, genericEnrichmentBolt.getStellarContext());
CacheKey cacheKey1 = new CacheKey("field1", "value1", sensorEnrichmentConfig);
CacheKey cacheKey2 = new CacheKey("field2", "value2", sensorEnrichmentConfig);
genericEnrichmentBolt.cache.invalidateAll();
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/JoinBoltTest.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/JoinBoltTest.java
similarity index 99%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/JoinBoltTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/JoinBoltTest.java
index 0da6eaa..bc7dace 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/JoinBoltTest.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/JoinBoltTest.java
@@ -41,7 +41,6 @@ import java.util.concurrent.ExecutionException;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyMap;
import static org.mockito.Matchers.argThat;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/SplitBoltTest.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/SplitBoltTest.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/SplitBoltTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/SplitBoltTest.java
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
similarity index 96%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
index 03e9558..a05c843 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
@@ -17,7 +17,17 @@
*/
package org.apache.metron.enrichment.bolt;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
import com.fasterxml.jackson.databind.JsonMappingException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig;
@@ -35,19 +45,10 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
public class ThreatIntelJoinBoltTest extends BaseEnrichmentBoltTest {
+ private static final String enrichmentConfigPath = "../" + sampleSensorEnrichmentConfigPath;
+
/**
* {
* "field1": "value1",
@@ -161,7 +162,7 @@ public class ThreatIntelJoinBoltTest extends BaseEnrichmentBoltTest {
threatIntelJoinBolt.setZKCache(cache);
SensorEnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(
- new FileInputStream(sampleSensorEnrichmentConfigPath), SensorEnrichmentConfig.class);
+ new FileInputStream(enrichmentConfigPath), SensorEnrichmentConfig.class);
boolean withThreatTriage = threatTriageConfig != null;
if (withThreatTriage) {
try {
@@ -177,7 +178,7 @@ public class ThreatIntelJoinBoltTest extends BaseEnrichmentBoltTest {
}
threatIntelJoinBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, enrichmentConfig);
HashMap<String, Object> globalConfig = new HashMap<>();
- String baseDir = UnitTestHelper.findDir("GeoLite");
+ String baseDir = UnitTestHelper.findDir(new File("../metron-enrichment-common"), "GeoLite");
File geoHdfsFile = new File(new File(baseDir), "GeoLite2-City.mmdb.gz");
globalConfig.put(GeoLiteCityDatabase.GEO_HDFS_FILE, geoHdfsFile.getAbsolutePath());
File asnHdfsFile = new File(new File(baseDir), "GeoLite2-ASN.tar.gz");
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBoltTest.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBoltTest.java
similarity index 92%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBoltTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBoltTest.java
index f7869e5..a27297f 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBoltTest.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBoltTest.java
@@ -28,13 +28,15 @@ import java.util.Map;
public class ThreatIntelSplitterBoltTest extends BaseEnrichmentBoltTest {
+ private static final String enrichmentConfigPath = "../" + sampleSensorEnrichmentConfigPath;
+
@Test
public void test() throws IOException {
String threatIntelType = "hbaseThreatIntel";
ThreatIntelSplitterBolt threatIntelSplitterBolt = new ThreatIntelSplitterBolt("zookeeperUrl");
threatIntelSplitterBolt.setCuratorFramework(client);
threatIntelSplitterBolt.setZKCache(cache);
- threatIntelSplitterBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, new FileInputStream(sampleSensorEnrichmentConfigPath));
+ threatIntelSplitterBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, new FileInputStream(enrichmentConfigPath));
threatIntelSplitterBolt.prepare(new HashMap<>(), topologyContext, outputCollector);
Map<String, Object> fieldMap = threatIntelSplitterBolt.getFieldMap(sensorType);
Assert.assertTrue(fieldMap.containsKey(threatIntelType));
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
similarity index 97%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
index 743c31d..6b047fb 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
@@ -44,7 +44,7 @@ import org.apache.metron.enrichment.adapters.maxmind.geo.GeoLiteCityDatabase;
import org.apache.metron.enrichment.converter.EnrichmentHelper;
import org.apache.metron.enrichment.converter.EnrichmentKey;
import org.apache.metron.enrichment.converter.EnrichmentValue;
-import org.apache.metron.enrichment.integration.components.ConfigUploadComponent;
+import org.apache.metron.integration.components.ConfigUploadComponent;
import org.apache.metron.enrichment.lookup.LookupKV;
import org.apache.metron.enrichment.lookup.accesstracker.PersistentBloomTrackerCreator;
import org.apache.metron.enrichment.stellar.SimpleHBaseEnrichmentFunctions;
@@ -91,14 +91,15 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest {
public static final String threatIntelTableName = "threat_intel";
public static final String enrichmentsTableName = "enrichments";
- protected String sampleParsedPath = TestConstants.SAMPLE_DATA_PARSED_PATH + "TestExampleParsed";
+ protected String enrichmentConfigPath = "../" + TestConstants.SAMPLE_CONFIG_PATH;
+ protected String sampleParsedPath = "../" + TestConstants.SAMPLE_DATA_PARSED_PATH + "TestExampleParsed";
private final List<byte[]> inputMessages = getInputMessages(sampleParsedPath);
private static File geoHdfsFile;
private static File asnHdfsFile;
protected String fluxPath() {
- return "../metron-enrichment/src/main/flux/enrichment/remote-splitjoin.yaml";
+ return "src/main/flux/enrichment/remote-splitjoin.yaml";
}
private static List<byte[]> getInputMessages(String path){
@@ -119,7 +120,7 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest {
@BeforeClass
public static void setupOnce() throws ParseException {
- String baseDir = UnitTestHelper.findDir("GeoLite");
+ String baseDir = UnitTestHelper.findDir(new File("../metron-enrichment-common"), "GeoLite");
geoHdfsFile = new File(new File(baseDir), "GeoLite2-City.mmdb.gz");
asnHdfsFile = new File(new File(baseDir), "GeoLite2-ASN.tar.gz");
}
@@ -130,7 +131,7 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest {
* @return The path to the topology properties template.
*/
public String getTemplatePath() {
- return "../metron-enrichment/src/main/config/enrichment-splitjoin.properties.j2";
+ return "src/main/config/enrichment-splitjoin.properties.j2";
}
/**
@@ -185,7 +186,7 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest {
}});
String globalConfigStr = null;
{
- File globalConfig = new File(new File(TestConstants.SAMPLE_CONFIG_PATH), "global.json");
+ File globalConfig = new File(enrichmentConfigPath, "global.json");
Map<String, Object> config = JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER);
config.put(SimpleHBaseEnrichmentFunctions.TABLE_PROVIDER_TYPE_CONF, MockHBaseTableProvider.class.getName());
config.put(SimpleHBaseEnrichmentFunctions.ACCESS_TRACKER_TYPE_CONF, "PERSISTENT_BLOOM");
@@ -198,7 +199,7 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest {
ConfigUploadComponent configUploadComponent = new ConfigUploadComponent()
.withTopologyProperties(topologyProperties)
.withGlobalConfig(globalConfigStr)
- .withEnrichmentConfigsPath(TestConstants.SAMPLE_CONFIG_PATH);
+ .withEnrichmentConfigsPath(enrichmentConfigPath);
//create MockHBaseTables
final MockHTable trackerTable = (MockHTable) MockHBaseTableProvider.addToCache(trackerHBaseTableName, cf);
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/UnifiedEnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/integration/UnifiedEnrichmentIntegrationTest.java
similarity index 95%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/UnifiedEnrichmentIntegrationTest.java
rename to metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/integration/UnifiedEnrichmentIntegrationTest.java
index 5c19b39..93c9700 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/UnifiedEnrichmentIntegrationTest.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/integration/UnifiedEnrichmentIntegrationTest.java
@@ -33,7 +33,7 @@ public class UnifiedEnrichmentIntegrationTest extends EnrichmentIntegrationTest
* @return The path to the topology properties template.
*/
public String getTemplatePath() {
- return "../metron-enrichment/src/main/config/enrichment-unified.properties.j2";
+ return "src/main/config/enrichment-unified.properties.j2";
}
/**
@@ -91,6 +91,6 @@ public class UnifiedEnrichmentIntegrationTest extends EnrichmentIntegrationTest
@Override
public String fluxPath() {
- return "../metron-enrichment/src/main/flux/enrichment/remote-unified.yaml";
+ return "src/main/flux/enrichment/remote-unified.yaml";
}
}
diff --git a/metron-platform/metron-enrichment/unified_enrichment_arch.svg b/metron-platform/metron-enrichment/metron-enrichment-storm/unified_enrichment_arch.svg
similarity index 100%
rename from metron-platform/metron-enrichment/unified_enrichment_arch.svg
rename to metron-platform/metron-enrichment/metron-enrichment-storm/unified_enrichment_arch.svg
diff --git a/metron-platform/metron-enrichment/unified_enrichment_arch_diagram.xml b/metron-platform/metron-enrichment/metron-enrichment-storm/unified_enrichment_arch_diagram.xml
similarity index 100%
rename from metron-platform/metron-enrichment/unified_enrichment_arch_diagram.xml
rename to metron-platform/metron-enrichment/metron-enrichment-storm/unified_enrichment_arch_diagram.xml
diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml
index 74eb4e4..9e689fa 100644
--- a/metron-platform/metron-enrichment/pom.xml
+++ b/metron-platform/metron-enrichment/pom.xml
@@ -12,27 +12,29 @@
the specific language governing permissions and limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
<version>0.7.1</version>
</parent>
+ <modelVersion>4.0.0</modelVersion>
+ <packaging>pom</packaging>
<artifactId>metron-enrichment</artifactId>
<name>metron-enrichment</name>
+ <description>Metron Enrichment Parent Project</description>
<url>https://metron.apache.org/</url>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <slf4j.version>1.7.7</slf4j.version>
- <storm.hdfs.version>0.1.2</storm.hdfs.version>
- <commons-compress.version>1.13</commons-compress.version>
- <geoip.version>2.12.0</geoip.version>
- <guava_version>${global_hbase_guava_version}</guava_version>
- </properties>
+ <modules>
+ <module>metron-enrichment-common</module>
+ <module>metron-enrichment-storm</module>
+ </modules>
+
<dependencies>
+
+ <!-- Metron -->
+
<dependency>
<groupId>org.apache.metron</groupId>
<artifactId>metron-common</artifactId>
@@ -40,398 +42,41 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-common</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-storm-kafka</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-writer</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
<artifactId>metron-hbase</artifactId>
<version>${project.parent.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-hbase</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>com.github.ben-manes.caffeine</groupId>
- <artifactId>caffeine</artifactId>
- <version>${global_caffeine_version}</version>
- </dependency>
+
+ <!-- Other -->
+
+ <!-- Test -->
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-profiler-client</artifactId>
+ <artifactId>metron-test-utilities</artifactId>
<version>${project.parent.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-statistics</artifactId>
+ <artifactId>metron-integration-test</artifactId>
<version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${global_jackson_version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- <version>${global_jackson_version}</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>${slf4j.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>ch.hsr</groupId>
- <artifactId>geohash</artifactId>
- <version>1.3.0</version>
- </dependency>
- <dependency>
- <groupId>org.locationtech.spatial4j</groupId>
- <artifactId>spatial4j</artifactId>
- <version>0.6</version>
- <exclusions>
- <exclusion>
- <groupId>com.vividsolutions</groupId>
- <artifactId>jts-core</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>com.maxmind.geoip2</groupId>
- <artifactId>geoip2</artifactId>
- <version>${geoip.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>jackson-core</artifactId>
- <groupId>com.fasterxml.jackson.core</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jackson-databind</artifactId>
- <groupId>com.fasterxml.jackson.core</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <version>${global_hbase_version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
- <version>${global_hadoop_version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${global_hadoop_version}</version>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>commons-httpclient</artifactId>
- <groupId>commons-httpclient</groupId>
- </exclusion>
- <exclusion>
- <artifactId>slf4j-log4j12</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${global_hadoop_version}</version>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- <version>${global_storm_version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-over-slf4j</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-slf4j-impl</artifactId>
- <groupId>org.apache.logging.log4j</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- <version>${global_log4j_core_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <version>${global_log4j_core_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava_version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
- <version>${global_kafka_version}</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- <version>${global_kafka_version}</version>
- <classifier>test</classifier>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${global_hadoop_version}</version>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>commons-httpclient</artifactId>
- <groupId>commons-httpclient</groupId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>${global_mockito_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>commons-validator</groupId>
- <artifactId>commons-validator</artifactId>
- <version>1.4.0</version>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-test-utilities</artifactId>
+ <artifactId>metron-common</artifactId>
<version>${project.parent.version}</version>
<scope>test</scope>
+ <type>test-jar</type>
</dependency>
<dependency>
+ <!-- Need mock hbase table -->
<groupId>org.apache.metron</groupId>
- <artifactId>metron-integration-test</artifactId>
+ <artifactId>metron-hbase</artifactId>
<version>${project.parent.version}</version>
<scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- <version>${commons-compress.version}</version>
+ <type>test-jar</type>
</dependency>
</dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <version>${global_jar_version}</version>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${global_shade_version}</version>
- <configuration>
- <createDependencyReducedPom>true</createDependencyReducedPom>
- <artifactSet>
- <excludes>
- <exclude>*slf4j*</exclude>
- </excludes>
- </artifactSet>
- </configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <shadedArtifactAttached>true</shadedArtifactAttached>
- <shadedClassifierName>uber</shadedClassifierName>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- <relocations>
- <relocation>
- <pattern>com.fasterxml.jackson</pattern>
- <shadedPattern>org.apache.metron.jackson</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.common</pattern>
- <shadedPattern>org.apache.metron.guava.${guava_version}</shadedPattern>
- </relocation>
- </relocations>
- <artifactSet>
- <excludes>
- <exclude>storm:storm-core:*</exclude>
- <exclude>storm:storm-lib:*</exclude>
- <exclude>org.slf4j.impl*</exclude>
- <exclude>org.slf4j:slf4j-log4j*</exclude>
- </excludes>
- </artifactSet>
- <transformers>
- <transformer implementation="org.atteo.classindex.ClassIndexTransformer"/>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
- <resources>
- <resource>.yaml</resource>
- <resource>LICENSE.txt</resource>
- <resource>ASL2.0</resource>
- <resource>NOTICE.txt</resource>
- </resources>
- </transformer>
- <!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE -->
- <!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
- <addHeader>false</addHeader>
- <projectName>${project.name}</projectName>
- </transformer-->
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass></mainClass>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>org.atteo.classindex</groupId>
- <artifactId>classindex-transformer</artifactId>
- <version>${global_classindex_version}</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <descriptor>src/main/assembly/assembly.xml</descriptor>
- </configuration>
- <executions>
- <execution>
- <id>make-assembly</id> <!-- this is used for inheritance merges -->
- <phase>package</phase> <!-- bind to the packaging phase -->
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
+
</project>
diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/client/HBaseClient.java b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/client/HBaseClient.java
index c416f47..9ddf9b7 100644
--- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/client/HBaseClient.java
+++ b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/client/HBaseClient.java
@@ -20,13 +20,13 @@
package org.apache.metron.hbase.client;
+import static org.apache.commons.collections4.CollectionUtils.size;
+
import java.io.Closeable;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.List;
-
-import org.apache.commons.collections4.CollectionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
@@ -41,8 +41,6 @@ import org.apache.metron.hbase.bolt.mapper.HBaseProjectionCriteria;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.commons.collections4.CollectionUtils.size;
-
/**
* A client that interacts with HBase.
*/
diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-indexing/pom.xml
index d2c0f6d..ccb1a58 100644
--- a/metron-platform/metron-indexing/pom.xml
+++ b/metron-platform/metron-indexing/pom.xml
@@ -31,15 +31,22 @@
<guava_version>${global_hbase_guava_version}</guava_version>
</properties>
<dependencies>
+
+ <!-- Metron -->
+
<dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <version>2.8.2</version>
- <scope>test</scope>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-common</artifactId>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-common</artifactId>
+ <artifactId>metron-writer</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-storm-kafka</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
@@ -47,6 +54,16 @@
<artifactId>metron-hbase</artifactId>
<version>${project.parent.version}</version>
</dependency>
+
+ <!-- Other -->
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>${global_log4j_core_version}</version>
+ <scope>runtime</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
@@ -76,6 +93,10 @@
<artifactId>servlet-api</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
+ <exclusion>
+ <artifactId>log4j</artifactId>
+ <groupId>log4j</groupId>
+ </exclusion>
</exclusions>
<scope>provided</scope>
</dependency>
@@ -137,6 +158,10 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
+ <exclusion>
+ <artifactId>log4j</artifactId>
+ <groupId>log4j</groupId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -150,6 +175,9 @@
<artifactId>commons-validator</artifactId>
<version>1.4.0</version>
</dependency>
+
+ <!-- Test -->
+
<dependency>
<groupId>org.apache.metron</groupId>
<artifactId>metron-test-utilities</artifactId>
@@ -169,13 +197,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-enrichment</artifactId>
- <version>${project.parent.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>${global_kafka_version}</version>
@@ -201,6 +222,13 @@
<version>1.3</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
index ad39c85..6fa6094 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
@@ -18,18 +18,28 @@
package org.apache.metron.indexing.integration;
+import static org.apache.metron.common.configuration.ConfigurationsUtils.getClient;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.curator.framework.CuratorFramework;
import org.apache.metron.TestConstants;
import org.apache.metron.common.Constants;
import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.common.field.FieldNameConverter;
import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.enrichment.integration.components.ConfigUploadComponent;
import org.apache.metron.integration.BaseIntegrationTest;
import org.apache.metron.integration.ComponentRunner;
import org.apache.metron.integration.InMemoryComponent;
import org.apache.metron.integration.Processor;
import org.apache.metron.integration.ProcessorResult;
+import org.apache.metron.integration.components.ConfigUploadComponent;
import org.apache.metron.integration.components.FluxTopologyComponent;
import org.apache.metron.integration.components.KafkaComponent;
import org.apache.metron.integration.components.ZKServerComponent;
@@ -38,12 +48,6 @@ import org.apache.zookeeper.KeeperException;
import org.junit.Assert;
import org.junit.Test;
-import java.io.File;
-import java.util.*;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static org.apache.metron.common.configuration.ConfigurationsUtils.getClient;
-
public abstract class IndexingIntegrationTest extends BaseIntegrationTest {
protected static final String ERROR_TOPIC = "indexing_error";
protected String sampleParsedPath = TestConstants.SAMPLE_DATA_PARSED_PATH + "TestExampleParsed";
diff --git a/metron-platform/metron-management/pom.xml b/metron-platform/metron-management/pom.xml
index 8dd6e62..4b272ff 100644
--- a/metron-platform/metron-management/pom.xml
+++ b/metron-platform/metron-management/pom.xml
@@ -64,6 +64,12 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
+ <artifactId>metron-enrichment-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
<artifactId>stellar-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
@@ -118,6 +124,9 @@
</exclusion>
</exclusions>
</dependency>
+
+ <!-- Test -->
+
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
diff --git a/metron-platform/metron-parsing/metron-parsers-common/pom.xml b/metron-platform/metron-parsing/metron-parsers-common/pom.xml
index 4b9b5d4..46c46ca 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/pom.xml
+++ b/metron-platform/metron-parsing/metron-parsers-common/pom.xml
@@ -41,17 +41,6 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-enrichment</artifactId>
- <version>${project.parent.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
<artifactId>metron-hbase</artifactId>
<version>${project.parent.version}</version>
<scope>test</scope>
@@ -59,13 +48,6 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-enrichment</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
<artifactId>metron-pcap</artifactId>
<version>${project.parent.version}</version>
</dependency>
@@ -281,7 +263,6 @@
</excludes>
</artifactSet>
<transformers>
- <transformer implementation="org.atteo.classindex.ClassIndexTransformer"/>
<transformer
implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
<resources>
@@ -302,6 +283,8 @@
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass></mainClass>
</transformer>
+ <!-- ClassIndexTransformer needs to go LAST. For some reason it will clobber other transformers from operating when it is put first -->
+ <transformer implementation="org.atteo.classindex.ClassIndexTransformer"/>
</transformers>
</configuration>
</execution>
diff --git a/metron-platform/metron-parsing/metron-parsing-storm/pom.xml b/metron-platform/metron-parsing/metron-parsing-storm/pom.xml
index 99a870a..72770c8 100644
--- a/metron-platform/metron-parsing/metron-parsing-storm/pom.xml
+++ b/metron-platform/metron-parsing/metron-parsing-storm/pom.xml
@@ -46,6 +46,11 @@
<artifactId>metron-parsers</artifactId>
<version>${project.parent.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-enrichment-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
<!-- Storm dependencies -->
<dependency>
@@ -99,7 +104,7 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-enrichment</artifactId>
+ <artifactId>metron-common</artifactId>
<version>${project.parent.version}</version>
<scope>test</scope>
<type>test-jar</type>
@@ -138,7 +143,6 @@
<version>${global_log4j_core_version}</version>
<scope>test</scope>
</dependency>
-
</dependencies>
<build>
<plugins>
diff --git a/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
index 467429c..46fbc94 100644
--- a/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
+++ b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
@@ -37,7 +37,7 @@ import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.enrichment.converter.EnrichmentConverter;
import org.apache.metron.enrichment.converter.EnrichmentKey;
import org.apache.metron.enrichment.converter.EnrichmentValue;
-import org.apache.metron.enrichment.integration.components.ConfigUploadComponent;
+import org.apache.metron.integration.components.ConfigUploadComponent;
import org.apache.metron.enrichment.lookup.LookupKV;
import org.apache.metron.hbase.mock.MockHBaseTableProvider;
import org.apache.metron.hbase.mock.MockHTable;
@@ -58,7 +58,7 @@ public class SimpleHbaseEnrichmentWriterIntegrationTest extends BaseIntegrationT
/**
* {
* "parserClassName": "org.apache.metron.parsers.csv.CSVParser",
- * "writerClassName": "org.apache.metron.enrichment.writer.SimpleHbaseEnrichmentWriter",
+ * "writerClassName": "org.apache.metron.writer.hbase.SimpleHbaseEnrichmentWriter",
* "sensorTopic": "dummy",
* "outputTopic": "output",
* "errorTopic": "error",
diff --git a/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
index cecba3d..3e0f9ca 100644
--- a/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
+++ b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
@@ -42,12 +42,12 @@ import org.apache.metron.common.Constants;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.common.field.validation.FieldValidation;
import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.enrichment.integration.components.ConfigUploadComponent;
import org.apache.metron.integration.BaseIntegrationTest;
import org.apache.metron.integration.ComponentRunner;
import org.apache.metron.integration.Processor;
import org.apache.metron.integration.ProcessorResult;
import org.apache.metron.integration.ReadinessState;
+import org.apache.metron.integration.components.ConfigUploadComponent;
import org.apache.metron.integration.components.KafkaComponent;
import org.apache.metron.integration.components.ZKServerComponent;
import org.apache.metron.integration.processors.KafkaMessageSet;
diff --git a/metron-platform/metron-pcap/pom.xml b/metron-platform/metron-pcap/pom.xml
index 4b4020c..cf2da3c 100644
--- a/metron-platform/metron-pcap/pom.xml
+++ b/metron-platform/metron-pcap/pom.xml
@@ -88,6 +88,10 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
+ <exclusion>
+ <artifactId>commons-beanutils</artifactId>
+ <groupId>commons-beanutils</groupId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/pom.xml
index 79077b7..bee5143 100644
--- a/metron-platform/metron-solr/pom.xml
+++ b/metron-platform/metron-solr/pom.xml
@@ -40,53 +40,6 @@
<version>${global_solr_version}</version>
</dependency>
<dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-enrichment</artifactId>
- <version>${project.parent.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-profiler-client</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.solr</groupId>
- <artifactId>solr-test-framework</artifactId>
- <version>${global_solr_version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <artifactId>fastutil</artifactId>
- <groupId>it.unimi.dsi</groupId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- <version>${global_log4j_core_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <version>${global_log4j_core_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>${global_storm_version}</version>
@@ -175,6 +128,56 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-indexing</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- Test -->
+
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-test-framework</artifactId>
+ <version>${global_solr_version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>fastutil</artifactId>
+ <groupId>it.unimi.dsi</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>caffeine</artifactId>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>${global_log4j_core_version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>${global_log4j_core_version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>${global_kafka_version}</version>
@@ -189,6 +192,13 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
+ <artifactId>metron-common</artifactId>
+ <version>${project.parent.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
<artifactId>metron-integration-test</artifactId>
<version>${project.parent.version}</version>
<scope>test</scope>
@@ -220,30 +230,6 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-enrichment</artifactId>
- <version>${project.parent.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-indexing</artifactId>
- <version>${project.parent.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.apache.metron</groupId>
<artifactId>metron-hbase</artifactId>
@@ -267,7 +253,18 @@
<version>1.3</version>
<scope>test</scope>
</dependency>
-
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${global_jackson_version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${global_jackson_version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/utils/SampleUtil.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/enrichment/integration/utils/SampleUtil.java
similarity index 100%
rename from metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/utils/SampleUtil.java
rename to metron-platform/metron-solr/src/test/java/org/apache/metron/enrichment/integration/utils/SampleUtil.java
diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/TestConstants.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/TestConstants.java
index 53e49db..825346c 100644
--- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/TestConstants.java
+++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/TestConstants.java
@@ -22,7 +22,7 @@ public class TestConstants {
public final static String SAMPLE_CONFIG_PATH = "../metron-integration-test/src/main/config/zookeeper/";
public final static String PARSER_CONFIGS_PATH = "../metron-parsing/metron-parsers/src/main/config/zookeeper/";
public final static String PARSER_COMMON_CONFIGS_PATH = "../metron-parsing/metron-parsers-common/src/main/config/zookeeper/";
- public final static String ENRICHMENTS_CONFIGS_PATH = "../metron-enrichment/src/main/config/zookeeper/";
+ public final static String ENRICHMENTS_CONFIGS_PATH = "../metron-enrichment/metron-enrichment-common/src/main/config/zookeeper/";
public final static String SAMPLE_DATA_PATH = "../metron-integration-test/src/main/sample/data/";
public final static String SAMPLE_DATA_INPUT_PATH = "../metron-integration-test/src/main/sample/data/yaf/raw/";
public final static String SAMPLE_DATA_PARSED_PATH = "../metron-integration-test/src/main/sample/data/test/parsed/";
diff --git a/metron-platform/metron-writer/pom.xml b/metron-platform/metron-writer/pom.xml
index 45cba9d..2376bff 100644
--- a/metron-platform/metron-writer/pom.xml
+++ b/metron-platform/metron-writer/pom.xml
@@ -162,6 +162,11 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${global_httpclient_version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.10</artifactId>
<version>${global_kafka_version}</version>
@@ -212,11 +217,16 @@
</exclusions>
<scope>provided</scope>
</dependency>
- <dependency>
+ <dependency>
<groupId>org.apache.metron</groupId>
<artifactId>metron-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-enrichment-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.metron</groupId>
<artifactId>metron-test-utilities</artifactId>
@@ -225,6 +235,13 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
+ <artifactId>metron-hbase</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
<artifactId>stellar-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java
similarity index 99%
rename from metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java
rename to metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java
index 10507c7..50b11e1 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.metron.enrichment.writer;
+package org.apache.metron.writer.hbase;
import org.apache.metron.common.writer.BulkMessage;
import org.apache.metron.common.writer.MessageId;
@@ -47,6 +47,9 @@ import java.io.Serializable;
import java.util.*;
import java.util.stream.Collectors;
+/**
+ * Used primarily for streaming enrichments.
+ */
public class SimpleHbaseEnrichmentWriter extends AbstractWriter implements BulkMessageWriter<JSONObject>, Serializable {
private static final Logger LOG = LoggerFactory.getLogger(SimpleHbaseEnrichmentWriter.class);
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hbase/SimpleHBaseEnrichmentWriterTest.java
similarity index 99%
rename from metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
rename to metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hbase/SimpleHBaseEnrichmentWriterTest.java
index 64d1d61..980c5f2 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
+++ b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hbase/SimpleHBaseEnrichmentWriterTest.java
@@ -16,10 +16,16 @@
* limitations under the License.
*/
-package org.apache.metron.writers;
+package org.apache.metron.writer.hbase;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
@@ -28,21 +34,13 @@ import org.apache.metron.enrichment.converter.EnrichmentConverter;
import org.apache.metron.enrichment.converter.EnrichmentKey;
import org.apache.metron.enrichment.converter.EnrichmentValue;
import org.apache.metron.enrichment.lookup.LookupKV;
-import org.apache.metron.enrichment.writer.SimpleHbaseEnrichmentWriter;
-import org.apache.metron.hbase.mock.MockHTable;
import org.apache.metron.hbase.mock.MockHBaseTableProvider;
+import org.apache.metron.hbase.mock.MockHTable;
import org.json.simple.JSONObject;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Arrays;
-
public class SimpleHBaseEnrichmentWriterTest {
private static final String SENSOR_TYPE= "dummy";
private static final String TABLE_NAME= SENSOR_TYPE;
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hdfs/HdfsWriterTest.java b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hdfs/HdfsWriterTest.java
index ed78bf8..88ba4fd 100644
--- a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hdfs/HdfsWriterTest.java
+++ b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hdfs/HdfsWriterTest.java
@@ -162,6 +162,7 @@ public class HdfsWriterTest {
writer.init(new HashMap<String, String>(), createTopologyContext(), config);
String filename = writer.fileNameFormat.getName(1,1);
Assert.assertEquals("prefix-Xcom-7-1-1.json", filename);
+ writer.close();
}
@Test
@@ -209,6 +210,7 @@ public class HdfsWriterTest {
JSONObject message = new JSONObject();
writer.getHdfsPathExtension(SENSOR_NAME, "{'key':'value'}", message);
+ writer.close();
}
@Test
@@ -223,6 +225,7 @@ public class HdfsWriterTest {
for(int i = 0; i < maxFiles; i++) {
writer.getSourceHandler(SENSOR_NAME, Integer.toString(i), null);
}
+ writer.close();
}
@Test(expected=IllegalStateException.class)
@@ -237,6 +240,7 @@ public class HdfsWriterTest {
for(int i = 0; i < maxFiles+1; i++) {
writer.getSourceHandler(SENSOR_NAME, Integer.toString(i), null);
}
+ writer.close();
}
@Test
diff --git a/site-book/bin/generate-md.sh b/site-book/bin/generate-md.sh
index 7ebb5f6..1d1e6f8 100755
--- a/site-book/bin/generate-md.sh
+++ b/site-book/bin/generate-md.sh
@@ -66,7 +66,8 @@ RESOURCE_LIST=(
metron-platform/metron-job/metron-job_state_statechart_diagram.svg
metron-platform/metron-parsing/parser_arch.png
metron-platform/metron-indexing/indexing_arch.png
- metron-platform/metron-enrichment/enrichment_arch.png
+ metron-platform/metron-enrichment/metron-enrichment-storm/enrichment_arch.png
+ metron-platform/metron-enrichment/metron-enrichment-storm/unified_enrichment_arch.svg
metron-analytics/metron-maas-service/maas_arch.png
metron-contrib/metron-performance/performance_measurement.png
use-cases/forensic_clustering/find_alerts.png
@@ -75,7 +76,6 @@ RESOURCE_LIST=(
use-cases/parser_chaining/aggregated_parser_chaining_flow.svg
use-cases/typosquat_detection/squid_search.png
use-cases/typosquat_detection/drill_down.png
- metron-platform/metron-enrichment/unified_enrichment_arch.svg
)
## This is a list of duples, flattened into a bash array. Even fields are relative paths to a .md file
@@ -93,7 +93,8 @@ HREF_REWRITE_LIST=(
metron-deployment/Kerberos-ambari-setup.md 's#(readme-images/enable-kerberos-configure-kerberos.png)#(../images/enable-kerberos-configure-kerberos.png)#g'
metron-deployment/Kerberos-ambari-setup.md 's#(readme-images/enable-kerberos-started.png)#(../images/enable-kerberos-started.png)#g'
metron-deployment/Kerberos-ambari-setup.md 's#(readme-images/enable-kerberos.png)#(../images/enable-kerberos.png)#g'
- metron-platform/metron-enrichment/README.md 's#(enrichment_arch.png)#(../../images/enrichment_arch.png)#g'
+ metron-platform/metron-enrichment/metron-enrichment-storm/README.md 's#(enrichment_arch.png)#(../../../images/enrichment_arch.png)#g'
+ metron-platform/metron-enrichment/metron-enrichment-storm/README.md 's#(unified_enrichment_arch.svg)#(../../../images/unified_enrichment_arch.svg)#g'
metron-platform/metron-indexing/README.md 's#(indexing_arch.png)#(../../images/indexing_arch.png)#g'
metron-platform/metron-job/README.md 's#(metron-job_state_statechart_diagram.svg)#(../../images/metron-job_state_statechart_diagram.svg)#g'
metron-platform/metron-parsing/README.md 's#(parser_arch.png)#(../../images/parser_arch.png)#g'
@@ -106,7 +107,6 @@ HREF_REWRITE_LIST=(
use-cases/parser_chaining/README.md 's#(aggregated_parser_chaining_flow.svg)#(../../images/aggregated_parser_chaining_flow.svg)#g'
use-cases/typosquat_detection/README.md 's#(squid_search.png)#(../../images/squid_search.png)#g'
use-cases/typosquat_detection/README.md 's#(drill_down.png)#(../../images/drill_down.png)#g'
- metron-platform/metron-enrichment/README.md 's#(unified_enrichment_arch.svg)#(../../images/unified_enrichment_arch.svg)#g'
)
TEMPLATES_DIR="$METRON_SOURCE/site-book/src/site/src-resources/templates"