You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2019/05/30 13:23:55 UTC
[metron] branch master updated: METRON-2087 Remove Storm dependency
from metron-indexing (merrimanr) closes apache/metron#1389
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new b9a130c METRON-2087 Remove Storm dependency from metron-indexing (merrimanr) closes apache/metron#1389
b9a130c is described below
commit b9a130ca96774add38865d3934ed61ca19599b87
Author: merrimanr <me...@gmail.com>
AuthorDate: Thu May 30 08:23:37 2019 -0500
METRON-2087 Remove Storm dependency from metron-indexing (merrimanr) closes apache/metron#1389
---
.../packaging/ambari/metron-mpack/pom.xml | 6 +-
.../common-services/METRON/CURRENT/metainfo.xml | 12 +-
.../packaging/docker/rpm-docker/SPECS/metron.spec | 122 ++++---
.../packaging/docker/rpm-docker/pom.xml | 18 +-
metron-interface/metron-rest-client/pom.xml | 2 +-
metron-interface/metron-rest/pom.xml | 4 +-
metron-platform/metron-elasticsearch/README.md | 378 +-------------------
.../{ => metron-elasticsearch-common}/README.md | 0
.../metron-elasticsearch-common}/pom.xml | 225 +++++-------
.../elasticsearch/bulk/BulkDocumentWriter.java | 0
.../bulk/BulkDocumentWriterResults.java | 0
.../bulk/ElasticsearchBulkDocumentWriter.java | 0
.../bulk/ElasticsearchImportExport.java | 0
.../metron/elasticsearch/bulk/WriteFailure.java | 0
.../metron/elasticsearch/bulk/WriteSuccess.java | 0
.../elasticsearch/client/ElasticsearchClient.java | 0
.../client/ElasticsearchClientFactory.java | 0
.../config/ElasticsearchClientConfig.java | 0
.../config/ElasticsearchClientOptions.java | 0
.../dao/ElasticsearchColumnMetadataDao.java | 0
.../metron/elasticsearch/dao/ElasticsearchDao.java | 0
.../dao/ElasticsearchMetaAlertDao.java | 0
.../ElasticsearchMetaAlertRetrieveLatestDao.java | 0
.../dao/ElasticsearchMetaAlertSearchDao.java | 0
.../dao/ElasticsearchMetaAlertUpdateDao.java | 0
.../dao/ElasticsearchRequestSubmitter.java | 0
.../dao/ElasticsearchRetrieveLatestDao.java | 0
.../elasticsearch/dao/ElasticsearchSearchDao.java | 0
.../elasticsearch/dao/ElasticsearchUpdateDao.java | 0
.../elasticsearch/utils/ElasticsearchUtils.java | 0
.../metron/elasticsearch/utils/FieldMapping.java | 0
.../elasticsearch/utils/FieldProperties.java | 0
.../elasticsearch/writer/ElasticsearchWriter.java | 0
.../writer/MessageIdBasedDocument.java | 0
.../src/main/resources/META-INF/LICENSE | 0
.../src/main/resources/META-INF/NOTICE | 0
.../bulk/ElasticsearchBulkDocumentWriterTest.java | 0
.../bulk/ElasticsearchImportExportTest.java | 0
.../dao/ElasticsearchColumnMetadataDaoTest.java | 0
.../elasticsearch/dao/ElasticsearchDaoTest.java | 0
.../dao/ElasticsearchMetaAlertDaoTest.java | 0
.../dao/ElasticsearchRequestSubmitterTest.java | 0
.../dao/ElasticsearchUpdateDaoTest.java | 0
...ticsearchBulkDocumentWriterIntegrationTest.java | 2 +-
.../ElasticsearchMetaAlertIntegrationTest.java | 0
.../ElasticsearchSearchIntegrationTest.java | 4 +-
.../ElasticsearchUpdateIntegrationTest.java | 0
.../components/ElasticSearchComponent.java | 0
.../writer/ElasticsearchWriterTest.java | 0
.../src/test/resources/log4j.properties | 0
.../metron-elasticsearch-storm/README.md | 20 ++
.../metron-elasticsearch-storm}/pom.xml | 218 ++++--------
.../src/main/assembly/assembly.xml | 0
.../src/main/config/elasticsearch.properties | 0
.../src/main/config/elasticsearch.properties.j2 | 0
.../main/scripts/start_elasticsearch_topology.sh | 0
.../ElasticsearchIndexingIntegrationTest.java | 6 +-
metron-platform/metron-elasticsearch/pom.xml | 392 ++-------------------
metron-platform/metron-indexing/README.md | 286 +--------------
.../{ => metron-indexing-common}/README.md | 65 +---
.../{ => metron-indexing-common}/pom.xml | 18 +-
.../src/main/assembly/assembly.xml | 19 -
.../zeppelin/metron/metron-connection-report.json | 0
.../metron/metron-connection-volume-report.json | 0
.../config/zeppelin/metron/metron-ip-report.json | 0
.../zeppelin/metron/metron-yaf-telemetry.json | 0
.../src/main/config/zookeeper/indexing/asa.json | 0
.../src/main/config/zookeeper/indexing/bro.json | 0
.../src/main/config/zookeeper/indexing/error.json | 0
.../src/main/config/zookeeper/indexing/snort.json | 0
.../main/config/zookeeper/indexing/websphere.json | 0
.../src/main/config/zookeeper/indexing/yaf.json | 0
.../apache/metron/indexing/dao/AccessConfig.java | 0
.../metron/indexing/dao/ColumnMetadataDao.java | 0
.../org/apache/metron/indexing/dao/HBaseDao.java | 0
.../org/apache/metron/indexing/dao/IndexDao.java | 0
.../metron/indexing/dao/IndexDaoFactory.java | 0
.../metron/indexing/dao/IndexUpdateCallback.java | 0
.../apache/metron/indexing/dao/MultiIndexDao.java | 0
.../metron/indexing/dao/RetrieveLatestDao.java | 0
.../dao/metaalert/DeferredMetaAlertIndexDao.java | 0
.../dao/metaalert/MetaAlertAddRemoveRequest.java | 0
.../indexing/dao/metaalert/MetaAlertConfig.java | 0
.../indexing/dao/metaalert/MetaAlertConstants.java | 0
.../dao/metaalert/MetaAlertCreateRequest.java | 0
.../indexing/dao/metaalert/MetaAlertDao.java | 0
.../dao/metaalert/MetaAlertRetrieveLatestDao.java | 0
.../indexing/dao/metaalert/MetaAlertSearchDao.java | 0
.../indexing/dao/metaalert/MetaAlertStatus.java | 0
.../indexing/dao/metaalert/MetaAlertUpdateDao.java | 0
.../metron/indexing/dao/metaalert/MetaScores.java | 0
.../lucene/AbstractLuceneMetaAlertUpdateDao.java | 0
.../metron/indexing/dao/search/AlertComment.java | 0
.../metron/indexing/dao/search/FieldType.java | 0
.../metron/indexing/dao/search/GetRequest.java | 0
.../apache/metron/indexing/dao/search/Group.java | 0
.../metron/indexing/dao/search/GroupOrder.java | 0
.../metron/indexing/dao/search/GroupOrderType.java | 0
.../metron/indexing/dao/search/GroupRequest.java | 0
.../metron/indexing/dao/search/GroupResponse.java | 0
.../metron/indexing/dao/search/GroupResult.java | 0
.../dao/search/InvalidCreateException.java | 0
.../dao/search/InvalidSearchException.java | 0
.../metron/indexing/dao/search/SearchDao.java | 0
.../metron/indexing/dao/search/SearchRequest.java | 0
.../metron/indexing/dao/search/SearchResponse.java | 0
.../metron/indexing/dao/search/SearchResult.java | 0
.../metron/indexing/dao/search/SortField.java | 0
.../metron/indexing/dao/search/SortOrder.java | 0
.../dao/update/CommentAddRemoveRequest.java | 0
.../metron/indexing/dao/update/Document.java | 0
.../dao/update/OriginalNotFoundException.java | 0
.../metron/indexing/dao/update/PatchRequest.java | 0
.../metron/indexing/dao/update/UpdateDao.java | 0
.../metron/indexing/util/IndexingCacheUtil.java | 0
.../src/main/resources/META-INF/LICENSE | 0
.../src/main/resources/META-INF/NOTICE | 0
.../InMemoryMetaAlertRetrieveLatestDao.java | 0
.../apache/metron/indexing/dao/HBaseDaoTest.java | 0
.../apache/metron/indexing/dao/InMemoryDao.java | 0
.../metron/indexing/dao/InMemoryMetaAlertDao.java | 0
.../indexing/dao/InMemoryMetaAlertUpdateDao.java | 0
.../metron/indexing/dao/MultiIndexDaoTest.java | 0
.../metron/indexing/dao/SearchIntegrationTest.java | 0
.../apache/metron/indexing/dao/UpdateDaoTest.java | 0
.../metron/indexing/dao/UpdateIntegrationTest.java | 0
.../dao/metaalert/MetaAlertIntegrationTest.java | 0
.../indexing/dao/metaalert/MetaScoresTest.java | 0
.../AbstractLuceneMetaAlertUpdateDaoTest.java | 0
.../integration/HBaseDaoIntegrationTest.java | 0
.../indexing/util/IndexingCacheUtilTest.java | 0
.../src/test/resources/log4j.properties | 0
.../metron-indexing-storm/README.md | 79 +++++
.../{ => metron-indexing-storm}/indexing_arch.png | Bin
.../metron-indexing/metron-indexing-storm/pom.xml | 131 +++++++
.../src/main/assembly/assembly.xml | 0
.../src/main/config/hdfs.properties.j2 | 0
.../src/main/flux/indexing/batch/remote.yaml | 0
.../main/flux/indexing/random_access/remote.yaml | 0
.../src/main/scripts/start_hdfs_topology.sh | 2 +-
.../integration/HDFSIndexingIntegrationTest.java | 7 +-
.../integration/IndexingIntegrationTest.java | 9 +-
metron-platform/metron-indexing/pom.xml | 290 ++-------------
metron-platform/metron-solr/README.md | 153 +-------
.../metron-solr/{ => metron-solr-common}/README.md | 2 +-
.../metron-solr/{ => metron-solr-common}/pom.xml | 122 +++----
.../src/main/assembly/assembly.xml | 8 -
.../src/main/config/schema/bro/schema.xml | 0
.../src/main/config/schema/bro}/solrconfig.xml | 0
.../src/main/config/schema/error/schema.xml | 0
.../src/main/config/schema/error}/solrconfig.xml | 0
.../src/main/config/schema/metaalert/schema.xml | 0
.../main/config/schema/metaalert}/solrconfig.xml | 0
.../src/main/config/schema/snort/schema.xml | 0
.../src/main/config/schema/snort}/solrconfig.xml | 0
.../src/main/config/schema/yaf/schema.xml | 0
.../src/main/config/schema/yaf}/solrconfig.xml | 0
.../java/org/apache/metron/solr/SolrConstants.java | 0
.../metron/solr/client/SolrClientFactory.java | 0
.../metron/solr/dao/SolrColumnMetadataDao.java | 0
.../java/org/apache/metron/solr/dao/SolrDao.java | 0
.../apache/metron/solr/dao/SolrMetaAlertDao.java | 0
.../solr/dao/SolrMetaAlertRetrieveLatestDao.java | 0
.../metron/solr/dao/SolrMetaAlertSearchDao.java | 0
.../metron/solr/dao/SolrMetaAlertUpdateDao.java | 0
.../metron/solr/dao/SolrRetrieveLatestDao.java | 0
.../org/apache/metron/solr/dao/SolrSearchDao.java | 0
.../org/apache/metron/solr/dao/SolrUpdateDao.java | 0
.../org/apache/metron/solr/dao/SolrUtilities.java | 0
.../org/apache/metron/solr/schema/FieldType.java | 0
.../metron/solr/schema/SchemaTranslator.java | 0
.../metron/solr/writer/MetronSolrClient.java | 0
.../org/apache/metron/solr/writer/SolrWriter.java | 0
.../src/main/scripts/create_collection.sh | 0
.../src/main/scripts/delete_collection.sh | 0
.../src/main/scripts/install_solr.sh | 0
.../src/main/scripts/start_solr.sh | 0
.../src/main/scripts/stop_solr.sh | 0
.../enrichment/integration/utils/SampleUtil.java | 6 +-
.../metron/solr/client/SolrClientFactoryTest.java | 0
.../metron/solr/dao/SolrColumnMetadataTest.java | 0
.../org/apache/metron/solr/dao/SolrDaoTest.java | 0
.../metron/solr/dao/SolrMetaAlertDaoTest.java | 0
.../apache/metron/solr/dao/SolrSearchDaoTest.java | 0
.../apache/metron/solr/dao/SolrUpdateDaoTest.java | 0
.../apache/metron/solr/dao/SolrUtilitiesTest.java | 0
.../integration/SolrMetaAlertIntegrationTest.java | 4 +-
.../SolrRetrieveLatestIntegrationTest.java | 4 +-
.../integration/SolrSearchIntegrationTest.java | 4 +-
.../integration/SolrUpdateIntegrationTest.java | 4 +-
.../solr/integration/components/SolrComponent.java | 0
.../schema/SchemaValidationIntegrationTest.java | 0
.../solr/matcher/ModifiableSolrParamsMatcher.java | 0
.../solr/matcher/SolrInputDocumentListMatcher.java | 0
.../solr/matcher/SolrInputDocumentMatcher.java | 0
.../metron/solr/matcher/SolrQueryMatcher.java | 0
.../metron/solr/writer/MetronSolrClientTest.java | 0
.../apache/metron/solr/writer/SolrWriterTest.java | 0
.../test/resources/config/test/conf/managed-schema | 0
.../resources/config/test/conf}/solrconfig.xml | 0
.../src/test/resources/example_data/bro | 0
.../src/test/resources/example_data/error | 0
.../src/test/resources/example_data/snort | 0
.../src/test/resources/example_data/yaf | 0
.../src/test/resources/log4j.properties | 0
.../src/test/resources/log4j2.xml | 0
.../test/resources/solr/conf/_rest_managed.json | 0
.../src/test/resources/solr/conf/currency.xml | 0
.../test/resources/solr/conf/lang/stopwords_en.txt | 0
.../src/test/resources/solr/conf/protwords.txt | 0
.../src/test/resources/solr/conf/schema.xml | 0
.../src/test/resources/solr/conf/solrconfig.xml | 0
.../src/test/resources/solr/conf/stopwords.txt | 0
.../src/test/resources/solr/conf/synonyms.txt | 0
.../src/test/resources/solr/solr.xml | 0
.../metron-solr/metron-solr-storm/README.md | 20 ++
.../metron-solr/{ => metron-solr-storm}/pom.xml | 249 +++----------
.../src/main/assembly/assembly.xml | 0
.../src/main/config/solr.properties | 0
.../src/main/config/solr.properties.j2 | 0
.../src/main/scripts/start_solr_topology.sh | 0
.../integration/SolrIndexingIntegrationTest.java | 15 +-
metron-platform/metron-solr/pom.xml | 373 ++------------------
223 files changed, 767 insertions(+), 2512 deletions(-)
diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
index c2687a9..ae62bd8 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
@@ -116,21 +116,21 @@
<filtering>false</filtering>
</resource>
<resource>
- <directory>${basedir}/../../../../metron-platform/metron-elasticsearch/src/main/config</directory>
+ <directory>${basedir}/../../../../metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/main/config</directory>
<includes>
<include>elasticsearch.properties.j2</include>
</includes>
<filtering>false</filtering>
</resource>
<resource>
- <directory>${basedir}/../../../../metron-platform/metron-indexing/src/main/config</directory>
+ <directory>${basedir}/../../../../metron-platform/metron-indexing/metron-indexing-storm/src/main/config</directory>
<includes>
<include>hdfs.properties.j2</include>
</includes>
<filtering>false</filtering>
</resource>
<resource>
- <directory>${basedir}/../../../../metron-platform/metron-solr/src/main/config</directory>
+ <directory>${basedir}/../../../../metron-platform/metron-solr/metron-solr-storm/src/main/config</directory>
<includes>
<include>solr.properties.j2</include>
</includes>
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index ddc56cf..cb044c2 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -471,13 +471,19 @@
<name>metron-profiler-repl</name>
</package>
<package>
- <name>metron-indexing</name>
+ <name>metron-indexing-common</name>
</package>
<package>
- <name>metron-elasticsearch</name>
+ <name>metron-indexing-storm</name>
</package>
<package>
- <name>metron-solr</name>
+ <name>metron-elasticsearch-storm</name>
+ </package>
+ <package>
+ <name>metron-solr-common</name>
+ </package>
+ <package>
+ <name>metron-solr-storm</name>
</package>
<package>
<name>metron-pcap</name>
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 8b68b6f..2047d10 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -45,25 +45,27 @@ Group: Applications/Internet
URL: %{url}
Source0: metron-common-%{full_version}-archive.tar.gz
Source1: metron-parsers-common-%{full_version}-archive.tar.gz
-Source2: metron-elasticsearch-%{full_version}-archive.tar.gz
+Source2: metron-elasticsearch-storm-%{full_version}-archive.tar.gz
Source3: metron-data-management-%{full_version}-archive.tar.gz
-Source4: metron-solr-%{full_version}-archive.tar.gz
-Source5: metron-enrichment-common-%{full_version}-archive.tar.gz
-Source6: metron-enrichment-storm-%{full_version}-archive.tar.gz
-Source7: metron-indexing-%{full_version}-archive.tar.gz
-Source8: metron-pcap-backend-%{full_version}-archive.tar.gz
-Source9: metron-profiler-storm-%{full_version}-archive.tar.gz
-Source10: metron-rest-%{full_version}-archive.tar.gz
-Source11: metron-config-%{full_version}-archive.tar.gz
-Source12: metron-management-%{full_version}-archive.tar.gz
-Source13: metron-maas-service-%{full_version}-archive.tar.gz
-Source14: metron-alerts-%{full_version}-archive.tar.gz
-Source15: metron-performance-%{full_version}-archive.tar.gz
-Source16: metron-profiler-spark-%{full_version}-archive.tar.gz
-Source17: metron-profiler-repl-%{full_version}-archive.tar.gz
-Source18: metron-parsing-storm-%{full_version}-archive.tar.gz
-Source19: metron-parsers-%{full_version}-archive.tar.gz
-Source20: metron-hbase-server-%{full_version}-archive.tar.gz
+Source4: metron-solr-common-%{full_version}-archive.tar.gz
+Source5: metron-solr-storm-%{full_version}-archive.tar.gz
+Source6: metron-enrichment-common-%{full_version}-archive.tar.gz
+Source7: metron-enrichment-storm-%{full_version}-archive.tar.gz
+Source8: metron-indexing-common-%{full_version}-archive.tar.gz
+Source9: metron-indexing-storm-%{full_version}-archive.tar.gz
+Source10: metron-pcap-backend-%{full_version}-archive.tar.gz
+Source11: metron-profiler-storm-%{full_version}-archive.tar.gz
+Source12: metron-rest-%{full_version}-archive.tar.gz
+Source13: metron-config-%{full_version}-archive.tar.gz
+Source14: metron-management-%{full_version}-archive.tar.gz
+Source15: metron-maas-service-%{full_version}-archive.tar.gz
+Source16: metron-alerts-%{full_version}-archive.tar.gz
+Source17: metron-performance-%{full_version}-archive.tar.gz
+Source18: metron-profiler-spark-%{full_version}-archive.tar.gz
+Source19: metron-profiler-repl-%{full_version}-archive.tar.gz
+Source20: metron-parsing-storm-%{full_version}-archive.tar.gz
+Source21: metron-parsers-%{full_version}-archive.tar.gz
+Source22: metron-hbase-server-%{full_version}-archive.tar.gz
%description
Apache Metron provides a scalable advanced security analytics framework
@@ -107,6 +109,8 @@ tar -xzf %{SOURCE17} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE18} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE19} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE20} -C %{buildroot}%{metron_home}
+tar -xzf %{SOURCE21} -C %{buildroot}%{metron_home}
+tar -xzf %{SOURCE22} -C %{buildroot}%{metron_home}
install %{buildroot}%{metron_home}/bin/metron-management-ui %{buildroot}/etc/init.d/
install %{buildroot}%{metron_home}/bin/metron-alerts-ui %{buildroot}/etc/init.d/
@@ -222,15 +226,15 @@ This package installs the Metron Parser Storm files
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-%package elasticsearch
-Summary: Metron Elasticsearch Files
+%package elasticsearch-storm
+Summary: Metron Elasticsearch Storm Files
Group: Applications/Internet
-Provides: elasticsearch = %{version}
+Provides: elasticsearch-storm = %{version}
-%description elasticsearch
-This package installs the Metron Elasticsearch files
+%description elasticsearch-storm
+This package installs the Metron Elasticsearch Storm files
-%files elasticsearch
+%files elasticsearch-storm
%defattr(-,root,root,755)
%dir %{metron_root}
%dir %{metron_home}
@@ -239,7 +243,7 @@ This package installs the Metron Elasticsearch files
%dir %{metron_home}/lib
%{metron_home}/bin/start_elasticsearch_topology.sh
%{metron_home}/config/elasticsearch.properties
-%attr(0644,root,root) %{metron_home}/lib/metron-elasticsearch-%{full_version}-uber.jar
+%attr(0644,root,root) %{metron_home}/lib/metron-elasticsearch-storm-%{full_version}-uber.jar
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -288,28 +292,26 @@ This package installs the Metron Parser files
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-%package solr
-Summary: Metron Solr Files
+%package solr-common
+Summary: Metron Solr Common Files
Group: Applications/Internet
-Provides: solr = %{version}
+Provides: solr-common = %{version}
-%description solr
-This package installs the Metron Solr files
+%description solr-common
+This package installs the Metron Solr Common files
-%files solr
+%files solr-common
%defattr(-,root,root,755)
%dir %{metron_root}
%dir %{metron_home}
%dir %{metron_home}/bin
%dir %{metron_home}/config
-%dir %{metron_home}/lib
%{metron_home}/bin/create_collection.sh
%{metron_home}/bin/delete_collection.sh
%{metron_home}/bin/install_solr.sh
%{metron_home}/bin/start_solr.sh
%{metron_home}/bin/start_solr_topology.sh
%{metron_home}/bin/stop_solr.sh
-%{metron_home}/config/solr.properties
%{metron_home}/config/schema/bro/schema.xml
%{metron_home}/config/schema/bro/solrconfig.xml
%{metron_home}/config/schema/error/schema.xml
@@ -320,7 +322,27 @@ This package installs the Metron Solr files
%{metron_home}/config/schema/snort/solrconfig.xml
%{metron_home}/config/schema/yaf/schema.xml
%{metron_home}/config/schema/yaf/solrconfig.xml
-%attr(0644,root,root) %{metron_home}/lib/metron-solr-%{full_version}-uber.jar
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+%package solr-storm
+Summary: Metron Solr Storm Files
+Group: Applications/Internet
+Provides: solr-storm = %{version}
+
+%description solr-storm
+This package installs the Metron Solr Storm files
+
+%files solr-storm
+%defattr(-,root,root,755)
+%dir %{metron_root}
+%dir %{metron_home}
+%dir %{metron_home}/bin
+%dir %{metron_home}/config
+%dir %{metron_home}/lib
+%{metron_home}/bin/start_solr_topology.sh
+%{metron_home}/config/solr.properties
+%attr(0644,root,root) %{metron_home}/lib/metron-solr-storm-%{full_version}-uber.jar
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -374,24 +396,21 @@ This package installs the Metron Enrichment Storm files
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-%package indexing
+%package indexing-common
Summary: Metron Indexing Files
Group: Applications/Internet
-Provides: indexing = %{version}
+Provides: indexing-common = %{version}
-%description indexing
+%description indexing-common
This package installs the Metron Indexing files
-%files indexing
+%files indexing-common
%defattr(-,root,root,755)
%dir %{metron_root}
%dir %{metron_home}
%dir %{metron_home}/bin
%dir %{metron_home}/flux
%dir %{metron_home}/flux/indexing
-%{metron_home}/bin/start_hdfs_topology.sh
-%{metron_home}/flux/indexing/batch/remote.yaml
-%{metron_home}/flux/indexing/random_access/remote.yaml
%{metron_home}/config/zookeeper/indexing/bro.json
%{metron_home}/config/zookeeper/indexing/snort.json
%{metron_home}/config/zookeeper/indexing/websphere.json
@@ -405,6 +424,27 @@ This package installs the Metron Indexing files
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+%package indexing-storm
+Summary: Metron Indexing Storm Files
+Group: Applications/Internet
+Provides: indexing-storm = %{version}
+
+%description indexing-storm
+This package installs the Metron Indexing Storm files
+
+%files indexing-storm
+%defattr(-,root,root,755)
+%dir %{metron_root}
+%dir %{metron_home}
+%dir %{metron_home}/bin
+%dir %{metron_home}/flux
+%dir %{metron_home}/flux/indexing
+%{metron_home}/bin/start_hdfs_topology.sh
+%{metron_home}/flux/indexing/batch/remote.yaml
+%{metron_home}/flux/indexing/random_access/remote.yaml
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
%package metron-management
Summary: Metron Management Libary
Group: Applications/Internet
diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml
index f0aebeb..81e0c10 100644
--- a/metron-deployment/packaging/docker/rpm-docker/pom.xml
+++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml
@@ -120,7 +120,7 @@
</includes>
</resource>
<resource>
- <directory>${metron_dir}/metron-platform/metron-elasticsearch/target/</directory>
+ <directory>${metron_dir}/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/target/</directory>
<includes>
<include>*.tar.gz</include>
</includes>
@@ -144,7 +144,13 @@
</includes>
</resource>
<resource>
- <directory>${metron_dir}/metron-platform/metron-indexing/target/</directory>
+ <directory>${metron_dir}/metron-platform/metron-indexing/metron-indexing-common/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-indexing/metron-indexing-storm/target/</directory>
<includes>
<include>*.tar.gz</include>
</includes>
@@ -180,7 +186,13 @@
</includes>
</resource>
<resource>
- <directory>${metron_dir}/metron-platform/metron-solr/target/</directory>
+ <directory>${metron_dir}/metron-platform/metron-solr/metron-solr-common/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-solr/metron-solr-storm/target/</directory>
<includes>
<include>*.tar.gz</include>
</includes>
diff --git a/metron-interface/metron-rest-client/pom.xml b/metron-interface/metron-rest-client/pom.xml
index ea308e6..f830c24 100644
--- a/metron-interface/metron-rest-client/pom.xml
+++ b/metron-interface/metron-rest-client/pom.xml
@@ -39,7 +39,7 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-indexing</artifactId>
+ <artifactId>metron-indexing-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml
index 8324517..dcf63a3 100644
--- a/metron-interface/metron-rest/pom.xml
+++ b/metron-interface/metron-rest/pom.xml
@@ -429,13 +429,13 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-indexing</artifactId>
+ <artifactId>metron-indexing-common</artifactId>
<version>${project.parent.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-indexing</artifactId>
+ <artifactId>metron-indexing-common</artifactId>
<version>${project.parent.version}</version>
<type>test-jar</type>
<scope>test</scope>
diff --git a/metron-platform/metron-elasticsearch/README.md b/metron-platform/metron-elasticsearch/README.md
index 2bb0322..89ac36d 100644
--- a/metron-platform/metron-elasticsearch/README.md
+++ b/metron-platform/metron-elasticsearch/README.md
@@ -15,381 +15,17 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
-# Elasticsearch in Metron
-
-## Table of Contents
-
-* [Introduction](#introduction)
-* [Properties](#properties)
-* [Upgrading from 2.3.3 to 5.6](#upgrading-from-233-to-56)
-* [Type Mappings](#type-mappings)
-* [Using Metron with Elasticsearch 5.6](#using-metron-with-elasticsearch-56)
-* [Installing Elasticsearch Templates](#installing-elasticsearch-templates)
+# Elasticsearch
## Introduction
-Elasticsearch can be used as the real-time portion of the datastore resulting from [metron-indexing](../metron-indexing/README.md).
-
-## Properties
-
-### `es.clustername`
-
-The name of the elasticsearch Cluster. See [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#cluster.name)
-
-### `es.ip`
-
-Specifies the nodes in the elasticsearch cluster to use for writing.
-The format is one of the following:
-* A hostname or IP address with a port (e.g. `hostname1:1234`), in which case `es.port` is ignored.
-* A hostname or IP address without a port (e.g. `hostname1`), in which case `es.port` is used.
-* A string containing a CSV of hostnames without ports (e.g. `hostname1,hostname2,hostname3`) without spaces between. `es.port` is assumed to be the port for each host.
-* A string containing a CSV of hostnames with ports (e.g. `hostname1:1234,hostname2:1234,hostname3:1234`) without spaces between. `es.port` is ignored.
-* A list of hostnames with ports (e.g. `[ "hostname1:1234", "hostname2:1234"]`). Note, `es.port` is NOT used in this construction.
-
-### `es.port`
-
-The port for the elasticsearch hosts. This will be used in accordance with the discussion of `es.ip`.
-
-### `es.date.format`
-
-The date format to use when constructing the indices. For every message, the date format will be applied
-to the current time and that will become the last part of the index name where the message is written to.
-
-For instance, an `es.date.format` of `yyyy.MM.dd.HH` would have the consequence that the indices would
-roll hourly, whereas an `es.date.format` of `yyyy.MM.dd` would have the consequence that the indices would
-roll daily.
-
-### `es.client.settings`
-
-This field in global config allows you to specify Elasticsearch REST client options. These are used in conjunction with the previously mentioned Elasticsearch properties
-when setting up client connections to an Elasticsearch cluster. The available properties should be supplied as an object map. Current available options are as follows:
-
-| Property Name | Type | Required? | Default Value | Description |
-|-------------------------------------|-----------|-----------|----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| connection.timeout.millis | Integer | No | 1000 | Sets connection timeout. |
-| socket.timeout.millis | Integer | No | 30000 | Sets socket timeout. |
-| max.retry.timeout.millis | Integer | No | 30000 | Sets the maximum timeout (in milliseconds) to honour in case of multiple retries of the same request. |
-| num.client.connection.threads | Integer | No | 1 | Number of worker threads used by the connection manager. Defaults to Runtime.getRuntime().availableProcessors(). |
-| xpack.username | String | No | null | X-Pack username. |
-| xpack.password.file | String | No | null | 1-line HDFS file where the X-Pack password is set. |
-| ssl.enabled | Boolean | No | false | Turn on SSL connections. |
-| keystore.type | String | No | "jks" | Allows you to specify a keytstore type. See https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#KeyStore for more details. |
-| keystore.path | String | No | null | Path to the Trust Store that holds your Elasticsearch certificate authorities and certificate. |
-| keystore.password.file | String | No | null | 1-line HDFS file where the keystore password is set. |
-
-__Note:__ The migration from Elasticsearch's TransportClient to the Java REST client has resulted in some existing properties to change. Below is a mapping of the old properties to the new ones:
-
-| Old Property Name | New Property Name |
-|----------------------------------------|-------------------------------------|
-| client.transport.ping_timeout | n/a |
-| n/a | connection.timeout.millis |
-| n/a | socket.timeout.millis |
-| n/a | max.retry.timeout.millis |
-| n/a | num.client.connection.threads |
-| es.client.class | n/a |
-| es.xpack.username | xpack.username |
-| es.xpack.password.file | xpack.password.file |
-| xpack.security.transport.ssl.enabled | ssl.enabled |
-| xpack.ssl.key | n/a |
-| xpack.ssl.certificate | n/a |
-| xpack.ssl.certificate_authorities | n/a |
-| n/a | keystore.type |
-| keystore.path | keystore.path |
-| n/a | keystore.password.file |
-
-__Notes:__
-* The transport client implementation provides for a 'xpack.security.user' property, however we never used this property directly. Rather, in order to secure the password we used custom properties for user/pass. These properties have been carried over as `xpack.username` and `xpack.password.file`.
-* See [https://www.elastic.co/guide/en/elasticsearch/client/java-rest/5.6/_common_configuration.html](https://www.elastic.co/guide/en/elasticsearch/client/java-rest/5.6/_common_configuration.html) for more specifics on the new client properties.
-* Other notes on JSSE - [https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html](https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html)
-
-## Upgrading from 2.3.3 to 5.6
-
-Users should be prepared to re-index when migrating from Elasticsearch 2.3.3 to 5.6. There are a number of template changes, most notably around
-string type handling, that may cause issues when upgrading.
-
-[https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setup-upgrade.html](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setup-upgrade.html)
-
-Be aware that if you add a new string value and want to be able to filter and search on this value from the Alerts UI, you **must** add a mapping for that type to
-the appropriate Elasticsearch template. Below is more detail on how to choose the appropriate mapping type for your string value.
-
-## Type Mappings
-
-Type mappings have changed quite a bit from ES 2.x -> 5.x. Here is a brief rundown of the biggest changes. More detailed references from Elasticsearch
-are provided in the [Type Mapping References](#type-mapping-references) section below.
-* string fields replaced by text/keyword type
-* strings have new default mappings as follows
-
- ```
- {
- "type": "text",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- ```
-
-* There is no longer a `_timestamp` field that you can set "enabled" on. This field now causes an exception on templates.
-Replace with an application-created timestamp of "date" type.
-
-The semantics for string types have changed. In 2.x, you have the concept of index settings as either "analyzed" or "not_analyzed" which basically means "full text" and "keyword", respectively.
-Analyzed text basically means the indexer will split the text using a text analyzer thus allowing you to search on substrings within the original text. "New York" is split and indexed as two buckets,
- "New" and "York", so you can search or query for aggregate counts for those terms independently and will match against the individual terms "New" or "York." "Keyword" means that the original text
- will not be split/analyzed during indexing and instead treated as a whole unit, i.e. "New" or "York" will not match in searches against the document containing "New York", but searching on "New York"
- as the full city name will. In 5.x language instead of using the "index" setting, you now set the "type" to either "text" for full text, or "keyword" for keywords.
-
-Below is a table depicting the changes to how String types are now handled.
-
-<table>
-<tr>
- <th>sort, aggregate, or access values</th>
- <th>ES 2.x</th>
- <th>ES 5.x</th>
- <th>Example</th>
-</tr>
-<tr>
- <td>no</td>
- <td>
-<pre><code>"my_property" : {
- "type": "string",
- "index": "analyzed"
-}
-</code></pre>
- </td>
- <td>
-<pre><code>"my_property" : {
- "type": "text"
-}
-</code></pre>
- Additional defaults: "index": "true", "fielddata": "false"
- </td>
- <td>
- "New York" handled via in-mem search as "New" and "York" buckets. <strong>No</strong> aggregation or sort.
- </td>
-</tr>
-<tr>
- <td>
- yes
- </td>
- <td>
-<pre><code>"my_property": {
- "type": "string",
- "index": "analyzed"
-}
-</code></pre>
- </td>
- <td>
-<pre><code>"my_property": {
- "type": "text",
- "fielddata": "true"
-}
-</code></pre>
- </td>
- <td>
- "New York" handled via in-mem search as "New" and "York" buckets. <strong>Can</strong> aggregate and sort.
- </td>
-</tr>
-<tr>
- <td>
- yes
- </td>
- <td>
-<pre><code>"my_property": {
- "type": "string",
- "index": "not_analyzed"
-}
-</code></pre>
- </td>
- <td>
-<pre><code>"my_property" : {
- "type": "keyword"
-}
-</code></pre>
- </td>
- <td>
- "New York" searchable as single value. <strong>Can</strong> aggregate and sort. A search for "New" or "York" will not match against the whole value.
- </td>
-</tr>
-<tr>
- <td>
- yes
- </td>
- <td>
-<pre><code>"my_property": {
- "type": "string",
- "index": "analyzed"
-}
-</code></pre>
- </td>
- <td>
-<pre><code>"my_property": {
- "type": "text",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
-}
-</code></pre>
- </td>
- <td>
- "New York" searchable as single value or as text document, can aggregate and sort on the sub term "keyword."
- </td>
-</tr>
-</table>
-
-If you want to set default string behavior for all strings for a given index and type, you can do so with a mapping similar to the following (replace ${your_type_here} accordingly):
-
-```
-# curl -XPUT 'http://${ES_HOST}:${ES_PORT}/_template/default_string_template' -d '
-{
- "template": "*",
- "mappings" : {
- "${your_type_here}": {
- "dynamic_templates": [
- {
- "strings": {
- "match_mapping_type": "string",
- "mapping": {
- "type": "text"
- }
- }
- }
- ]
- }
- }
-}
-'
-```
-
-By specifying the "template" property with value "*" the template will apply to all indexes that have documents indexed of the specified type (${your_type_here}). This results in the following template.
-
-```
-# curl -XGET 'http://${ES_HOST}:${ES_PORT}/_template/default_string_template?pretty'
-{
- "default_string_template" : {
- "order" : 0,
- "template" : "*",
- "settings" : { },
- "mappings" : {
- "${your_type_here}" : {
- "dynamic_templates" : [
- {
- "strings" : {
- "match_mapping_type" : "string",
- "mapping" : {
- "type" : "text"
- }
- }
- }
- ]
- }
- },
- "aliases" : { }
- }
-}
-```
-
-Notes on other settings for types in ES
-* doc_values
- * on-disk data structure
- * provides access for sorting, aggregation, and field values
- * stores same values as _source, but in column-oriented fashion better for sorting and aggregating
- * not supported on text fields
- * enabled by default
-* fielddata
- * in-memory data structure
- * provides access for sorting, aggregation, and field values
- * primarily for text fields
- * disabled by default because the heap space required can be large
-
-
-##### Type Mapping References
-* [https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping.html](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping.html)
-* [https://www.elastic.co/guide/en/elasticsearch/reference/5.6/breaking_50_mapping_changes.html](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/breaking_50_mapping_changes.html)
-* [https://www.elastic.co/blog/strings-are-dead-long-live-strings](https://www.elastic.co/blog/strings-are-dead-long-live-strings)
-
-### Metron Properties
-
-Metron depends on some internal fields being defined in sensor templates. A field is defined in Elasticsearch by adding an entry to the `properties` section of the template:
-```
-"properties": {
- "metron_field": {
- "type": "keyword"
- }
-}
-```
-
-The following is a list of properties that need to be defined along with their type:
-* source:type - keyword
-* alert_status - keyword
-* metron_alert - nested
-
-## Using Metron with Elasticsearch 5.6
-
-Although infrequent, sometimes an internal field is added in Metron and existing templates must be updated. The following steps outlines how to do this, using `metron_alert` as an example.
-
-With the addition of the meta alert feature, there is a requirement that all sensors templates have a nested `metron_alert` field defined. This field is a dummy field. See [Ignoring Unmapped Fields](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-sort.html#_ignoring_unmapped_fields) for more information
-
-Without this field, an error will be thrown during ALL searches (including from UIs, resulting in no alerts being found for any sensor). This error will be found in the REST service's logs.
-
-Exception seen:
-```
-QueryParsingException[[nested] failed to find nested object under path [metron_alert]];
-```
-
-There are two steps to resolve this issue. First is to update the Elasticsearch template for each sensor, so any new indices have the field. This requires retrieving the template, removing an extraneous JSON field so we can put it back later, and adding our new field.
-
-Make sure to set the ELASTICSEARCH variable appropriately. $SENSOR can contain wildcards, so if rollover has occurred, it's not necessary to do each index individually. The example here appends `index*` to get all indexes for the provided sensor.
-
-```
-export ELASTICSEARCH="node1"
-export SENSOR="bro"
-curl -XGET "http://${ELASTICSEARCH}:9200/_template/${SENSOR}_index*?pretty=true" -o "${SENSOR}.template"
-sed -i '' '2d;$d' ./${SENSOR}.template
-sed -i '' '/"properties" : {/ a\
-"metron_alert": { "type": "nested"},' ${SENSOR}.template
-```
-
-To manually verify this, you can optionally pretty print it again with:
-```
-python -m json.tool bro.template
-```
-
-We'll want to put the template back into Elasticsearch:
-```
-curl -XPUT "http://${ELASTICSEARCH}:9200/_template/${SENSOR}_index" -d @${SENSOR}.template
-```
-
-To update existing indexes, update Elasticsearch mappings with the new field for each sensor.
-
-```
-curl -XPUT "http://${ELASTICSEARCH}:9200/${SENSOR}_index*/_mapping/${SENSOR}_doc" -d '
-{
- "properties" : {
- "metron_alert" : {
- "type" : "nested"
- }
- }
-}
-'
-rm ${SENSOR}.template
-```
-
-## Installing Elasticsearch Templates
-
-The stock set of Elasticsearch templates for bro, snort, yaf, error index and meta index are installed automatically during the first time install and startup of Metron Indexing service.
-
-It is possible that Elasticsearch service is not available when the Metron Indexing Service startup, in that case the Elasticsearch template will not be installed.
+The `elasticsearch` module is a module dedicated to indexing data into the Elasticsearch search engine.
-For such a scenario, an Admin can have the template installed in two ways:
+## Deployment Options
-_Method 1_ - Manually from the Ambari UI by following the flow:
-Ambari UI -> Services -> Metron -> Service Actions -> Elasticsearch Template Install
+There is currently one option for indexing into Elasticsearch, which is the `random_access_indexing` Storm topology.
-_Method 2_ - Stop the Metron Indexing service, and start it again from Ambari UI. Note that the Metron Indexing service tracks if it has successfully installed the Elasticsearch templates, and will attempt to do so each time it is Started until successful.
+## Submodules
-> Note: If you have made any customization to your index templates, then installing Elasticsearch templates afresh will lead to overwriting your existing changes. Please exercise caution.
+* metron-elasticsearch-common - this module contains common Elasticsearch code including Elasticsearch implementations of IndexDao related-classes.
+* metron-elasticsearch-storm - this module is home to Storm-specific code such as Flux property files, topology start script, and Storm indexing integration tests.
\ No newline at end of file
diff --git a/metron-platform/metron-elasticsearch/README.md b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/README.md
similarity index 100%
copy from metron-platform/metron-elasticsearch/README.md
copy to metron-platform/metron-elasticsearch/metron-elasticsearch-common/README.md
diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/pom.xml
similarity index 56%
copy from metron-platform/metron-indexing/pom.xml
copy to metron-platform/metron-elasticsearch/metron-elasticsearch-common/pom.xml
index f7067b1..a3a6dd7 100644
--- a/metron-platform/metron-indexing/pom.xml
+++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/pom.xml
@@ -17,58 +17,59 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-platform</artifactId>
+ <artifactId>metron-elasticsearch</artifactId>
<version>0.7.2</version>
</parent>
- <artifactId>metron-indexing</artifactId>
- <name>metron-indexing</name>
+ <artifactId>metron-elasticsearch-common</artifactId>
+ <name>metron-elasticsearch-common</name>
<url>https://metron.apache.org/</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <slf4j.version>1.7.7</slf4j.version>
- <storm.hdfs.version>0.1.2</storm.hdfs.version>
<guava_version>${global_hbase_guava_version}</guava_version>
</properties>
<dependencies>
-
- <!-- Metron -->
-
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-common</artifactId>
+ <artifactId>elasticsearch-shaded</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-writer-storm</artifactId>
+ <artifactId>metron-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-storm-kafka</artifactId>
+ <artifactId>metron-writer-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
<artifactId>metron-hbase</artifactId>
<version>${project.parent.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
-
- <!-- Other -->
-
<dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <version>${global_log4j_core_version}</version>
- <scope>runtime</scope>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava_version}</version>
</dependency>
-
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${global_hbase_version}</version>
- <scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -79,8 +80,12 @@
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -94,22 +99,15 @@
<groupId>javax.servlet</groupId>
</exclusion>
<exclusion>
- <artifactId>log4j</artifactId>
- <groupId>log4j</groupId>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
</exclusion>
</exclusions>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava_version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${global_hadoop_version}</version>
- <scope>provided</scope>
<exclusions>
<exclusion>
<artifactId>servlet-api</artifactId>
@@ -119,42 +117,34 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
- <exclusion>
- <artifactId>log4j</artifactId>
- <groupId>log4j</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <version>${global_mockito_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>commons-validator</groupId>
- <artifactId>commons-validator</artifactId>
- <version>1.4.0</version>
+ </exclusions>
</dependency>
-
- <!-- Test -->
-
<dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-test-utilities</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.10</artifactId>
+ <version>${global_kafka_version}</version>
<exclusions>
<exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-integration-test</artifactId>
- <version>${project.parent.version}</version>
+ <!-- must be near the top of the dependencies to force retrieval of the correct netty version -->
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>4.1.13.Final</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.plugin</groupId>
+ <artifactId>transport-netty4-client</artifactId>
+ <version>${global_elasticsearch_version}</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -172,10 +162,23 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-hbase</artifactId>
+ <artifactId>metron-integration-test</artifactId>
<version>${project.parent.version}</version>
- <type>test-jar</type>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
@@ -184,11 +187,35 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>${global_mockito_version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-common</artifactId>
+ <artifactId>metron-indexing-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-indexing-common</artifactId>
<version>${project.parent.version}</version>
+ <type>test-jar</type>
<scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-common</artifactId>
+ <version>${project.parent.version}</version>
<type>test-jar</type>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
<build>
@@ -205,84 +232,6 @@
</execution>
</executions>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${global_shade_version}</version>
- <configuration>
- <createDependencyReducedPom>true</createDependencyReducedPom>
- <artifactSet>
- <excludes>
- <exclude>*slf4j*</exclude>
- </excludes>
- </artifactSet>
- </configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <shadedArtifactAttached>true</shadedArtifactAttached>
- <shadedClassifierName>uber</shadedClassifierName>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- <relocations>
- <relocation>
- <pattern>com.google.common</pattern>
- <shadedPattern>org.apache.metron.guava.${guava_version}</shadedPattern>
- </relocation>
- </relocations>
- <transformers>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
- <resources>
- <resource>.yaml</resource>
- <resource>LICENSE.txt</resource>
- <resource>ASL2.0</resource>
- <resource>NOTICE.txt</resource>
- </resources>
- </transformer>
- <!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE -->
- <!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
- <addHeader>false</addHeader>
- <projectName>${project.name}</projectName>
- </transformer-->
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass></mainClass>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <descriptor>src/main/assembly/assembly.xml</descriptor>
- </configuration>
- <executions>
- <execution>
- <id>make-assembly</id> <!-- this is used for inheritance merges -->
- <phase>package</phase> <!-- bind to the packaging phase -->
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
</project>
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/BulkDocumentWriter.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/bulk/BulkDocumentWriter.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/BulkDocumentWriter.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/bulk/BulkDocumentWriter.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/BulkDocumentWriterResults.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/bulk/BulkDocumentWriterResults.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/BulkDocumentWriterResults.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/bulk/BulkDocumentWriterResults.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchBulkDocumentWriter.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchBulkDocumentWriter.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchBulkDocumentWriter.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchBulkDocumentWriter.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/WriteFailure.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/bulk/WriteFailure.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/WriteFailure.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/bulk/WriteFailure.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/WriteSuccess.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/bulk/WriteSuccess.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/WriteSuccess.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/bulk/WriteSuccess.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/client/ElasticsearchClient.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/client/ElasticsearchClient.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/client/ElasticsearchClient.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/client/ElasticsearchClient.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/client/ElasticsearchClientFactory.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/client/ElasticsearchClientFactory.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/client/ElasticsearchClientFactory.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/client/ElasticsearchClientFactory.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/config/ElasticsearchClientConfig.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/config/ElasticsearchClientConfig.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/config/ElasticsearchClientConfig.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/config/ElasticsearchClientConfig.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/config/ElasticsearchClientOptions.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/config/ElasticsearchClientOptions.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/config/ElasticsearchClientOptions.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/config/ElasticsearchClientOptions.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertRetrieveLatestDao.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertRetrieveLatestDao.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertRetrieveLatestDao.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertRetrieveLatestDao.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertSearchDao.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertSearchDao.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertSearchDao.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertSearchDao.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchRequestSubmitter.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchRequestSubmitter.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchRequestSubmitter.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchRequestSubmitter.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchRetrieveLatestDao.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchRetrieveLatestDao.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchRetrieveLatestDao.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchRetrieveLatestDao.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/FieldMapping.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/utils/FieldMapping.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/FieldMapping.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/utils/FieldMapping.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/FieldProperties.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/utils/FieldProperties.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/FieldProperties.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/utils/FieldProperties.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/MessageIdBasedDocument.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/writer/MessageIdBasedDocument.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/MessageIdBasedDocument.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/writer/MessageIdBasedDocument.java
diff --git a/metron-platform/metron-elasticsearch/src/main/resources/META-INF/LICENSE b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/resources/META-INF/LICENSE
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/resources/META-INF/LICENSE
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/resources/META-INF/LICENSE
diff --git a/metron-platform/metron-elasticsearch/src/main/resources/META-INF/NOTICE b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/resources/META-INF/NOTICE
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/resources/META-INF/NOTICE
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/resources/META-INF/NOTICE
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/bulk/ElasticsearchBulkDocumentWriterTest.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/bulk/ElasticsearchBulkDocumentWriterTest.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/bulk/ElasticsearchBulkDocumentWriterTest.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/bulk/ElasticsearchBulkDocumentWriterTest.java
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExportTest.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExportTest.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExportTest.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExportTest.java
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDaoTest.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDaoTest.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDaoTest.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDaoTest.java
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchRequestSubmitterTest.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchRequestSubmitterTest.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchRequestSubmitterTest.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchRequestSubmitterTest.java
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDaoTest.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDaoTest.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDaoTest.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDaoTest.java
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchBulkDocumentWriterIntegrationTest.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchBulkDocumentWriterIntegrationTest.java
similarity index 97%
rename from metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchBulkDocumentWriterIntegrationTest.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchBulkDocumentWriterIntegrationTest.java
index df4aeb0..9074c7d 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchBulkDocumentWriterIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchBulkDocumentWriterIntegrationTest.java
@@ -60,7 +60,7 @@ public class ElasticsearchBulkDocumentWriterIntegrationTest {
@ClassRule
public static TemporaryFolder indexDir = new TemporaryFolder();
- private static String broTemplatePath = "../../metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template";
+ private static String broTemplatePath = "../../../metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template";
private static ElasticSearchComponent elasticsearch;
private ElasticsearchClient client;
private ElasticsearchBulkDocumentWriter<Document> writer;
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
similarity index 97%
rename from metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index 705beda..7da94e9 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -66,8 +66,8 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static String indexDir = "target/elasticsearch_search";
private static String dateFormat = "yyyy.MM.dd.HH";
- private static String broTemplatePath = "../../metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template";
- private static String snortTemplatePath = "../../metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template";
+ private static String broTemplatePath = "../../../metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template";
+ private static String snortTemplatePath = "../../../metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template";
protected static final String BRO_INDEX = "bro_index_2017.01.01.01";
protected static final String SNORT_INDEX = "snort_index_2017.01.01.02";
protected static Map<String, Object> globalConfig;
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriterTest.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriterTest.java
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriterTest.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriterTest.java
diff --git a/metron-platform/metron-solr/src/test/resources/log4j.properties b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/resources/log4j.properties
similarity index 100%
rename from metron-platform/metron-solr/src/test/resources/log4j.properties
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/resources/log4j.properties
diff --git a/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/README.md b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/README.md
new file mode 100644
index 0000000..9bfa0de
--- /dev/null
+++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/README.md
@@ -0,0 +1,20 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# Metron Elasticsearch Storm
+
+This module contains assets for running the Random Access Indexing Storm topology with Elasticsearch. An integration test for this topology is also included.
diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/pom.xml
similarity index 52%
copy from metron-platform/metron-indexing/pom.xml
copy to metron-platform/metron-elasticsearch/metron-elasticsearch-storm/pom.xml
index f7067b1..1804098 100644
--- a/metron-platform/metron-indexing/pom.xml
+++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/pom.xml
@@ -17,31 +17,23 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-platform</artifactId>
+ <artifactId>metron-elasticsearch</artifactId>
<version>0.7.2</version>
</parent>
- <artifactId>metron-indexing</artifactId>
- <name>metron-indexing</name>
+ <artifactId>metron-elasticsearch-storm</artifactId>
+ <name>metron-elasticsearch-storm</name>
+ <description>Components common to Storm</description>
<url>https://metron.apache.org/</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <slf4j.version>1.7.7</slf4j.version>
- <storm.hdfs.version>0.1.2</storm.hdfs.version>
+ <commons.config.version>1.10</commons.config.version>
<guava_version>${global_hbase_guava_version}</guava_version>
</properties>
<dependencies>
-
- <!-- Metron -->
-
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-common</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-writer-storm</artifactId>
+ <artifactId>elasticsearch-shaded</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
@@ -50,172 +42,83 @@
<version>${project.parent.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-hbase</artifactId>
- <version>${project.parent.version}</version>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>flux-core</artifactId>
+ <version>${global_flux_version}</version>
</dependency>
-
- <!-- Other -->
-
<dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <version>${global_log4j_core_version}</version>
- <scope>runtime</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <version>${global_hbase_version}</version>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ <version>${global_storm_version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${global_hadoop_version}</version>
- <exclusions>
- <exclusion>
<artifactId>servlet-api</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
<exclusion>
- <artifactId>log4j</artifactId>
- <groupId>log4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <groupId>org.slf4j</groupId>
</exclusion>
- </exclusions>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava_version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${global_hadoop_version}</version>
- <scope>provided</scope>
- <exclusions>
<exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
</exclusion>
<exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
</exclusion>
<exclusion>
- <artifactId>log4j</artifactId>
- <groupId>log4j</groupId>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <version>${global_mockito_version}</version>
+ <groupId>org.elasticsearch.plugin</groupId>
+ <artifactId>transport-netty4-client</artifactId>
+ <version>${global_elasticsearch_version}</version>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>commons-validator</groupId>
- <artifactId>commons-validator</artifactId>
- <version>1.4.0</version>
- </dependency>
-
- <!-- Test -->
-
- <dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-test-utilities</artifactId>
+ <artifactId>metron-indexing-storm</artifactId>
<version>${project.parent.version}</version>
- <scope>test</scope>
<exclusions>
<exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-integration-test</artifactId>
+ <artifactId>metron-indexing-storm</artifactId>
<version>${project.parent.version}</version>
+ <type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- <version>${global_kafka_version}</version>
- <classifier>test</classifier>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-hbase</artifactId>
+ <artifactId>metron-elasticsearch-common</artifactId>
<version>${project.parent.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-core</artifactId>
- <version>1.3</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-common</artifactId>
+ <artifactId>metron-elasticsearch-common</artifactId>
<version>${project.parent.version}</version>
- <scope>test</scope>
<type>test-jar</type>
+ <scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <version>${global_jar_version}</version>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>${global_shade_version}</version>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
- <artifactSet>
- <excludes>
- <exclude>*slf4j*</exclude>
- </excludes>
- </artifactSet>
</configuration>
<executions>
<execution>
@@ -227,46 +130,65 @@
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
<filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
</filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
<shadedPattern>org.apache.metron.guava.${guava_version}</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>org.apache.metron.jackson</shadedPattern>
+ </relocation>
</relocations>
+ <artifactSet>
+ <excludes>
+ <exclude>org.slf4j.impl*</exclude>
+ <exclude>org.slf4j:slf4j-log4j*</exclude>
+ </excludes>
+ </artifactSet>
<transformers>
<transformer
- implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
- <resources>
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resources>
<resource>.yaml</resource>
<resource>LICENSE.txt</resource>
<resource>ASL2.0</resource>
<resource>NOTICE.txt</resource>
- </resources>
+ </resources>
</transformer>
<!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE -->
<!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
<addHeader>false</addHeader>
<projectName>${project.name}</projectName>
- </transformer-->
+ </transformer>-->
<transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass></mainClass>
</transformer>
+ <!-- ClassIndexTransformer needs to go LAST. For some reason it will clobber other transformers from operating when it is put first -->
+ <transformer implementation="org.atteo.classindex.ClassIndexTransformer"/>
</transformers>
</configuration>
</execution>
</executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.atteo.classindex</groupId>
+ <artifactId>classindex-transformer</artifactId>
+ <version>${global_classindex_version}</version>
+ </dependency>
+ </dependencies>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
@@ -274,13 +196,13 @@
<descriptor>src/main/assembly/assembly.xml</descriptor>
</configuration>
<executions>
- <execution>
- <id>make-assembly</id> <!-- this is used for inheritance merges -->
- <phase>package</phase> <!-- bind to the packaging phase -->
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
+ <execution>
+ <id>make-assembly</id> <!-- this is used for inheritance merges -->
+ <phase>package</phase> <!-- bind to the packaging phase -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
</executions>
</plugin>
</plugins>
diff --git a/metron-platform/metron-elasticsearch/src/main/assembly/assembly.xml b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/main/assembly/assembly.xml
similarity index 100%
copy from metron-platform/metron-elasticsearch/src/main/assembly/assembly.xml
copy to metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/main/assembly/assembly.xml
diff --git a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/main/config/elasticsearch.properties
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/main/config/elasticsearch.properties
diff --git a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties.j2 b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/main/config/elasticsearch.properties.j2
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties.j2
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/main/config/elasticsearch.properties.j2
diff --git a/metron-platform/metron-elasticsearch/src/main/scripts/start_elasticsearch_topology.sh b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/main/scripts/start_elasticsearch_topology.sh
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/main/scripts/start_elasticsearch_topology.sh
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/main/scripts/start_elasticsearch_topology.sh
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/test/java/org/apache/metron/indexing/integration/ElasticsearchIndexingIntegrationTest.java
similarity index 96%
rename from metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java
rename to metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/test/java/org/apache/metron/indexing/integration/ElasticsearchIndexingIntegrationTest.java
index 2784db3..e4867f6 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/test/java/org/apache/metron/indexing/integration/ElasticsearchIndexingIntegrationTest.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.metron.elasticsearch.integration;
+package org.apache.metron.indexing.integration;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.field.FieldNameConverter;
@@ -146,11 +146,11 @@ public class ElasticsearchIndexingIntegrationTest extends IndexingIntegrationTes
@Override
public String getTemplatePath() {
- return "../metron-elasticsearch/src/main/config/elasticsearch.properties.j2";
+ return "./src/main/config/elasticsearch.properties.j2";
}
@Override
public String getFluxPath() {
- return "../metron-indexing/src/main/flux/indexing/random_access/remote.yaml";
+ return "../../metron-indexing/metron-indexing-storm/src/main/flux/indexing/random_access/remote.yaml";
}
}
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index 917d484..99d3add 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -12,365 +12,35 @@
the specific language governing permissions and limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-platform</artifactId>
- <version>0.7.2</version>
- </parent>
- <artifactId>metron-elasticsearch</artifactId>
- <name>metron-elasticsearch</name>
- <url>https://metron.apache.org/</url>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <guava_version>${global_hbase_guava_version}</guava_version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>elasticsearch-shaded</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-common</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-storm-kafka</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-writer-common</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-hbase</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>flux-core</artifactId>
- <version>${global_flux_version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- <version>${global_storm_version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-over-slf4j</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-slf4j-impl</artifactId>
- <groupId>org.apache.logging.log4j</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- <version>4.4.9</version>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-hbase</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava_version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <version>${global_hbase_version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${global_hadoop_version}</version>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${global_hadoop_version}</version>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
- <version>${global_kafka_version}</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <!-- must be near the top of the dependencies to force retrieval of the correct netty version -->
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- <version>4.1.13.Final</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>${global_httpclient_version}</version>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch.plugin</groupId>
- <artifactId>transport-netty4-client</artifactId>
- <version>${global_elasticsearch_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- <version>${global_kafka_version}</version>
- <classifier>test</classifier>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-integration-test</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-core</artifactId>
- <version>1.3</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <version>${global_mockito_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-indexing</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-indexing</artifactId>
- <version>${project.parent.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-common</artifactId>
- <version>${project.parent.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <version>${global_jar_version}</version>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${global_shade_version}</version>
- <configuration>
- <createDependencyReducedPom>true</createDependencyReducedPom>
- </configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <shadedArtifactAttached>true</shadedArtifactAttached>
- <shadedClassifierName>uber</shadedClassifierName>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- <relocations>
- <relocation>
- <pattern>com.google.common</pattern>
- <shadedPattern>org.apache.metron.guava.${guava_version}</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.fasterxml.jackson</pattern>
- <shadedPattern>org.apache.metron.jackson</shadedPattern>
- </relocation>
- </relocations>
- <artifactSet>
- <excludes>
- <exclude>storm:storm-core:*</exclude>
- <exclude>storm:storm-lib:*</exclude>
- <exclude>org.slf4j.impl*</exclude>
- <exclude>org.slf4j:slf4j-log4j*</exclude>
- </excludes>
- </artifactSet>
- <transformers>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
- <resources>
- <resource>.yaml</resource>
- <resource>LICENSE.txt</resource>
- <resource>ASL2.0</resource>
- <resource>NOTICE.txt</resource>
- </resources>
- </transformer>
- <!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE -->
- <!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
- <addHeader>false</addHeader>
- <projectName>${project.name}</projectName>
- </transformer>-->
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass></mainClass>
- </transformer>
- <!-- ClassIndexTransformer needs to go LAST. For some reason it will clobber other transformers from operating when it is put first -->
- <transformer implementation="org.atteo.classindex.ClassIndexTransformer"/>
- </transformers>
- </configuration>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>org.atteo.classindex</groupId>
- <artifactId>classindex-transformer</artifactId>
- <version>${global_classindex_version}</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <descriptor>src/main/assembly/assembly.xml</descriptor>
- </configuration>
- <executions>
- <execution>
- <id>make-assembly</id> <!-- this is used for inheritance merges -->
- <phase>package</phase> <!-- bind to the packaging phase -->
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>metron-elasticsearch</artifactId>
+ <packaging>pom</packaging>
+ <name>metron-elasticsearch</name>
+ <parent>
+ <artifactId>metron-platform</artifactId>
+ <groupId>org.apache.metron</groupId>
+ <version>0.7.2</version>
+ </parent>
+ <description>Metron parsers and platform support</description>
+ <url>https://metron.apache.org/</url>
+ <scm>
+ <connection>scm:git:https://gitbox.apache.org/repos/asf/metron.git</connection>
+ <developerConnection>scm:git:https://gitbox.apache.org/repos/asf/metron.git</developerConnection>
+ <tag>HEAD</tag>
+ <url>https://gitbox.apache.org/repos/asf/metron.git</url>
+ </scm>
+ <licenses>
+ <license>
+ <name>The Apache Software License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ </license>
+ </licenses>
+ <modules>
+ <module>metron-elasticsearch-common</module>
+ <module>metron-elasticsearch-storm</module>
+ </modules>
+</project>
\ No newline at end of file
diff --git a/metron-platform/metron-indexing/README.md b/metron-platform/metron-indexing/README.md
index 990a5a1..872fce3 100644
--- a/metron-platform/metron-indexing/README.md
+++ b/metron-platform/metron-indexing/README.md
@@ -19,285 +19,15 @@ limitations under the License.
## Introduction
-The `indexing` topology is a topology dedicated to taking the data
-from the enrichment topology that have been enriched and storing the data in one or more supported indices
-* HDFS as rolled text files, one JSON blob per line
-* Elasticsearch
-* Solr
+The `indexing` module is a module dedicated to indexing data from
+enrichment (or possible directly from parsers) into various data stores
+including Elasticsearch, Solr and HDFS.
-By default, this topology writes out to both HDFS and one of
-Elasticsearch and Solr.
+## Deployment Options
-Indices are written in batch and the batch size and batch timeout are specified in the
-[Sensor Indexing Configuration](#sensor-indexing-configuration) via the `batchSize` and `batchTimeout` parameters.
-These configs are variable by sensor type.
+There is currently one option for running indexing in Metron, which is as a Storm topology.
-## Minimal Assumptions for Message Structure
+## Submodules
-At minimum, a message should have a `source.type` field.
-Without this field, the message tuple will be failed and not written
-with an appropriate error indicated in the Storm UI and logs.
-
-## Indexing Architecture
-
-
-
-The indexing topology is extremely simple. Data is ingested into kafka
-and sent to
-* An indexing bolt configured to write to either elasticsearch or Solr
-* An indexing bolt configured to write to HDFS under `/apps/metron/enrichment/indexed`
-
-By default, errors during indexing are sent back into the `indexing` kafka queue so that they can be indexed and archived.
-
-## Sensor Indexing Configuration
-
-
-The sensor specific configuration is intended to configure the
-indexing used for a given sensor type (e.g. `snort`).
-
-Just like the global config, the format is a JSON stored in zookeeper and on disk at `$METRON_HOME/config/zookeeper/indexing`. Within the sensor-specific configuration, you can configure the individual writers. The writers currently supported are:
-* `elasticsearch`
-* `hdfs`
-* `solr`
-
-Depending on how you start the indexing topology, it will have either Elasticsearch or Solr and HDFS writers running.
-
-| Property | Description | Default Value |
-|----------------------|---------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------|
-| `index` | The name of the index to write to. | Defaults to the name of the sensor. |
-| `batchSize` | The size of the batch that is written to the indices at once. | Defaults to `1`; no batching. |
-| `batchTimeout` | The timeout after which a batch will be flushed even if `batchSize` has not been met. | Defaults to a duration which is a fraction of the Storm parameter `topology.message.timeout.secs`, if left undefined or set to 0. Ignored if batchSize is `1`, since this disables batching.|
-| `enabled` | A boolean indicating whether the writer is enabled. | Defaults to `true` |
-| `fieldNameConverter` | Defines how field names are transformed before being written to the index. Only applicable to `elasticsearch`. | Defaults to `DEDOT`. Acceptable values are `DEDOT` that replaces all '.' with ':' or `NOOP` that does not change the field names . |
-| `setDocumentId` | A boolean indicating whether the writer should use the document id generated by Metron| Defaults to `false`. This setting only applies to Elasticsearch, the id used with Solr is configured in the Solr schemas.
-
-
-### Meta Alerts
-Alerts can be grouped, after appropriate searching, into a set of alerts called a meta alert. A meta alert is useful for maintaining the context of searching and grouping during further investigations. Standard searches can return meta alerts, but grouping and other aggregation or sorting requests will not, because there's not a clear way to aggregate in many cases if there are multiple alerts contained in the meta alert. All meta alerts will have the source type of metaalert, regardles [...]
-
-### Elasticsearch
-Metron comes with built-in templates for the default sensors for Elasticsearch. When adding a new sensor, it will be necessary to add a new template defining the output fields appropriately. In addition, there is a requirement for a field `alert` of type `nested` for Elasticsearch 2.x installs. This is detailed at [Using Metron with Elasticsearch 2.x](../metron-elasticsearch/README.md#using-metron-with-elasticsearch-2x).
-
-Metron is configured by default to let Elasticsearch [use auto-generated ids](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/tune-for-indexing-speed.html#_use_auto_generated_ids) for performance reasons. However, due to Storm's at least once processing guarantee, it is possible for duplicate messages to be indexed when messages are replayed for whatever reason. If this scenario is less desirable, the Metron generated id stored in the `guid` field of the message can be used [...]
-```
-{
- "elasticsearch": {
- "enabled": true,
- "index": "bro",
- "setDocumentId": true
- }
-}
-```
-This can also be set for all sensors in the [Global Configuration](../metron-common#global-configuration) by setting the `indexing.writer.elasticsearch.setDocumentId` setting to true. These settings are applied in this order of precedence (highest to lowest):
-1. Sensor indexing configuration
-2. Global configuration
-3. False by default
-
-### Solr
-
-Metron comes with built-in schemas for the default sensors for Solr. When adding a new sensor, it will be necessary to add a new schema defining the output fields appropriately. In addition, these fields are used internally by Metron and also required:
-
-* `<field name="guid" type="string" indexed="true" stored="true" required="true" multiValued="false" />`
-* `<field name="source.type" type="string" indexed="true" stored="true" />`
-* `<field name="timestamp" type="timestamp" indexed="true" stored="true" />`
-* `<field name="comments" type="string" indexed="true" stored="true" multiValued="true"/>`
-* `<field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/>`
-
-The unique key should be set to `guid` by including `<uniqueKey>guid</uniqueKey>` in the schema.
-
-It is strongly suggested the `fieldTypes` match those in the built-in schemas.
-
-### Indexing Configuration Examples
-For a given sensor, the following scenarios would be indicated by
-the following cases:
-#### Base Case
-```
-{
-}
-```
-or no file at all.
-
-* elasticsearch writer
- * enabled
- * batch size of 1
- * batch timeout system default
- * index name the same as the sensor
-* hdfs writer
- * enabled
- * batch size of 1
- * batch timeout system default
- * index name the same as the sensor
-
-If a writer config is unspecified, then a warning is indicated in the
-Storm console. e.g.:
-`WARNING: Default and (likely) unoptimized writer config used for hdfs writer and sensor squid`
-
-#### Fully specified
-```
-{
- "elasticsearch": {
- "index": "foo",
- "batchSize" : 100,
- "batchTimeout" : 0,
- "enabled" : true
- },
- "hdfs": {
- "index": "foo",
- "batchSize": 1,
- "batchTimeout" : 0,
- "enabled" : true
- }
-}
-```
-* elasticsearch writer
- * enabled
- * batch size of 100
- * batch timeout system default
- * index name of "foo"
-* hdfs writer
- * enabled
- * batch size of 1
- * batch timeout system default
- * index name of "foo"
-
-#### HDFS Writer turned off
-```
-{
- "elasticsearch": {
- "index": "foo",
- "enabled" : true
- },
- "hdfs": {
- "index": "foo",
- "batchSize": 100,
- "batchTimeout" : 0,
- "enabled" : false
- }
-}
-```
-* elasticsearch writer
- * enabled
- * batch size of 1
- * batch timeout system default
- * index name of "foo"
-* hdfs writer
- * disabled
-
-# Updates to Indexed Data
-
-There are clear usecases where we would want to incorporate the capability to update indexed data.
-Thus far, we have limited capabilities provided to support this use-case:
-* Updates to the random access index (e.g. Elasticsearch and Solr) should be supported
-* Updates to the cold storage index (e.g. HDFS) is not supported currently, however to support the batch
-use-case updated documents will be provided in a NoSQL write-ahead log (e.g. a HBase table) and an Java API
-will be provided to retrieve those updates scalably (i.e. a scan-free architecture).
-
-Put simply, the random access index will be always up-to-date, but the HDFS index will need to be
-joined to the NoSQL write-ahead log to get current updates.
-
-## The `IndexDao` Abstraction
-
-The indices mentioned above as part of Update should be pluggable by the developer so that
-new write-ahead logs or real-time indices can be supported by providing an implementation supporting
-the data access patterns.
-
-To support a new index, one would need to implement the `org.apache.metron.indexing.dao.IndexDao` abstraction
-and provide update and search capabilities. IndexDaos may be composed and updates will be performed
-in parallel. This enables a flexible strategy for specifying your backing store for updates at runtime.
-For instance, currently the REST API supports the update functionality and may be configured with a list of
-IndexDao implementations to use to support the updates.
-
-Updates with the IndexDao.update method replace the current object with the new object. For partial updates,
-use IndexDao.patch instead.
-
-### The `HBaseDao`
-
-Updates will be written to HBase. The key structure includes the GUID and sensor type and
-for each new version, a new column is created with value as the message.
-
-The HBase table and column family are configured via fields in the global configuration.
-
-#### `update.hbase.table`
-The HBase table to use for message updates.
-
-#### `update.hbase.cf`
-The HBase column family to use for message updates.
-
-### The `MetaAlertDao`
-
-The goal of meta alerts is to be able to group together a set of alerts while being able to transparently perform actions
-like searches, as if meta alerts were normal alerts. `org.apache.metron.indexing.dao.metaalert.MetaAlertDao` extends `IndexDao` and
-enables several features:
-* the ability to get all meta alerts associated with an alert
-* creation of a meta alert
-* adding alerts to a meta alert
-* removing alerts from a meta alert
-* changing a meta alert's status
-
-The implementation of this is to denormalize the relationship between alerts and meta alerts, and store alerts as a nested field within a meta alert.
-The use of nested fields is to avoid the limitations of parent-child relationships (one-to-many) and merely linking by IDs
-(which causes issues with pagination as a result of being unable to join indices). A list of containing meta alerts is stored
-on an alert for the purpose of keeping source alerts and alerts contained in meta alerts in sync.
-
-The search functionality of `IndexDao` is wrapped by the `MetaAlertDao` in order to provide both regular and meta alerts side-by-side with sorting.
-The updating capabilities are similarly wrapped, in order to ensure updates are carried through both the alerts and associated meta alerts.
-Both of these functions are handled under the hood.
-
-In addition, API endpoints have been added to expose the features listed above.
-The denormalization handles the case of going from meta alert to alert automatically.
-
-# Notes on Performance Tuning
-
-Default installed Metron is untuned for production deployment. By far
-and wide, the most likely piece to require TLC from a performance
-perspective is the indexing layer. An index that does not keep up will
-back up and you will see errors in the kafka bolt. There
-are a few knobs to tune to get the most out of your system.
-
-## Kafka Queue
-The `indexing` kafka queue is a collection point from the enrichment
-topology. As such, make sure that the number of partitions in
-the kafka topic is sufficient to handle the throughput that you expect.
-
-## Indexing Topology
-The `indexing` topology as started by the `$METRON_HOME/bin/start_elasticsearch_topology.sh`
-or `$METRON_HOME/bin/start_solr_topology.sh`
-script uses a default of one executor per bolt. In a real production system, this should
-be customized by modifying the flux file in
-`$METRON_HOME/flux/indexing/remote.yaml`.
-* Add a `parallelism` field to the bolts to give Storm a parallelism
- hint for the various components. Give bolts which appear to be bottlenecks (e.g. the indexing bolt) a larger hint.
-* Add a `parallelism` field to the kafka spout which matches the number of partitions for the enrichment kafka queue.
-* Adjust the number of workers for the topology by adjusting the
- `topology.workers` field for the topology.
-
-Finally, if workers and executors are new to you or you don't know where
-to modify the flux file, the following might be of use to you:
-* [Understanding the Parallelism of a Storm Topology](http://www.michael-noll.com/blog/2012/10/16/understanding-the-parallelism-of-a-storm-topology/)
-* [Flux Docs](http://storm.apache.org/releases/current/flux.html)
-
-### Rest endpoints
-There are rest endpoints available to perform operations like start, stop, activate, deactivate on the `indexing` topologies.
-
-
-| |
-| ---------- |
-| [ `GET /api/v1/storm/indexing/batch`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingbatch)|
-| [ `GET /api/v1/storm/indexing/batch/activate`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingbatchactivate)|
-| [ `GET /api/v1/storm/indexing/batch/deactivate`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingbatchdeactivate)|
-| [ `GET /api/v1/storm/indexing/batch/start`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingbatchstart)|
-| [ `GET /api/v1/storm/indexing/batch/stop`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingbatchstop)|
-| [ `GET /api/v1/storm/indexing/randomaccess`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingrandomaccess)|
-| [ `GET /api/v1/storm/indexing/randomaccess/activate`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingrandomaccessactivate)|
-| [ `GET /api/v1/storm/indexing/randomaccess/deactivate`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingrandomaccessdeactivate)|
-| [ `GET /api/v1/storm/indexing/randomaccess/start`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingrandomaccessstart)|
-| [ `GET /api/v1/storm/indexing/randomaccess/stop`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingrandomaccessstop)|
-
-
-## Zeppelin Notebooks
-Zeppelin notebooks can be added to `/src/main/config/zeppelin/` (and subdirectories can be created for organization). The placed files must be .json files and be named appropriately.
-These files must be added to the metron.spec file and the RPMs rebuilt to be available to be loaded into Ambari.
-
-The notebook files will be found on the server in `$METRON_HOME/config/zeppelin`
-
-The Ambari Management Pack has a custom action to load these templates, ZEPPELIN_DASHBOARD_INSTALL, that will import them into Zeppelin.
+* metron-indexing-common - this module contains common indexing code including IndexDao related-classes.
+* metron-indexing-storm - this module is home to Storm-specific code such as Flux files and Storm indexing integration tests.
\ No newline at end of file
diff --git a/metron-platform/metron-indexing/README.md b/metron-platform/metron-indexing/metron-indexing-common/README.md
similarity index 76%
copy from metron-platform/metron-indexing/README.md
copy to metron-platform/metron-indexing/metron-indexing-common/README.md
index 990a5a1..b572488 100644
--- a/metron-platform/metron-indexing/README.md
+++ b/metron-platform/metron-indexing/metron-indexing-common/README.md
@@ -19,14 +19,7 @@ limitations under the License.
## Introduction
-The `indexing` topology is a topology dedicated to taking the data
-from the enrichment topology that have been enriched and storing the data in one or more supported indices
-* HDFS as rolled text files, one JSON blob per line
-* Elasticsearch
-* Solr
-
-By default, this topology writes out to both HDFS and one of
-Elasticsearch and Solr.
+This modules holds shared code for indexing activities.
Indices are written in batch and the batch size and batch timeout are specified in the
[Sensor Indexing Configuration](#sensor-indexing-configuration) via the `batchSize` and `batchTimeout` parameters.
@@ -34,20 +27,8 @@ These configs are variable by sensor type.
## Minimal Assumptions for Message Structure
-At minimum, a message should have a `source.type` field.
-Without this field, the message tuple will be failed and not written
-with an appropriate error indicated in the Storm UI and logs.
-
-## Indexing Architecture
-
-
-
-The indexing topology is extremely simple. Data is ingested into kafka
-and sent to
-* An indexing bolt configured to write to either elasticsearch or Solr
-* An indexing bolt configured to write to HDFS under `/apps/metron/enrichment/indexed`
-
-By default, errors during indexing are sent back into the `indexing` kafka queue so that they can be indexed and archived.
+At minimum, a message should have a `source.type` field. This field is used to lookup the [Sensor Indexing Configuration](#sensor-indexing-configuration)
+for that message.
## Sensor Indexing Configuration
@@ -69,7 +50,6 @@ Depending on how you start the indexing topology, it will have either Elasticsea
| `batchTimeout` | The timeout after which a batch will be flushed even if `batchSize` has not been met. | Defaults to a duration which is a fraction of the Storm parameter `topology.message.timeout.secs`, if left undefined or set to 0. Ignored if batchSize is `1`, since this disables batching.|
| `enabled` | A boolean indicating whether the writer is enabled. | Defaults to `true` |
| `fieldNameConverter` | Defines how field names are transformed before being written to the index. Only applicable to `elasticsearch`. | Defaults to `DEDOT`. Acceptable values are `DEDOT` that replaces all '.' with ':' or `NOOP` that does not change the field names . |
-| `setDocumentId` | A boolean indicating whether the writer should use the document id generated by Metron| Defaults to `false`. This setting only applies to Elasticsearch, the id used with Solr is configured in the Solr schemas.
### Meta Alerts
@@ -251,49 +231,14 @@ The denormalization handles the case of going from meta alert to alert automatic
Default installed Metron is untuned for production deployment. By far
and wide, the most likely piece to require TLC from a performance
perspective is the indexing layer. An index that does not keep up will
-back up and you will see errors in the kafka bolt. There
-are a few knobs to tune to get the most out of your system.
+back up and you will see errors in the streaming platform (Storm logs for example).
+There are a few knobs to tune to get the most out of your system.
## Kafka Queue
The `indexing` kafka queue is a collection point from the enrichment
topology. As such, make sure that the number of partitions in
the kafka topic is sufficient to handle the throughput that you expect.
-## Indexing Topology
-The `indexing` topology as started by the `$METRON_HOME/bin/start_elasticsearch_topology.sh`
-or `$METRON_HOME/bin/start_solr_topology.sh`
-script uses a default of one executor per bolt. In a real production system, this should
-be customized by modifying the flux file in
-`$METRON_HOME/flux/indexing/remote.yaml`.
-* Add a `parallelism` field to the bolts to give Storm a parallelism
- hint for the various components. Give bolts which appear to be bottlenecks (e.g. the indexing bolt) a larger hint.
-* Add a `parallelism` field to the kafka spout which matches the number of partitions for the enrichment kafka queue.
-* Adjust the number of workers for the topology by adjusting the
- `topology.workers` field for the topology.
-
-Finally, if workers and executors are new to you or you don't know where
-to modify the flux file, the following might be of use to you:
-* [Understanding the Parallelism of a Storm Topology](http://www.michael-noll.com/blog/2012/10/16/understanding-the-parallelism-of-a-storm-topology/)
-* [Flux Docs](http://storm.apache.org/releases/current/flux.html)
-
-### Rest endpoints
-There are rest endpoints available to perform operations like start, stop, activate, deactivate on the `indexing` topologies.
-
-
-| |
-| ---------- |
-| [ `GET /api/v1/storm/indexing/batch`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingbatch)|
-| [ `GET /api/v1/storm/indexing/batch/activate`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingbatchactivate)|
-| [ `GET /api/v1/storm/indexing/batch/deactivate`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingbatchdeactivate)|
-| [ `GET /api/v1/storm/indexing/batch/start`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingbatchstart)|
-| [ `GET /api/v1/storm/indexing/batch/stop`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingbatchstop)|
-| [ `GET /api/v1/storm/indexing/randomaccess`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingrandomaccess)|
-| [ `GET /api/v1/storm/indexing/randomaccess/activate`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingrandomaccessactivate)|
-| [ `GET /api/v1/storm/indexing/randomaccess/deactivate`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingrandomaccessdeactivate)|
-| [ `GET /api/v1/storm/indexing/randomaccess/start`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingrandomaccessstart)|
-| [ `GET /api/v1/storm/indexing/randomaccess/stop`](../../metron-interface/metron-rest/README.md#get-apiv1stormindexingrandomaccessstop)|
-
-
## Zeppelin Notebooks
Zeppelin notebooks can be added to `/src/main/config/zeppelin/` (and subdirectories can be created for organization). The placed files must be .json files and be named appropriately.
These files must be added to the metron.spec file and the RPMs rebuilt to be available to be loaded into Ambari.
diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-indexing/metron-indexing-common/pom.xml
similarity index 95%
copy from metron-platform/metron-indexing/pom.xml
copy to metron-platform/metron-indexing/metron-indexing-common/pom.xml
index f7067b1..74c0389 100644
--- a/metron-platform/metron-indexing/pom.xml
+++ b/metron-platform/metron-indexing/metron-indexing-common/pom.xml
@@ -17,11 +17,11 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-platform</artifactId>
+ <artifactId>metron-indexing</artifactId>
<version>0.7.2</version>
</parent>
- <artifactId>metron-indexing</artifactId>
- <name>metron-indexing</name>
+ <artifactId>metron-indexing-common</artifactId>
+ <name>metron-indexing-common</name>
<url>https://metron.apache.org/</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -41,12 +41,7 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-writer-storm</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-storm-kafka</artifactId>
+ <artifactId>metron-writer-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
@@ -131,11 +126,6 @@
<version>${global_mockito_version}</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>commons-validator</groupId>
- <artifactId>commons-validator</artifactId>
- <version>1.4.0</version>
- </dependency>
<!-- Test -->
diff --git a/metron-platform/metron-elasticsearch/src/main/assembly/assembly.xml b/metron-platform/metron-indexing/metron-indexing-common/src/main/assembly/assembly.xml
similarity index 67%
rename from metron-platform/metron-elasticsearch/src/main/assembly/assembly.xml
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/assembly/assembly.xml
index f8ce2da..aec6d04 100644
--- a/metron-platform/metron-elasticsearch/src/main/assembly/assembly.xml
+++ b/metron-platform/metron-indexing/metron-indexing-common/src/main/assembly/assembly.xml
@@ -29,26 +29,7 @@
</excludes>
<fileMode>0644</fileMode>
<lineEnding>unix</lineEnding>
- </fileSet>
- <fileSet>
- <directory>${project.basedir}/src/main/scripts</directory>
- <outputDirectory>bin</outputDirectory>
- <useDefaultExcludes>true</useDefaultExcludes>
- <excludes>
- <exclude>**/*.formatted</exclude>
- <exclude>**/*.filtered</exclude>
- </excludes>
- <fileMode>0755</fileMode>
- <lineEnding>unix</lineEnding>
<filtered>true</filtered>
</fileSet>
- <fileSet>
- <directory>${project.basedir}/target</directory>
- <includes>
- <include>${project.artifactId}-${project.version}-uber.jar</include>
- </includes>
- <outputDirectory>lib</outputDirectory>
- <useDefaultExcludes>true</useDefaultExcludes>
- </fileSet>
</fileSets>
</assembly>
diff --git a/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-connection-report.json b/metron-platform/metron-indexing/metron-indexing-common/src/main/config/zeppelin/metron/metron-connection-report.json
similarity index 100%
rename from metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-connection-report.json
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/config/zeppelin/metron/metron-connection-report.json
diff --git a/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-connection-volume-report.json b/metron-platform/metron-indexing/metron-indexing-common/src/main/config/zeppelin/metron/metron-connection-volume-report.json
similarity index 100%
rename from metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-connection-volume-report.json
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/config/zeppelin/metron/metron-connection-volume-report.json
diff --git a/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-ip-report.json b/metron-platform/metron-indexing/metron-indexing-common/src/main/config/zeppelin/metron/metron-ip-report.json
similarity index 100%
rename from metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-ip-report.json
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/config/zeppelin/metron/metron-ip-report.json
diff --git a/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-yaf-telemetry.json b/metron-platform/metron-indexing/metron-indexing-common/src/main/config/zeppelin/metron/metron-yaf-telemetry.json
similarity index 100%
rename from metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-yaf-telemetry.json
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/config/zeppelin/metron/metron-yaf-telemetry.json
diff --git a/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/asa.json b/metron-platform/metron-indexing/metron-indexing-common/src/main/config/zookeeper/indexing/asa.json
similarity index 100%
rename from metron-platform/metron-indexing/src/main/config/zookeeper/indexing/asa.json
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/config/zookeeper/indexing/asa.json
diff --git a/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/bro.json b/metron-platform/metron-indexing/metron-indexing-common/src/main/config/zookeeper/indexing/bro.json
similarity index 100%
rename from metron-platform/metron-indexing/src/main/config/zookeeper/indexing/bro.json
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/config/zookeeper/indexing/bro.json
diff --git a/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/error.json b/metron-platform/metron-indexing/metron-indexing-common/src/main/config/zookeeper/indexing/error.json
similarity index 100%
rename from metron-platform/metron-indexing/src/main/config/zookeeper/indexing/error.json
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/config/zookeeper/indexing/error.json
diff --git a/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/snort.json b/metron-platform/metron-indexing/metron-indexing-common/src/main/config/zookeeper/indexing/snort.json
similarity index 100%
rename from metron-platform/metron-indexing/src/main/config/zookeeper/indexing/snort.json
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/config/zookeeper/indexing/snort.json
diff --git a/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/websphere.json b/metron-platform/metron-indexing/metron-indexing-common/src/main/config/zookeeper/indexing/websphere.json
similarity index 100%
rename from metron-platform/metron-indexing/src/main/config/zookeeper/indexing/websphere.json
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/config/zookeeper/indexing/websphere.json
diff --git a/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/yaf.json b/metron-platform/metron-indexing/metron-indexing-common/src/main/config/zookeeper/indexing/yaf.json
similarity index 100%
rename from metron-platform/metron-indexing/src/main/config/zookeeper/indexing/yaf.json
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/config/zookeeper/indexing/yaf.json
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/AccessConfig.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/AccessConfig.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/AccessConfig.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/AccessConfig.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/ColumnMetadataDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/ColumnMetadataDao.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/ColumnMetadataDao.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/ColumnMetadataDao.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDaoFactory.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/IndexDaoFactory.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDaoFactory.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/IndexDaoFactory.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexUpdateCallback.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/IndexUpdateCallback.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexUpdateCallback.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/IndexUpdateCallback.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/RetrieveLatestDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/RetrieveLatestDao.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/RetrieveLatestDao.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/RetrieveLatestDao.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/DeferredMetaAlertIndexDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/DeferredMetaAlertIndexDao.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/DeferredMetaAlertIndexDao.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/DeferredMetaAlertIndexDao.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertAddRemoveRequest.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertAddRemoveRequest.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertAddRemoveRequest.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertAddRemoveRequest.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConfig.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConfig.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConfig.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConfig.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConstants.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConstants.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConstants.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConstants.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateRequest.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateRequest.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateRequest.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateRequest.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertDao.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertDao.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertDao.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertRetrieveLatestDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertRetrieveLatestDao.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertRetrieveLatestDao.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertRetrieveLatestDao.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertSearchDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertSearchDao.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertSearchDao.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertSearchDao.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertStatus.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertStatus.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertStatus.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertStatus.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertUpdateDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertUpdateDao.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertUpdateDao.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertUpdateDao.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/AlertComment.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/AlertComment.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/AlertComment.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/AlertComment.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GetRequest.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/GetRequest.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GetRequest.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/GetRequest.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/Group.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/Group.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/Group.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/Group.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrder.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/GroupOrder.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrder.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/GroupOrder.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrderType.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/GroupOrderType.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrderType.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/GroupOrderType.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupRequest.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/GroupRequest.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupRequest.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/GroupRequest.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResponse.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/GroupResponse.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResponse.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/GroupResponse.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResult.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/GroupResult.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResult.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/GroupResult.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/InvalidCreateException.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/InvalidCreateException.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/InvalidCreateException.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/InvalidCreateException.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/InvalidSearchException.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/InvalidSearchException.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/InvalidSearchException.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/InvalidSearchException.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/SearchDao.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchDao.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/SearchDao.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResponse.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/SearchResponse.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResponse.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/SearchResponse.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortField.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/SortField.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortField.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/SortField.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortOrder.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/SortOrder.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortOrder.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/search/SortOrder.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/CommentAddRemoveRequest.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/CommentAddRemoveRequest.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/CommentAddRemoveRequest.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/CommentAddRemoveRequest.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/Document.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/Document.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/OriginalNotFoundException.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/OriginalNotFoundException.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/OriginalNotFoundException.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/OriginalNotFoundException.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchRequest.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchRequest.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchRequest.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchRequest.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/util/IndexingCacheUtil.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/util/IndexingCacheUtil.java
similarity index 100%
rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/util/IndexingCacheUtil.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/util/IndexingCacheUtil.java
diff --git a/metron-platform/metron-indexing/src/main/resources/META-INF/LICENSE b/metron-platform/metron-indexing/metron-indexing-common/src/main/resources/META-INF/LICENSE
similarity index 100%
rename from metron-platform/metron-indexing/src/main/resources/META-INF/LICENSE
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/resources/META-INF/LICENSE
diff --git a/metron-platform/metron-indexing/src/main/resources/META-INF/NOTICE b/metron-platform/metron-indexing/metron-indexing-common/src/main/resources/META-INF/NOTICE
similarity index 100%
rename from metron-platform/metron-indexing/src/main/resources/META-INF/NOTICE
rename to metron-platform/metron-indexing/metron-indexing-common/src/main/resources/META-INF/NOTICE
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/InMemoryMetaAlertRetrieveLatestDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/InMemoryMetaAlertRetrieveLatestDao.java
similarity index 100%
rename from metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/InMemoryMetaAlertRetrieveLatestDao.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/InMemoryMetaAlertRetrieveLatestDao.java
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/HBaseDaoTest.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/HBaseDaoTest.java
similarity index 100%
rename from metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/HBaseDaoTest.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/HBaseDaoTest.java
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
similarity index 100%
rename from metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
similarity index 100%
rename from metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertUpdateDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertUpdateDao.java
similarity index 100%
rename from metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertUpdateDao.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertUpdateDao.java
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/MultiIndexDaoTest.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/MultiIndexDaoTest.java
similarity index 100%
rename from metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/MultiIndexDaoTest.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/MultiIndexDaoTest.java
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
similarity index 100%
rename from metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateDaoTest.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/UpdateDaoTest.java
similarity index 100%
rename from metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateDaoTest.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/UpdateDaoTest.java
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
similarity index 100%
rename from metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
similarity index 100%
rename from metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaScoresTest.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaScoresTest.java
similarity index 100%
rename from metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaScoresTest.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaScoresTest.java
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
similarity index 100%
rename from metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java
similarity index 100%
rename from metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/util/IndexingCacheUtilTest.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/util/IndexingCacheUtilTest.java
similarity index 100%
rename from metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/util/IndexingCacheUtilTest.java
rename to metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/util/IndexingCacheUtilTest.java
diff --git a/metron-platform/metron-indexing/src/test/resources/log4j.properties b/metron-platform/metron-indexing/metron-indexing-common/src/test/resources/log4j.properties
similarity index 100%
rename from metron-platform/metron-indexing/src/test/resources/log4j.properties
rename to metron-platform/metron-indexing/metron-indexing-common/src/test/resources/log4j.properties
diff --git a/metron-platform/metron-indexing/metron-indexing-storm/README.md b/metron-platform/metron-indexing/metron-indexing-storm/README.md
new file mode 100644
index 0000000..94e7a3b
--- /dev/null
+++ b/metron-platform/metron-indexing/metron-indexing-storm/README.md
@@ -0,0 +1,79 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# Indexing
+
+## Introduction
+
+The `indexing` topology is a topology dedicated to taking the data
+from the enrichment topology that have been enriched and storing the data in one or more supported indices
+* HDFS as rolled text files, one JSON blob per line
+* Elasticsearch
+* Solr
+
+By default, this topology writes out to both HDFS and one of
+Elasticsearch and Solr.
+
+## Minimal Assumptions for Message Structure
+
+If a message is missing the `source.type` field, the message tuple will be failed and not written
+with an appropriate error indicated in the Storm UI and logs.
+
+## Indexing Architecture
+
+
+
+The indexing topology is extremely simple. Data is ingested into kafka
+and sent to
+* An indexing bolt configured to write to either elasticsearch or Solr
+* An indexing bolt configured to write to HDFS under `/apps/metron/enrichment/indexed`
+
+By default, errors during indexing are sent back into the `indexing` kafka queue so that they can be indexed and archived.
+
+## Indexing Topology
+The `indexing` topology as started by the `$METRON_HOME/bin/start_elasticsearch_topology.sh`
+or `$METRON_HOME/bin/start_solr_topology.sh`
+script uses a default of one executor per bolt. In a real production system, this should
+be customized by modifying the flux file in
+`$METRON_HOME/flux/indexing/remote.yaml`.
+* Add a `parallelism` field to the bolts to give Storm a parallelism
+ hint for the various components. Give bolts which appear to be bottlenecks (e.g. the indexing bolt) a larger hint.
+* Add a `parallelism` field to the kafka spout which matches the number of partitions for the enrichment kafka queue.
+* Adjust the number of workers for the topology by adjusting the
+ `topology.workers` field for the topology.
+
+Finally, if workers and executors are new to you or you don't know where
+to modify the flux file, the following might be of use to you:
+* [Understanding the Parallelism of a Storm Topology](http://www.michael-noll.com/blog/2012/10/16/understanding-the-parallelism-of-a-storm-topology/)
+* [Flux Docs](http://storm.apache.org/releases/current/flux.html)
+
+### Rest endpoints
+There are rest endpoints available to perform operations like start, stop, activate, deactivate on the `indexing` topologies.
+
+
+| |
+| ---------- |
+| [ `GET /api/v1/storm/indexing/batch`](../../../metron-interface/metron-rest/README.md#get-apiv1stormindexingbatch)|
+| [ `GET /api/v1/storm/indexing/batch/activate`](../../../metron-interface/metron-rest/README.md#get-apiv1stormindexingbatchactivate)|
+| [ `GET /api/v1/storm/indexing/batch/deactivate`](../../../metron-interface/metron-rest/README.md#get-apiv1stormindexingbatchdeactivate)|
+| [ `GET /api/v1/storm/indexing/batch/start`](../../../metron-interface/metron-rest/README.md#get-apiv1stormindexingbatchstart)|
+| [ `GET /api/v1/storm/indexing/batch/stop`](../../../metron-interface/metron-rest/README.md#get-apiv1stormindexingbatchstop)|
+| [ `GET /api/v1/storm/indexing/randomaccess`](../../../metron-interface/metron-rest/README.md#get-apiv1stormindexingrandomaccess)|
+| [ `GET /api/v1/storm/indexing/randomaccess/activate`](../../../metron-interface/metron-rest/README.md#get-apiv1stormindexingrandomaccessactivate)|
+| [ `GET /api/v1/storm/indexing/randomaccess/deactivate`](../../../metron-interface/metron-rest/README.md#get-apiv1stormindexingrandomaccessdeactivate)|
+| [ `GET /api/v1/storm/indexing/randomaccess/start`](../../../metron-interface/metron-rest/README.md#get-apiv1stormindexingrandomaccessstart)|
+| [ `GET /api/v1/storm/indexing/randomaccess/stop`](../../../metron-interface/metron-rest/README.md#get-apiv1stormindexingrandomaccessstop)|
diff --git a/metron-platform/metron-indexing/indexing_arch.png b/metron-platform/metron-indexing/metron-indexing-storm/indexing_arch.png
similarity index 100%
rename from metron-platform/metron-indexing/indexing_arch.png
rename to metron-platform/metron-indexing/metron-indexing-storm/indexing_arch.png
diff --git a/metron-platform/metron-indexing/metron-indexing-storm/pom.xml b/metron-platform/metron-indexing/metron-indexing-storm/pom.xml
new file mode 100644
index 0000000..581e819
--- /dev/null
+++ b/metron-platform/metron-indexing/metron-indexing-storm/pom.xml
@@ -0,0 +1,131 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-indexing</artifactId>
+ <version>0.7.2</version>
+ </parent>
+ <artifactId>metron-indexing-storm</artifactId>
+ <name>metron-indexing-storm</name>
+ <description>Components common to Storm</description>
+ <url>https://metron.apache.org/</url>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ <commons.config.version>1.10</commons.config.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-indexing-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-writer-storm</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-storm-kafka</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-integration-test</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-common</artifactId>
+ <version>${project.parent.version}</version>
+ <type>test-jar</type>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${global_kafka_version}</version>
+ <classifier>test</classifier>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>${global_jar_version}</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id> <!-- this is used for inheritance merges -->
+ <phase>package</phase> <!-- bind to the packaging phase -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/metron-platform/metron-indexing/src/main/assembly/assembly.xml b/metron-platform/metron-indexing/metron-indexing-storm/src/main/assembly/assembly.xml
similarity index 100%
rename from metron-platform/metron-indexing/src/main/assembly/assembly.xml
rename to metron-platform/metron-indexing/metron-indexing-storm/src/main/assembly/assembly.xml
diff --git a/metron-platform/metron-indexing/src/main/config/hdfs.properties.j2 b/metron-platform/metron-indexing/metron-indexing-storm/src/main/config/hdfs.properties.j2
similarity index 100%
rename from metron-platform/metron-indexing/src/main/config/hdfs.properties.j2
rename to metron-platform/metron-indexing/metron-indexing-storm/src/main/config/hdfs.properties.j2
diff --git a/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml b/metron-platform/metron-indexing/metron-indexing-storm/src/main/flux/indexing/batch/remote.yaml
similarity index 100%
rename from metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml
rename to metron-platform/metron-indexing/metron-indexing-storm/src/main/flux/indexing/batch/remote.yaml
diff --git a/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml b/metron-platform/metron-indexing/metron-indexing-storm/src/main/flux/indexing/random_access/remote.yaml
similarity index 100%
rename from metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
rename to metron-platform/metron-indexing/metron-indexing-storm/src/main/flux/indexing/random_access/remote.yaml
diff --git a/metron-platform/metron-indexing/src/main/scripts/start_hdfs_topology.sh b/metron-platform/metron-indexing/metron-indexing-storm/src/main/scripts/start_hdfs_topology.sh
similarity index 94%
rename from metron-platform/metron-indexing/src/main/scripts/start_hdfs_topology.sh
rename to metron-platform/metron-indexing/metron-indexing-storm/src/main/scripts/start_hdfs_topology.sh
index 544d944..77007be 100755
--- a/metron-platform/metron-indexing/src/main/scripts/start_hdfs_topology.sh
+++ b/metron-platform/metron-indexing/metron-indexing-storm/src/main/scripts/start_hdfs_topology.sh
@@ -18,5 +18,5 @@
#
METRON_VERSION=${project.version}
METRON_HOME=/usr/metron/$METRON_VERSION
-TOPOLOGY_JAR=metron-elasticsearch-$METRON_VERSION-uber.jar
+TOPOLOGY_JAR=metron-elasticsearch-storm-$METRON_VERSION-uber.jar
storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/batch/remote.yaml --filter $METRON_HOME/config/hdfs.properties
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java b/metron-platform/metron-indexing/metron-indexing-storm/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java
similarity index 95%
rename from metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java
rename to metron-platform/metron-indexing/metron-indexing-storm/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java
index 2aff2ef..26e65ba 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java
+++ b/metron-platform/metron-indexing/metron-indexing-storm/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java
@@ -19,6 +19,7 @@
package org.apache.metron.indexing.integration;
import com.google.common.collect.Iterables;
+import org.apache.metron.TestConstants;
import org.apache.metron.common.field.FieldNameConverter;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.integration.*;
@@ -102,7 +103,7 @@ public class HDFSIndexingIntegrationTest extends IndexingIntegrationTest {
throw new IllegalStateException("Unable to retrieve indexed documents.", e);
}
if (docs.size() < inputMessages.size()) {
- errors = kafkaComponent.readMessages(ERROR_TOPIC);
+ errors = kafkaComponent.readMessages(IndexingIntegrationTest.ERROR_TOPIC);
if(errors.size() > 0 && errors.size() + docs.size() == inputMessages.size()){
return ReadinessState.READY;
}
@@ -151,11 +152,11 @@ public class HDFSIndexingIntegrationTest extends IndexingIntegrationTest {
@Override
public String getTemplatePath() {
- return "../metron-indexing/src/main/config/hdfs.properties.j2";
+ return "../metron-indexing-storm/src/main/config/hdfs.properties.j2";
}
@Override
public String getFluxPath() {
- return "../metron-indexing/src/main/flux/indexing/batch/remote.yaml";
+ return "../metron-indexing-storm/src/main/flux/indexing/batch/remote.yaml";
}
}
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java b/metron-platform/metron-indexing/metron-indexing-storm/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
similarity index 96%
rename from metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
rename to metron-platform/metron-indexing/metron-indexing-storm/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
index 6fa6094..b6f3a70 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
+++ b/metron-platform/metron-indexing/metron-indexing-storm/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
@@ -50,7 +50,8 @@ import org.junit.Test;
public abstract class IndexingIntegrationTest extends BaseIntegrationTest {
protected static final String ERROR_TOPIC = "indexing_error";
- protected String sampleParsedPath = TestConstants.SAMPLE_DATA_PARSED_PATH + "TestExampleParsed";
+ protected String sampleParsedPath = "../" + TestConstants.SAMPLE_DATA_PARSED_PATH + "TestExampleParsed";
+ protected String sampleConfigPath = "../" + TestConstants.SAMPLE_CONFIG_PATH;
protected String testSensorType = "test";
protected final int NUM_RETRIES = 100;
protected final long TOTAL_TIME_MS = 150000L;
@@ -88,9 +89,9 @@ public abstract class IndexingIntegrationTest extends BaseIntegrationTest {
final AtomicBoolean isLoaded = new AtomicBoolean(false);
ConfigUploadComponent configUploadComponent = new ConfigUploadComponent()
.withTopologyProperties(topologyProperties)
- .withGlobalConfigsPath(TestConstants.SAMPLE_CONFIG_PATH)
- .withEnrichmentConfigsPath(TestConstants.SAMPLE_CONFIG_PATH)
- .withIndexingConfigsPath(TestConstants.SAMPLE_CONFIG_PATH)
+ .withGlobalConfigsPath(sampleConfigPath)
+ .withEnrichmentConfigsPath(sampleConfigPath)
+ .withIndexingConfigsPath(sampleConfigPath)
.withPostStartCallback(component -> {
try {
waitForIndex(component.getTopologyProperties().getProperty(ZKServerComponent.ZOOKEEPER_PROPERTY));
diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-indexing/pom.xml
index f7067b1..9da6209 100644
--- a/metron-platform/metron-indexing/pom.xml
+++ b/metron-platform/metron-indexing/pom.xml
@@ -12,277 +12,35 @@
the specific language governing permissions and limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
+ <artifactId>metron-indexing</artifactId>
+ <packaging>pom</packaging>
+ <name>metron-indexing</name>
<parent>
- <groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
+ <groupId>org.apache.metron</groupId>
<version>0.7.2</version>
</parent>
- <artifactId>metron-indexing</artifactId>
- <name>metron-indexing</name>
+ <description>Metron parsers and platform support</description>
<url>https://metron.apache.org/</url>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <slf4j.version>1.7.7</slf4j.version>
- <storm.hdfs.version>0.1.2</storm.hdfs.version>
- <guava_version>${global_hbase_guava_version}</guava_version>
- </properties>
- <dependencies>
-
- <!-- Metron -->
-
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-common</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-writer-storm</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-storm-kafka</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-hbase</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
-
- <!-- Other -->
-
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <version>${global_log4j_core_version}</version>
- <scope>runtime</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <version>${global_hbase_version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${global_hadoop_version}</version>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j</artifactId>
- <groupId>log4j</groupId>
- </exclusion>
- </exclusions>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava_version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${global_hadoop_version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>log4j</artifactId>
- <groupId>log4j</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <version>${global_mockito_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>commons-validator</groupId>
- <artifactId>commons-validator</artifactId>
- <version>1.4.0</version>
- </dependency>
-
- <!-- Test -->
-
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-test-utilities</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-integration-test</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- <version>${global_kafka_version}</version>
- <classifier>test</classifier>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-hbase</artifactId>
- <version>${project.parent.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-core</artifactId>
- <version>1.3</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-common</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <version>${global_jar_version}</version>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${global_shade_version}</version>
- <configuration>
- <createDependencyReducedPom>true</createDependencyReducedPom>
- <artifactSet>
- <excludes>
- <exclude>*slf4j*</exclude>
- </excludes>
- </artifactSet>
- </configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <shadedArtifactAttached>true</shadedArtifactAttached>
- <shadedClassifierName>uber</shadedClassifierName>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- <relocations>
- <relocation>
- <pattern>com.google.common</pattern>
- <shadedPattern>org.apache.metron.guava.${guava_version}</shadedPattern>
- </relocation>
- </relocations>
- <transformers>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
- <resources>
- <resource>.yaml</resource>
- <resource>LICENSE.txt</resource>
- <resource>ASL2.0</resource>
- <resource>NOTICE.txt</resource>
- </resources>
- </transformer>
- <!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE -->
- <!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
- <addHeader>false</addHeader>
- <projectName>${project.name}</projectName>
- </transformer-->
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass></mainClass>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <descriptor>src/main/assembly/assembly.xml</descriptor>
- </configuration>
- <executions>
- <execution>
- <id>make-assembly</id> <!-- this is used for inheritance merges -->
- <phase>package</phase> <!-- bind to the packaging phase -->
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
+ <scm>
+ <connection>scm:git:https://gitbox.apache.org/repos/asf/metron.git</connection>
+ <developerConnection>scm:git:https://gitbox.apache.org/repos/asf/metron.git</developerConnection>
+ <tag>HEAD</tag>
+ <url>https://gitbox.apache.org/repos/asf/metron.git</url>
+ </scm>
+ <licenses>
+ <license>
+ <name>The Apache Software License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ </license>
+ </licenses>
+ <modules>
+ <module>metron-indexing-common</module>
+ <module>metron-indexing-storm</module>
+ </modules>
</project>
diff --git a/metron-platform/metron-solr/README.md b/metron-platform/metron-solr/README.md
index cba4dd5..26d84ea 100644
--- a/metron-platform/metron-solr/README.md
+++ b/metron-platform/metron-solr/README.md
@@ -15,156 +15,17 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
-# Solr in Metron
-
-## Table of Contents
-
-* [Introduction](#introduction)
-* [Configuration](#configuration)
-* [Installing](#installing)
-* [Schemas](#schemas)
-* [Collections](#collections)
+# Solr
## Introduction
-Metron ships with Solr 6.6.2 support. Solr Cloud can be used as the real-time portion of the datastore resulting from [metron-indexing](../metron-indexing/README.md).
-
-## Configuration
-
-### The Indexing Topology
-
-Solr is a viable option for the `random access topology` and, similar to the Elasticsearch Writer, can be configured
-via the global config. The following settings are possible as part of the global config:
-* `solr.zookeeper`
- * The zookeeper quorum associated with the SolrCloud instance. This is a required field with no default.
-* `solr.commitPerBatch`
- * This is a boolean which defines whether the writer commits every batch. The default is `true`.
- * _WARNING_: If you set this to `false`, then commits will happen based on the SolrClient's internal mechanism and
- worker failure *may* result data being acknowledged in storm but not written in Solr.
-* `solr.commit.soft`
- * This is a boolean which defines whether the writer makes a soft commit or a durable commit. See [here](https://lucene.apache.org/solr/guide/6_6/near-real-time-searching.html#NearRealTimeSearching-AutoCommits) The default is `false`.
- * _WARNING_: If you set this to `true`, then commits will happen based on the SolrClient's internal mechanism and
- worker failure *may* result data being acknowledged in storm but not written in Solr.
-* `solr.commit.waitSearcher`
- * This is a boolean which defines whether the writer blocks the commit until the data is available to search. See [here](https://lucene.apache.org/solr/guide/6_6/near-real-time-searching.html#NearRealTimeSearching-AutoCommits) The default is `true`.
- * _WARNING_: If you set this to `false`, then commits will happen based on the SolrClient's internal mechanism and
- worker failure *may* result data being acknowledged in storm but not written in Solr.
-* `solr.commit.waitFlush`
- * This is a boolean which defines whether the writer blocks the commit until the data is flushed. See [here](https://lucene.apache.org/solr/guide/6_6/near-real-time-searching.html#NearRealTimeSearching-AutoCommits) The default is `true`.
- * _WARNING_: If you set this to `false`, then commits will happen based on the SolrClient's internal mechanism and
- worker failure *may* result data being acknowledged in storm but not written in Solr.
-* `solr.collection`
- * The default solr collection (if unspecified, the name is `metron`). By default, sensors will write to a collection associated with the index name in the
- indexing config for that sensor. If that index name is the empty string, then the default collection will be used.
-* `solr.http.config`
- * This is a map which allows users to configure the Solr client's HTTP client.
- * Possible fields here are:
- * `socketTimeout` : Socket timeout measured in ms, closes a socket if read takes longer than x ms to complete
- throws `java.net.SocketTimeoutException: Read timed out exception`
- * `connTimeout` : Connection timeout measures in ms, closes a socket if connection cannot be established within x ms
- with a `java.net.SocketTimeoutException: Connection timed out`
- * `maxConectionsPerHost` : Maximum connections allowed per host
- * `maxConnections` : Maximum total connections allowed
- * `retry` : Retry http requests on error
- * `allowCompression` : Allow compression (deflate,gzip) if server supports it
- * `followRedirects` : Follow redirects
- * `httpBasicAuthUser` : Basic auth username
- * `httpBasicAuthPassword` : Basic auth password
- * `solr.ssl.checkPeerName` : Check peer name
-
-
-## Installing
-
-Solr is installed in the [full dev environment for CentOS](../../metron-deployment/development/centos6) by default but is not started initially. Navigate to `$METRON_HOME/bin`
-and start Solr Cloud by running `start_solr.sh`.
-
-Metron's Ambari MPack installs several scripts in `$METRON_HOME/bin` that can be used to manage Solr. A script is also provided for installing Solr Cloud outside of full dev.
-The script performs the following tasks
-
-* Stops ES and Kibana
-* Downloads Solr
-* Installs Solr
-* Starts Solr Cloud
-
-_Note: for details on setting up Solr Cloud in production mode, see https://lucene.apache.org/solr/guide/6_6/taking-solr-to-production.html_
-
-Navigate to `$METRON_HOME/bin` and spin up Solr Cloud by running `install_solr.sh`. After running this script,
-Elasticsearch and Kibana will have been stopped and you should now have an instance of Solr Cloud up and running at http://localhost:8983/solr/#/~cloud. This manner of starting Solr
-will also spin up an embedded Zookeeper instance at port 9983. More information can be found [here](https://lucene.apache.org/solr/guide/6_6/getting-started-with-solrcloud.html)
-
-Solr can also be installed using [HDP Search 3](https://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.6.4/bk_solr-search-installation/content/ch_hdp_search_30.html). HDP Search 3 sets the Zookeeper root to
-`/solr` so this will need to be added to each url in the comma-separated list in Ambari UI -> Services -> Metron -> Configs -> Index Settings -> Solr Zookeeper Urls. For example, in full dev
-this would be `node1:2181/solr`.
-
-## Enabling Solr
-
-Elasticsearch is the real-time store used by default in Metron. Solr can be enabled following these steps:
-
-1. Stop the Metron Indexing component in Ambari.
-1. Update Ambari UI -> Services -> Metron -> Configs -> Index Settings -> Solr Zookeeper Urls to match the Solr installation described in the previous section.
-1. Change Ambari UI -> Services -> Metron -> Configs -> Indexing -> Index Writer - Random Access -> Random Access Search Engine to `Solr`.
-1. Change Ambari UI -> Services -> Metron -> Configs -> REST -> Source Type Field Name to `source.type`.
-1. Change Ambari UI -> Services -> Metron -> Configs -> REST -> Threat Triage Score Field Name to `threat.triage.score`.
-1. Start the Metron Indexing component in Ambari.
-1. Restart Metron REST and the Alerts UI in Ambari.
-
-This will automatically create collections for the schemas shipped with Metron:
-
-* bro
-* snort
-* yaf
-* error (used internally by Metron)
-* metaalert (used internall by Metron)
-
-Any other collections must be created manually before starting the Indexing component. Alerts should be present in the Alerts UI after enabling Solr.
-
-## Schemas
-
-As of now, we have mapped out the Schemas in `src/main/config/schema`.
-Ambari will eventually install these, but at the moment it's manual and
-you should refer to the Solr documentation [https://lucene.apache.org/solr/guide/6_6](here) in general
-and [here](https://lucene.apache.org/solr/guide/6_6/documents-fields-and-schema-design.html) if you'd like to know more about schemas in Solr.
-
-In Metron's Solr DAO implementation, document updates involve reading a document, applying the update and replacing the original by reindexing the whole document.
-Indexing LatLonType and PointType field types stores data in internal fields that should not be returned in search results. For these fields a dynamic field type matching the suffix needs to be added to store the data points.
-Solr 6+ comes with a new LatLonPointSpatialField field type that should be used instead of LatLonType if possible. Otherwise, a LatLongType field should be defined as:
-```
-<dynamicField name="*.location_point" type="location" multiValued="false" docValues="false"/>
-<dynamicField name="*_coordinate" type="pdouble" indexed="true" stored="false" docValues="false"/>
-<fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/>
-```
-A PointType field should be defined as:
-```
-<dynamicField name="*.point" type="point" multiValued="false" docValues="false"/>
-<dynamicField name="*_point" type="pdouble" indexed="true" stored="false" docValues="false"/>
-<fieldType name="point" class="solr.PointType" subFieldSuffix="_point"/>
-```
-If any copy fields are defined, stored and docValues should be set to false.
+The `solr` module is a module dedicated to indexing data into the Solr search engine.
-## Collections
+## Deployment Options
-Convenience scripts are provided with Metron to create and delete collections. Ambari uses these scripts to automatically create collections. To use them outside of Ambari, a few environment variables must be set first:
-```
-# Path to the zookeeper node used by Solr
-export ZOOKEEPER=node1:2181/solr
-# Set to true if Kerberos is enabled
-export SECURITY_ENABLED=true
-```
-The scripts can then be called directly with the collection name as the first argument . For example, to create the bro collection:
-```
-$METRON_HOME/bin/create_collection.sh bro
-```
-To delete the bro collection:
-```
-$METRON_HOME/bin/delete_collection.sh bro
-```
-The `create_collection.sh` script depends on schemas installed in `$METRON_HOME/config/schema`. There are several schemas that come with Metron:
+There is currently one option for indexing into Solr, which is the `random_access_indexing` Storm topology.
-* bro
-* snort
-* yaf
-* metaalert
-* error
+## Submodules
-Additional schemas should be installed in that location if using the `create_collection.sh` script. Any collection can be deleted with the `delete_collection.sh` script.
-These scripts use the [Solr Collection API](http://lucene.apache.org/solr/guide/6_6/collections-api.html).
\ No newline at end of file
+* metron-solr-common - this module contains common Solr code including Solr implementations of IndexDao related-classes.
+* metron-solr-storm - this module is home to Storm-specific code such as Flux property files, topology start script, and Storm indexing integration tests.
\ No newline at end of file
diff --git a/metron-platform/metron-solr/README.md b/metron-platform/metron-solr/metron-solr-common/README.md
similarity index 98%
copy from metron-platform/metron-solr/README.md
copy to metron-platform/metron-solr/metron-solr-common/README.md
index cba4dd5..081436a 100644
--- a/metron-platform/metron-solr/README.md
+++ b/metron-platform/metron-solr/metron-solr-common/README.md
@@ -33,7 +33,7 @@ Metron ships with Solr 6.6.2 support. Solr Cloud can be used as the real-time po
### The Indexing Topology
-Solr is a viable option for the `random access topology` and, similar to the Elasticsearch Writer, can be configured
+Solr is a viable option for indexing data in Metron and, similar to the Elasticsearch Writer, can be configured
via the global config. The following settings are possible as part of the global config:
* `solr.zookeeper`
* The zookeeper quorum associated with the SolrCloud instance. This is a required field with no default.
diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/metron-solr-common/pom.xml
similarity index 83%
copy from metron-platform/metron-solr/pom.xml
copy to metron-platform/metron-solr/metron-solr-common/pom.xml
index 6952769..24f8365 100644
--- a/metron-platform/metron-solr/pom.xml
+++ b/metron-platform/metron-solr/metron-solr-common/pom.xml
@@ -17,11 +17,11 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-platform</artifactId>
+ <artifactId>metron-solr</artifactId>
<version>0.7.2</version>
</parent>
- <artifactId>metron-solr</artifactId>
- <name>metron-solr</name>
+ <artifactId>metron-solr-common</artifactId>
+ <name>metron-solr-common</name>
<url>https://metron.apache.org/</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -40,34 +40,6 @@
<version>${global_solr_version}</version>
</dependency>
<dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- <version>${global_storm_version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-over-slf4j</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-slf4j-impl</artifactId>
- <groupId>org.apache.logging.log4j</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${global_hbase_version}</version>
@@ -129,7 +101,7 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-indexing</artifactId>
+ <artifactId>metron-indexing-common</artifactId>
<version>${project.parent.version}</version>
<exclusions>
<exclusion>
@@ -157,6 +129,10 @@
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
<exclusion>
@@ -212,8 +188,8 @@
<artifactId>hadoop-common</artifactId>
</exclusion>
<exclusion>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-core</artifactId>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -225,34 +201,34 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-indexing</artifactId>
+ <artifactId>metron-indexing-common</artifactId>
<version>${project.parent.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-hbase</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-core</artifactId>
- <version>1.3</version>
- <scope>test</scope>
- </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-hbase</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
+ <version>1.3</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
@@ -271,6 +247,18 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>${global_jar_version}</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>${global_shade_version}</version>
<configuration>
@@ -286,14 +274,14 @@
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
<filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
</filters>
<relocations>
<relocation>
@@ -307,8 +295,6 @@
</relocations>
<artifactSet>
<excludes>
- <exclude>storm:storm-core:*</exclude>
- <exclude>storm:storm-lib:*</exclude>
<exclude>org.slf4j.impl*</exclude>
<exclude>org.slf4j:slf4j-log4j*</exclude>
</excludes>
diff --git a/metron-platform/metron-solr/src/main/assembly/assembly.xml b/metron-platform/metron-solr/metron-solr-common/src/main/assembly/assembly.xml
similarity index 86%
copy from metron-platform/metron-solr/src/main/assembly/assembly.xml
copy to metron-platform/metron-solr/metron-solr-common/src/main/assembly/assembly.xml
index aabc537..d515d9e 100644
--- a/metron-platform/metron-solr/src/main/assembly/assembly.xml
+++ b/metron-platform/metron-solr/metron-solr-common/src/main/assembly/assembly.xml
@@ -43,13 +43,5 @@
<lineEnding>unix</lineEnding>
<filtered>true</filtered>
</fileSet>
- <fileSet>
- <directory>${project.basedir}/target</directory>
- <includes>
- <include>${project.artifactId}-${project.version}-uber.jar</include>
- </includes>
- <outputDirectory>lib</outputDirectory>
- <useDefaultExcludes>true</useDefaultExcludes>
- </fileSet>
</fileSets>
</assembly>
diff --git a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml b/metron-platform/metron-solr/metron-solr-common/src/main/config/schema/bro/schema.xml
similarity index 100%
rename from metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
rename to metron-platform/metron-solr/metron-solr-common/src/main/config/schema/bro/schema.xml
diff --git a/metron-platform/metron-solr/src/test/resources/config/test/conf/solrconfig.xml b/metron-platform/metron-solr/metron-solr-common/src/main/config/schema/bro/solrconfig.xml
similarity index 100%
rename from metron-platform/metron-solr/src/test/resources/config/test/conf/solrconfig.xml
rename to metron-platform/metron-solr/metron-solr-common/src/main/config/schema/bro/solrconfig.xml
diff --git a/metron-platform/metron-solr/src/main/config/schema/error/schema.xml b/metron-platform/metron-solr/metron-solr-common/src/main/config/schema/error/schema.xml
similarity index 100%
rename from metron-platform/metron-solr/src/main/config/schema/error/schema.xml
rename to metron-platform/metron-solr/metron-solr-common/src/main/config/schema/error/schema.xml
diff --git a/metron-platform/metron-solr/src/main/config/schema/yaf/solrconfig.xml b/metron-platform/metron-solr/metron-solr-common/src/main/config/schema/error/solrconfig.xml
similarity index 100%
rename from metron-platform/metron-solr/src/main/config/schema/yaf/solrconfig.xml
rename to metron-platform/metron-solr/metron-solr-common/src/main/config/schema/error/solrconfig.xml
diff --git a/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml b/metron-platform/metron-solr/metron-solr-common/src/main/config/schema/metaalert/schema.xml
similarity index 100%
rename from metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml
rename to metron-platform/metron-solr/metron-solr-common/src/main/config/schema/metaalert/schema.xml
diff --git a/metron-platform/metron-solr/src/main/config/schema/snort/solrconfig.xml b/metron-platform/metron-solr/metron-solr-common/src/main/config/schema/metaalert/solrconfig.xml
similarity index 100%
rename from metron-platform/metron-solr/src/main/config/schema/snort/solrconfig.xml
rename to metron-platform/metron-solr/metron-solr-common/src/main/config/schema/metaalert/solrconfig.xml
diff --git a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml b/metron-platform/metron-solr/metron-solr-common/src/main/config/schema/snort/schema.xml
similarity index 100%
rename from metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
rename to metron-platform/metron-solr/metron-solr-common/src/main/config/schema/snort/schema.xml
diff --git a/metron-platform/metron-solr/src/main/config/schema/metaalert/solrconfig.xml b/metron-platform/metron-solr/metron-solr-common/src/main/config/schema/snort/solrconfig.xml
similarity index 100%
rename from metron-platform/metron-solr/src/main/config/schema/metaalert/solrconfig.xml
rename to metron-platform/metron-solr/metron-solr-common/src/main/config/schema/snort/solrconfig.xml
diff --git a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml b/metron-platform/metron-solr/metron-solr-common/src/main/config/schema/yaf/schema.xml
similarity index 100%
rename from metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
rename to metron-platform/metron-solr/metron-solr-common/src/main/config/schema/yaf/schema.xml
diff --git a/metron-platform/metron-solr/src/main/config/schema/error/solrconfig.xml b/metron-platform/metron-solr/metron-solr-common/src/main/config/schema/yaf/solrconfig.xml
similarity index 100%
rename from metron-platform/metron-solr/src/main/config/schema/error/solrconfig.xml
rename to metron-platform/metron-solr/metron-solr-common/src/main/config/schema/yaf/solrconfig.xml
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/SolrConstants.java
similarity index 100%
rename from metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java
rename to metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/SolrConstants.java
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/client/SolrClientFactory.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/client/SolrClientFactory.java
similarity index 100%
rename from metron-platform/metron-solr/src/main/java/org/apache/metron/solr/client/SolrClientFactory.java
rename to metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/client/SolrClientFactory.java
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
similarity index 100%
rename from metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
rename to metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrDao.java
similarity index 100%
rename from metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
rename to metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrDao.java
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
similarity index 100%
rename from metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
rename to metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertRetrieveLatestDao.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertRetrieveLatestDao.java
similarity index 100%
rename from metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertRetrieveLatestDao.java
rename to metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertRetrieveLatestDao.java
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java
similarity index 100%
rename from metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java
rename to metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
similarity index 100%
rename from metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
rename to metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
similarity index 100%
rename from metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
rename to metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
similarity index 100%
rename from metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
rename to metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
similarity index 100%
rename from metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
rename to metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
similarity index 100%
rename from metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
rename to metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/schema/FieldType.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/schema/FieldType.java
similarity index 100%
rename from metron-platform/metron-solr/src/main/java/org/apache/metron/solr/schema/FieldType.java
rename to metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/schema/FieldType.java
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/schema/SchemaTranslator.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/schema/SchemaTranslator.java
similarity index 100%
rename from metron-platform/metron-solr/src/main/java/org/apache/metron/solr/schema/SchemaTranslator.java
rename to metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/schema/SchemaTranslator.java
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/MetronSolrClient.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/MetronSolrClient.java
similarity index 100%
rename from metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/MetronSolrClient.java
rename to metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/MetronSolrClient.java
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/SolrWriter.java
similarity index 100%
rename from metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java
rename to metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/SolrWriter.java
diff --git a/metron-platform/metron-solr/src/main/scripts/create_collection.sh b/metron-platform/metron-solr/metron-solr-common/src/main/scripts/create_collection.sh
similarity index 100%
rename from metron-platform/metron-solr/src/main/scripts/create_collection.sh
rename to metron-platform/metron-solr/metron-solr-common/src/main/scripts/create_collection.sh
diff --git a/metron-platform/metron-solr/src/main/scripts/delete_collection.sh b/metron-platform/metron-solr/metron-solr-common/src/main/scripts/delete_collection.sh
similarity index 100%
rename from metron-platform/metron-solr/src/main/scripts/delete_collection.sh
rename to metron-platform/metron-solr/metron-solr-common/src/main/scripts/delete_collection.sh
diff --git a/metron-platform/metron-solr/src/main/scripts/install_solr.sh b/metron-platform/metron-solr/metron-solr-common/src/main/scripts/install_solr.sh
similarity index 100%
rename from metron-platform/metron-solr/src/main/scripts/install_solr.sh
rename to metron-platform/metron-solr/metron-solr-common/src/main/scripts/install_solr.sh
diff --git a/metron-platform/metron-solr/src/main/scripts/start_solr.sh b/metron-platform/metron-solr/metron-solr-common/src/main/scripts/start_solr.sh
similarity index 100%
rename from metron-platform/metron-solr/src/main/scripts/start_solr.sh
rename to metron-platform/metron-solr/metron-solr-common/src/main/scripts/start_solr.sh
diff --git a/metron-platform/metron-solr/src/main/scripts/stop_solr.sh b/metron-platform/metron-solr/metron-solr-common/src/main/scripts/stop_solr.sh
similarity index 100%
rename from metron-platform/metron-solr/src/main/scripts/stop_solr.sh
rename to metron-platform/metron-solr/metron-solr-common/src/main/scripts/stop_solr.sh
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/enrichment/integration/utils/SampleUtil.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/enrichment/integration/utils/SampleUtil.java
similarity index 93%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/enrichment/integration/utils/SampleUtil.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/enrichment/integration/utils/SampleUtil.java
index bffae1c..1616419 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/enrichment/integration/utils/SampleUtil.java
+++ b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/enrichment/integration/utils/SampleUtil.java
@@ -27,7 +27,7 @@ public class SampleUtil {
public static Configurations getSampleConfigs() throws IOException {
Configurations configurations = new Configurations();
- configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
+ configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile("../" + TestConstants.SAMPLE_CONFIG_PATH));
return configurations;
}
@@ -53,8 +53,8 @@ public class SampleUtil {
public static IndexingConfigurations getSampleIndexingConfigs() throws IOException {
IndexingConfigurations configurations = new IndexingConfigurations();
- configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
- Map<String, byte[]> sensorIndexingConfigs = ConfigurationsUtils.readSensorIndexingConfigsFromFile(TestConstants.SAMPLE_CONFIG_PATH);
+ configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile("../" + TestConstants.SAMPLE_CONFIG_PATH));
+ Map<String, byte[]> sensorIndexingConfigs = ConfigurationsUtils.readSensorIndexingConfigsFromFile("../" + TestConstants.SAMPLE_CONFIG_PATH);
for(String sensorType: sensorIndexingConfigs.keySet()) {
configurations.updateSensorIndexingConfig(sensorType, sensorIndexingConfigs.get(sensorType));
}
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/client/SolrClientFactoryTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/client/SolrClientFactoryTest.java
similarity index 100%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/client/SolrClientFactoryTest.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/client/SolrClientFactoryTest.java
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrColumnMetadataTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrColumnMetadataTest.java
similarity index 100%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrColumnMetadataTest.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrColumnMetadataTest.java
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java
similarity index 100%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java
similarity index 100%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java
similarity index 100%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
similarity index 100%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUtilitiesTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrUtilitiesTest.java
similarity index 100%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUtilitiesTest.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrUtilitiesTest.java
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java
similarity index 98%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java
index 0e4b71f..084660f 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java
+++ b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java
@@ -129,8 +129,8 @@ public class SolrMetaAlertIntegrationTest extends MetaAlertIntegrationTest {
public void setup()
throws IOException, InterruptedException, SolrServerException, KeeperException {
solr.addCollection(METAALERTS_COLLECTION,
- "../metron-solr/src/main/config/schema//metaalert");
- solr.addCollection(SENSOR_NAME, "../metron-solr/src/test/resources/config/test/conf");
+ "./src/main/config/schema//metaalert");
+ solr.addCollection(SENSOR_NAME, "./src/test/resources/config/test/conf");
}
@AfterClass
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrRetrieveLatestIntegrationTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrRetrieveLatestIntegrationTest.java
similarity index 97%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrRetrieveLatestIntegrationTest.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrRetrieveLatestIntegrationTest.java
index 3978052..8c18981 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrRetrieveLatestIntegrationTest.java
+++ b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrRetrieveLatestIntegrationTest.java
@@ -63,8 +63,8 @@ public class SolrRetrieveLatestIntegrationTest {
@Before
public void setup() throws Exception {
solrComponent
- .addCollection(TEST_COLLECTION, "../metron-solr/src/test/resources/config/test/conf");
- solrComponent.addCollection(BRO_SENSOR, "../metron-solr/src/main/config/schema/bro");
+ .addCollection(TEST_COLLECTION, "./src/test/resources/config/test/conf");
+ solrComponent.addCollection(BRO_SENSOR, "./src/main/config/schema/bro");
AccessConfig accessConfig = new AccessConfig();
Map<String, Object> globalConfig = new HashMap<>();
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
similarity index 98%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
index 081aa87..de01df9 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
+++ b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
@@ -58,8 +58,8 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest {
// The data is all static for searches, so we can set it up here, and not do anything between tests.
broData = SearchIntegrationTest.broData.replace("source:type", "source.type");
snortData = SearchIntegrationTest.snortData.replace("source:type", "source.type");
- solrComponent.addCollection("bro", "../metron-solr/src/main/config/schema/bro");
- solrComponent.addCollection("snort", "../metron-solr/src/main/config/schema/snort");
+ solrComponent.addCollection("bro", "./src/main/config/schema/bro");
+ solrComponent.addCollection("snort", "./src/main/config/schema/snort");
loadTestData();
}
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
similarity index 97%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
index b2017ea..d49f4b4 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
+++ b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
@@ -70,8 +70,8 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
@Before
public void setup() throws Exception {
- solrComponent.addCollection(SENSOR_NAME, "../metron-solr/src/test/resources/config/test/conf");
- solrComponent.addCollection("error", "../metron-solr/src/main/config/schema/error");
+ solrComponent.addCollection(SENSOR_NAME, "./src/test/resources/config/test/conf");
+ solrComponent.addCollection("error", "./src/main/config/schema/error");
Configuration config = HBaseConfiguration.create();
MockHBaseTableProvider tableProvider = new MockHBaseTableProvider();
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
similarity index 100%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
similarity index 100%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/ModifiableSolrParamsMatcher.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/matcher/ModifiableSolrParamsMatcher.java
similarity index 100%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/ModifiableSolrParamsMatcher.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/matcher/ModifiableSolrParamsMatcher.java
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/SolrInputDocumentListMatcher.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/matcher/SolrInputDocumentListMatcher.java
similarity index 100%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/SolrInputDocumentListMatcher.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/matcher/SolrInputDocumentListMatcher.java
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/SolrInputDocumentMatcher.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/matcher/SolrInputDocumentMatcher.java
similarity index 100%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/SolrInputDocumentMatcher.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/matcher/SolrInputDocumentMatcher.java
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/SolrQueryMatcher.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/matcher/SolrQueryMatcher.java
similarity index 100%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/SolrQueryMatcher.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/matcher/SolrQueryMatcher.java
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/MetronSolrClientTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/MetronSolrClientTest.java
similarity index 100%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/MetronSolrClientTest.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/MetronSolrClientTest.java
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
similarity index 100%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
rename to metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
diff --git a/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema b/metron-platform/metron-solr/metron-solr-common/src/test/resources/config/test/conf/managed-schema
similarity index 100%
rename from metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema
rename to metron-platform/metron-solr/metron-solr-common/src/test/resources/config/test/conf/managed-schema
diff --git a/metron-platform/metron-solr/src/main/config/schema/bro/solrconfig.xml b/metron-platform/metron-solr/metron-solr-common/src/test/resources/config/test/conf/solrconfig.xml
similarity index 100%
rename from metron-platform/metron-solr/src/main/config/schema/bro/solrconfig.xml
rename to metron-platform/metron-solr/metron-solr-common/src/test/resources/config/test/conf/solrconfig.xml
diff --git a/metron-platform/metron-solr/src/test/resources/example_data/bro b/metron-platform/metron-solr/metron-solr-common/src/test/resources/example_data/bro
similarity index 100%
rename from metron-platform/metron-solr/src/test/resources/example_data/bro
rename to metron-platform/metron-solr/metron-solr-common/src/test/resources/example_data/bro
diff --git a/metron-platform/metron-solr/src/test/resources/example_data/error b/metron-platform/metron-solr/metron-solr-common/src/test/resources/example_data/error
similarity index 100%
rename from metron-platform/metron-solr/src/test/resources/example_data/error
rename to metron-platform/metron-solr/metron-solr-common/src/test/resources/example_data/error
diff --git a/metron-platform/metron-solr/src/test/resources/example_data/snort b/metron-platform/metron-solr/metron-solr-common/src/test/resources/example_data/snort
similarity index 100%
rename from metron-platform/metron-solr/src/test/resources/example_data/snort
rename to metron-platform/metron-solr/metron-solr-common/src/test/resources/example_data/snort
diff --git a/metron-platform/metron-solr/src/test/resources/example_data/yaf b/metron-platform/metron-solr/metron-solr-common/src/test/resources/example_data/yaf
similarity index 100%
rename from metron-platform/metron-solr/src/test/resources/example_data/yaf
rename to metron-platform/metron-solr/metron-solr-common/src/test/resources/example_data/yaf
diff --git a/metron-platform/metron-elasticsearch/src/test/resources/log4j.properties b/metron-platform/metron-solr/metron-solr-common/src/test/resources/log4j.properties
similarity index 100%
rename from metron-platform/metron-elasticsearch/src/test/resources/log4j.properties
rename to metron-platform/metron-solr/metron-solr-common/src/test/resources/log4j.properties
diff --git a/metron-platform/metron-solr/src/test/resources/log4j2.xml b/metron-platform/metron-solr/metron-solr-common/src/test/resources/log4j2.xml
similarity index 100%
rename from metron-platform/metron-solr/src/test/resources/log4j2.xml
rename to metron-platform/metron-solr/metron-solr-common/src/test/resources/log4j2.xml
diff --git a/metron-platform/metron-solr/src/test/resources/solr/conf/_rest_managed.json b/metron-platform/metron-solr/metron-solr-common/src/test/resources/solr/conf/_rest_managed.json
similarity index 100%
rename from metron-platform/metron-solr/src/test/resources/solr/conf/_rest_managed.json
rename to metron-platform/metron-solr/metron-solr-common/src/test/resources/solr/conf/_rest_managed.json
diff --git a/metron-platform/metron-solr/src/test/resources/solr/conf/currency.xml b/metron-platform/metron-solr/metron-solr-common/src/test/resources/solr/conf/currency.xml
similarity index 100%
rename from metron-platform/metron-solr/src/test/resources/solr/conf/currency.xml
rename to metron-platform/metron-solr/metron-solr-common/src/test/resources/solr/conf/currency.xml
diff --git a/metron-platform/metron-solr/src/test/resources/solr/conf/lang/stopwords_en.txt b/metron-platform/metron-solr/metron-solr-common/src/test/resources/solr/conf/lang/stopwords_en.txt
similarity index 100%
rename from metron-platform/metron-solr/src/test/resources/solr/conf/lang/stopwords_en.txt
rename to metron-platform/metron-solr/metron-solr-common/src/test/resources/solr/conf/lang/stopwords_en.txt
diff --git a/metron-platform/metron-solr/src/test/resources/solr/conf/protwords.txt b/metron-platform/metron-solr/metron-solr-common/src/test/resources/solr/conf/protwords.txt
similarity index 100%
rename from metron-platform/metron-solr/src/test/resources/solr/conf/protwords.txt
rename to metron-platform/metron-solr/metron-solr-common/src/test/resources/solr/conf/protwords.txt
diff --git a/metron-platform/metron-solr/src/test/resources/solr/conf/schema.xml b/metron-platform/metron-solr/metron-solr-common/src/test/resources/solr/conf/schema.xml
similarity index 100%
rename from metron-platform/metron-solr/src/test/resources/solr/conf/schema.xml
rename to metron-platform/metron-solr/metron-solr-common/src/test/resources/solr/conf/schema.xml
diff --git a/metron-platform/metron-solr/src/test/resources/solr/conf/solrconfig.xml b/metron-platform/metron-solr/metron-solr-common/src/test/resources/solr/conf/solrconfig.xml
similarity index 100%
rename from metron-platform/metron-solr/src/test/resources/solr/conf/solrconfig.xml
rename to metron-platform/metron-solr/metron-solr-common/src/test/resources/solr/conf/solrconfig.xml
diff --git a/metron-platform/metron-solr/src/test/resources/solr/conf/stopwords.txt b/metron-platform/metron-solr/metron-solr-common/src/test/resources/solr/conf/stopwords.txt
similarity index 100%
rename from metron-platform/metron-solr/src/test/resources/solr/conf/stopwords.txt
rename to metron-platform/metron-solr/metron-solr-common/src/test/resources/solr/conf/stopwords.txt
diff --git a/metron-platform/metron-solr/src/test/resources/solr/conf/synonyms.txt b/metron-platform/metron-solr/metron-solr-common/src/test/resources/solr/conf/synonyms.txt
similarity index 100%
rename from metron-platform/metron-solr/src/test/resources/solr/conf/synonyms.txt
rename to metron-platform/metron-solr/metron-solr-common/src/test/resources/solr/conf/synonyms.txt
diff --git a/metron-platform/metron-solr/src/test/resources/solr/solr.xml b/metron-platform/metron-solr/metron-solr-common/src/test/resources/solr/solr.xml
similarity index 100%
rename from metron-platform/metron-solr/src/test/resources/solr/solr.xml
rename to metron-platform/metron-solr/metron-solr-common/src/test/resources/solr/solr.xml
diff --git a/metron-platform/metron-solr/metron-solr-storm/README.md b/metron-platform/metron-solr/metron-solr-storm/README.md
new file mode 100644
index 0000000..bb2df89
--- /dev/null
+++ b/metron-platform/metron-solr/metron-solr-storm/README.md
@@ -0,0 +1,20 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# Metron Solr Storm
+
+This module contains assets for running the Random Access Indexing Storm topology with Solr. An integration test for this topology is also included.
diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/metron-solr-storm/pom.xml
similarity index 51%
copy from metron-platform/metron-solr/pom.xml
copy to metron-platform/metron-solr/metron-solr-storm/pom.xml
index 6952769..22c8c50 100644
--- a/metron-platform/metron-solr/pom.xml
+++ b/metron-platform/metron-solr/metron-solr-storm/pom.xml
@@ -17,130 +17,43 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-platform</artifactId>
+ <artifactId>metron-solr</artifactId>
<version>0.7.2</version>
</parent>
- <artifactId>metron-solr</artifactId>
- <name>metron-solr</name>
+ <artifactId>metron-solr-storm</artifactId>
+ <name>metron-solr-storm</name>
+ <description>Components common to Storm</description>
<url>https://metron.apache.org/</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <guava_version>${global_guava_version}</guava_version>
+ <commons.config.version>1.10</commons.config.version>
</properties>
<dependencies>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava_version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.solr</groupId>
- <artifactId>solr-solrj</artifactId>
- <version>${global_solr_version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- <version>${global_storm_version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-over-slf4j</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-slf4j-impl</artifactId>
- <groupId>org.apache.logging.log4j</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <version>${global_hbase_version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${global_hadoop_version}</version>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- </exclusions>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-solr-common</artifactId>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${global_hadoop_version}</version>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>commons-httpclient</artifactId>
- <groupId>commons-httpclient</groupId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-solr-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
</dependency>
<dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
- <version>${global_kafka_version}</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-indexing-storm</artifactId>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-indexing</artifactId>
+ <artifactId>metron-indexing-storm</artifactId>
<version>${project.parent.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </exclusion>
- </exclusions>
+ <scope>test</scope>
+ <type>test-jar</type>
</dependency>
-
- <!-- Test -->
-
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-test-framework</artifactId>
@@ -157,6 +70,10 @@
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
<exclusion>
@@ -166,107 +83,31 @@
</exclusions>
</dependency>
<dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- <version>${global_log4j_core_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <version>${global_log4j_core_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- <version>${global_kafka_version}</version>
- <classifier>test</classifier>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-common</artifactId>
- <version>${project.parent.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>flux-core</artifactId>
+ <version>${global_flux_version}</version>
</dependency>
<dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-integration-test</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ <version>${global_storm_version}</version>
+ <scope>provided</scope>
<exclusions>
<exclusion>
<artifactId>servlet-api</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
<exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-core</artifactId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <version>${global_mockito_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-indexing</artifactId>
- <version>${project.parent.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-hbase</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-core</artifactId>
- <version>1.3</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${global_jackson_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- <version>${global_jackson_version}</version>
- <scope>test</scope>
- </dependency>
</dependencies>
-
<build>
<plugins>
<plugin>
@@ -286,29 +127,23 @@
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
<filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
</filters>
<relocations>
<relocation>
- <pattern>com.google.common</pattern>
- <shadedPattern>org.apache.metron.guava.${guava_version}</shadedPattern>
- </relocation>
- <relocation>
<pattern>com.fasterxml.jackson</pattern>
<shadedPattern>org.apache.metron.jackson</shadedPattern>
</relocation>
</relocations>
<artifactSet>
<excludes>
- <exclude>storm:storm-core:*</exclude>
- <exclude>storm:storm-lib:*</exclude>
<exclude>org.slf4j.impl*</exclude>
<exclude>org.slf4j:slf4j-log4j*</exclude>
</excludes>
diff --git a/metron-platform/metron-solr/src/main/assembly/assembly.xml b/metron-platform/metron-solr/metron-solr-storm/src/main/assembly/assembly.xml
similarity index 100%
rename from metron-platform/metron-solr/src/main/assembly/assembly.xml
rename to metron-platform/metron-solr/metron-solr-storm/src/main/assembly/assembly.xml
diff --git a/metron-platform/metron-solr/src/main/config/solr.properties b/metron-platform/metron-solr/metron-solr-storm/src/main/config/solr.properties
similarity index 100%
rename from metron-platform/metron-solr/src/main/config/solr.properties
rename to metron-platform/metron-solr/metron-solr-storm/src/main/config/solr.properties
diff --git a/metron-platform/metron-solr/src/main/config/solr.properties.j2 b/metron-platform/metron-solr/metron-solr-storm/src/main/config/solr.properties.j2
similarity index 100%
rename from metron-platform/metron-solr/src/main/config/solr.properties.j2
rename to metron-platform/metron-solr/metron-solr-storm/src/main/config/solr.properties.j2
diff --git a/metron-platform/metron-solr/src/main/scripts/start_solr_topology.sh b/metron-platform/metron-solr/metron-solr-storm/src/main/scripts/start_solr_topology.sh
similarity index 100%
rename from metron-platform/metron-solr/src/main/scripts/start_solr_topology.sh
rename to metron-platform/metron-solr/metron-solr-storm/src/main/scripts/start_solr_topology.sh
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java b/metron-platform/metron-solr/metron-solr-storm/src/test/java/org/apache/metron/indexing/integration/SolrIndexingIntegrationTest.java
similarity index 91%
rename from metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
rename to metron-platform/metron-solr/metron-solr-storm/src/test/java/org/apache/metron/indexing/integration/SolrIndexingIntegrationTest.java
index 2f9b285..2fb48b7 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
+++ b/metron-platform/metron-solr/metron-solr-storm/src/test/java/org/apache/metron/indexing/integration/SolrIndexingIntegrationTest.java
@@ -15,15 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.metron.solr.integration;
-
-import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER;
+package org.apache.metron.indexing.integration;
import com.google.common.base.Function;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import javax.annotation.Nullable;
+
+import org.apache.metron.TestConstants;
import org.apache.metron.common.configuration.Configurations;
import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.common.field.FieldNameConverter;
@@ -37,6 +37,7 @@ import org.apache.metron.integration.ProcessorResult;
import org.apache.metron.integration.ReadinessState;
import org.apache.metron.integration.components.KafkaComponent;
import org.apache.metron.integration.components.ZKServerComponent;
+import org.apache.metron.solr.SolrConstants;
import org.apache.metron.solr.integration.components.SolrComponent;
@@ -53,7 +54,7 @@ public class SolrIndexingIntegrationTest extends IndexingIntegrationTest {
@Override
public InMemoryComponent getSearchComponent(final Properties topologyProperties) throws Exception {
SolrComponent solrComponent = new SolrComponent.Builder()
- .addInitialCollection(collection, "../metron-solr/src/main/config/schema/yaf")
+ .addInitialCollection(collection, "../metron-solr-common/src/main/config/schema/yaf")
.withPostStartCallback(new Function<SolrComponent, Void>() {
@Nullable
@Override
@@ -63,7 +64,7 @@ public class SolrIndexingIntegrationTest extends IndexingIntegrationTest {
String testZookeeperUrl = topologyProperties.getProperty(ZKServerComponent.ZOOKEEPER_PROPERTY);
Configurations configurations = SampleUtil.getSampleConfigs();
Map<String, Object> globalConfig = configurations.getGlobalConfig();
- globalConfig.put(SOLR_ZOOKEEPER, solrComponent.getZookeeperUrl());
+ globalConfig.put(SolrConstants.SOLR_ZOOKEEPER, solrComponent.getZookeeperUrl());
ConfigurationsUtils.writeGlobalConfigToZookeeper(JSONUtils.INSTANCE.toJSONPretty(globalConfig), testZookeeperUrl);
} catch (Exception e) {
e.printStackTrace();
@@ -126,11 +127,11 @@ public class SolrIndexingIntegrationTest extends IndexingIntegrationTest {
@Override
public String getTemplatePath() {
- return "../metron-solr/src/main/config/solr.properties.j2";
+ return "./src/main/config/solr.properties.j2";
}
@Override
public String getFluxPath() {
- return "../metron-indexing/src/main/flux/indexing/random_access/remote.yaml";
+ return "../../metron-indexing/metron-indexing-storm/src/main/flux/indexing/random_access/remote.yaml";
}
}
diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/pom.xml
index 6952769..8ced553 100644
--- a/metron-platform/metron-solr/pom.xml
+++ b/metron-platform/metron-solr/pom.xml
@@ -12,346 +12,35 @@
the specific language governing permissions and limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-platform</artifactId>
- <version>0.7.2</version>
- </parent>
- <artifactId>metron-solr</artifactId>
- <name>metron-solr</name>
- <url>https://metron.apache.org/</url>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <guava_version>${global_guava_version}</guava_version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava_version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.solr</groupId>
- <artifactId>solr-solrj</artifactId>
- <version>${global_solr_version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- <version>${global_storm_version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-over-slf4j</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-slf4j-impl</artifactId>
- <groupId>org.apache.logging.log4j</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <version>${global_hbase_version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${global_hadoop_version}</version>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${global_hadoop_version}</version>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>commons-httpclient</artifactId>
- <groupId>commons-httpclient</groupId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
- <version>${global_kafka_version}</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-indexing</artifactId>
- <version>${project.parent.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <!-- Test -->
-
- <dependency>
- <groupId>org.apache.solr</groupId>
- <artifactId>solr-test-framework</artifactId>
- <version>${global_solr_version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <artifactId>fastutil</artifactId>
- <groupId>it.unimi.dsi</groupId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>caffeine</artifactId>
- <groupId>com.github.ben-manes.caffeine</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- <version>${global_log4j_core_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <version>${global_log4j_core_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- <version>${global_kafka_version}</version>
- <classifier>test</classifier>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-common</artifactId>
- <version>${project.parent.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-integration-test</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-core</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <version>${global_mockito_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-indexing</artifactId>
- <version>${project.parent.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-hbase</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-core</artifactId>
- <version>1.3</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${global_jackson_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- <version>${global_jackson_version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${global_shade_version}</version>
- <configuration>
- <createDependencyReducedPom>true</createDependencyReducedPom>
- </configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <shadedArtifactAttached>true</shadedArtifactAttached>
- <shadedClassifierName>uber</shadedClassifierName>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- <relocations>
- <relocation>
- <pattern>com.google.common</pattern>
- <shadedPattern>org.apache.metron.guava.${guava_version}</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.fasterxml.jackson</pattern>
- <shadedPattern>org.apache.metron.jackson</shadedPattern>
- </relocation>
- </relocations>
- <artifactSet>
- <excludes>
- <exclude>storm:storm-core:*</exclude>
- <exclude>storm:storm-lib:*</exclude>
- <exclude>org.slf4j.impl*</exclude>
- <exclude>org.slf4j:slf4j-log4j*</exclude>
- </excludes>
- </artifactSet>
- <transformers>
- <transformer implementation="org.atteo.classindex.ClassIndexTransformer"/>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
- <resource>.yaml</resource>
- </transformer>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass></mainClass>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>org.atteo.classindex</groupId>
- <artifactId>classindex-transformer</artifactId>
- <version>${global_classindex_version}</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <descriptor>src/main/assembly/assembly.xml</descriptor>
- </configuration>
- <executions>
- <execution>
- <id>make-assembly</id> <!-- this is used for inheritance merges -->
- <phase>package</phase> <!-- bind to the packaging phase -->
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>metron-solr</artifactId>
+ <packaging>pom</packaging>
+ <name>metron-solr</name>
+ <parent>
+ <artifactId>metron-platform</artifactId>
+ <groupId>org.apache.metron</groupId>
+ <version>0.7.2</version>
+ </parent>
+ <description>Metron parsers and platform support</description>
+ <url>https://metron.apache.org/</url>
+ <scm>
+ <connection>scm:git:https://gitbox.apache.org/repos/asf/metron.git</connection>
+ <developerConnection>scm:git:https://gitbox.apache.org/repos/asf/metron.git</developerConnection>
+ <tag>HEAD</tag>
+ <url>https://gitbox.apache.org/repos/asf/metron.git</url>
+ </scm>
+ <licenses>
+ <license>
+ <name>The Apache Software License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ </license>
+ </licenses>
+ <modules>
+ <module>metron-solr-common</module>
+ <module>metron-solr-storm</module>
+ </modules>
+</project>
\ No newline at end of file