You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2018/12/19 00:31:52 UTC
[metron] branch master updated: METRON-1815: Separate
metron-parsers into metron-parsers-common and metron-parsers-storm
(justinleet via mmiklavc) closes apache/metron#1249
This is an automated email from the ASF dual-hosted git repository.
mmiklavcic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new d7d4fd9 METRON-1815: Separate metron-parsers into metron-parsers-common and metron-parsers-storm (justinleet via mmiklavc) closes apache/metron#1249
d7d4fd9 is described below
commit d7d4fd9afb19e2bd2e66babb7e1514a19eae07d0
Author: justinleet <ju...@gmail.com>
AuthorDate: Tue Dec 18 17:31:08 2018 -0700
METRON-1815: Separate metron-parsers into metron-parsers-common and metron-parsers-storm (justinleet via mmiklavc) closes apache/metron#1249
---
README.md | 2 +-
dependencies_with_url.csv | 2 +
metron-contrib/metron-docker/README.md | 2 +-
.../metron-docker/compose/storm/Dockerfile | 2 +-
metron-contrib/metron-docker/install-metron.sh | 3 +-
metron-contrib/metron-docker/pom.xml | 6 +-
.../common-services/METRON/CURRENT/metainfo.xml | 6 +
.../packaging/docker/deb-docker/pom.xml | 14 +-
.../packaging/docker/rpm-docker/SPECS/metron.spec | 62 ++++-
.../packaging/docker/rpm-docker/pom.xml | 14 +-
metron-interface/metron-rest/pom.xml | 34 ++-
.../org/apache/metron/rest/config/GrokConfig.java | 3 +-
.../metron/rest/service/impl/GrokServiceImpl.java | 3 +-
.../metron-rest/src/main/scripts/metron-rest.sh | 3 +-
.../controller/GrokControllerIntegrationTest.java | 2 +-
.../rest/service/impl/GrokServiceImplTest.java | 3 +-
.../impl/SensorParserConfigServiceImplTest.java | 1 +
metron-platform/Performance-tuning-guide.md | 6 +-
metron-platform/metron-common/README.md | 49 ++--
.../metron-common/src/main/scripts/stellar | 3 +-
.../src/main/scripts/zk_load_configs.sh | 3 +-
.../configuration/ConfigurationsUtilsTest.java | 1 +
metron-platform/metron-management/pom.xml | 12 +
.../org/apache/metron/parsers/SettingsLoader.java | 166 -------------
.../parsers/integration/AsaIntegrationTest.java | 37 ---
.../parsers/integration/BroIntegrationTest.java | 37 ---
.../integration/JSONMapIntegrationTest.java | 37 ---
.../integration/JSONMapQueryIntegrationTest.java | 36 ---
.../parsers/integration/SnortIntegrationTest.java | 37 ---
.../parsers/integration/SquidIntegrationTest.java | 37 ---
.../Syslog5424ParserIntegrationTest.java | 37 ---
.../integration/WebSphereIntegrationTest.java | 40 ----
.../parsers/integration/YafIntegrationTest.java | 37 ---
.../{metron-parsers => metron-parsing}/README.md | 152 ++----------
.../metron-parsers-common}/3rdPartyParser.md | 2 +-
.../metron-parsers-common}/ParserChaining.md | 4 +-
.../metron-parsers-common}/README.md | 19 +-
.../message-parser-implementation-notes.md | 0
.../metron-parsers-common}/parser-testing.md | 41 +---
.../metron-parsers-common}/parser_arch.png | Bin
.../metron-parsers-common}/pom.xml | 59 +----
.../src/main/assembly/assembly.xml | 0
.../src/main/config/zookeeper/parsers/jsonMap.json | 0
.../config/zookeeper/parsers/jsonMapQuery.json | 0
.../zookeeper/parsers/jsonMapWrappedQuery.json | 0
.../org/apache/metron/parsers/BasicParser.java | 0
.../metron/parsers/DefaultMessageParserResult.java | 0
.../metron/parsers/DefaultParserRunnerResults.java | 2 +-
.../java/org/apache/metron/parsers/GrokParser.java | 1 -
.../org/apache/metron/parsers/ParseException.java | 0
.../apache/metron/parsers}/ParserComponent.java | 5 +-
.../org/apache/metron/parsers/ParserRunner.java | 0
.../apache/metron/parsers/ParserRunnerImpl.java | 30 ++-
.../apache/metron/parsers/ParserRunnerResults.java | 2 +-
.../org/apache/metron/parsers/csv/CSVParser.java | 0
.../parsers/filters/AbstractMessageFilter.java | 0
.../metron/parsers/filters/BroMessageFilter.java | 0
.../org/apache/metron/parsers/filters/Filters.java | 0
.../metron/parsers/filters/StellarFilter.java | 0
.../metron/parsers/interfaces/Configurable.java | 0
.../metron/parsers/interfaces/MessageFilter.java | 0
.../metron/parsers/interfaces/MessageParser.java | 0
.../parsers/interfaces/MessageParserResult.java | 0
.../apache/metron/parsers/json/JSONMapParser.java | 0
.../org/apache/metron/parsers/utils/DateUtils.java | 0
.../org/apache/metron/parsers/utils/GrokUtils.java | 0
.../apache/metron/parsers/utils/ParserUtils.java | 0
.../apache/metron/parsers/utils/SyslogUtils.java | 0
.../src/main/resources/META-INF/LICENSE | 0
.../src/main/resources/META-INF/NOTICE | 2 +-
.../src/main/resources/patterns/common | 0
.../org/apache/metron/filters/FiltersTest.java | 0
.../metron/parsers/AbstractParserConfigTest.java | 0
.../org/apache/metron/parsers/GrokParserTest.java | 0
.../apache/metron/parsers/MessageParserTest.java | 0
.../metron/parsers/MultiLineGrokParserTest.java | 2 +-
.../parsers/MultiLineWithErrorsGrokParserTest.java | 2 +-
.../metron/parsers/ParserRunnerImplTest.java | 38 ++-
.../metron/parsers/SampleGrokParserTest.java | 2 +-
.../apache/metron/parsers/csv/CSVParserTest.java | 0
.../EnvelopedParserIntegrationTest.java | 61 ++---
.../parsers/integration/ParserIntegrationTest.java | 49 ++--
.../parsers/integration/ParserValidation.java | 0
.../integration/validation/ParserDriver.java | 60 +++++
.../validation/SampleDataValidation.java | 2 +-
.../parsers/json/JSONMapParserQueryTest.java | 0
.../metron/parsers/json/JSONMapParserTest.java | 0
.../json/JSONMapParserWrappedQueryTest.java | 0
.../org/apache/metron/parsers/paloalto/README.md | 0
.../parsers/regex/RegularExpressionsParser.java | 0
.../regex/RegularExpressionsParserTest.java | 0
.../metron/parsers/utils/ParserUtilsTest.java | 0
.../metron/parsers/utils/SyslogUtilsTest.java | 0
.../writers/SimpleHBaseEnrichmentWriterTest.java | 0
.../src/test/resources/TestSchemas/BroSchema.json | 0
.../src/test/resources/TestSchemas/IseSchema.json | 0
.../test/resources/TestSchemas/LancopeSchema.json | 0
.../src/test/resources/TestSchemas/PcapSchema.json | 0
.../resources/TestSchemas/SourcefireSchema.json | 0
.../resources/config/BasicFireEyeParserTest.config | 0
.../resources/config/BasicIseParserTest.config | 0
.../resources/config/BasicLancopeParserTest.config | 0
.../config/BasicPaloAltoFirewallParserTest.config | 0
.../config/BasicSourcefireParserTest.config | 0
.../src/test/resources/config/BroParserTest.config | 0
.../test/resources/logData/FireEyeParserTest.txt | 0
.../src/test/resources/logData/IseParserTest.txt | 0
.../test/resources/logData/LancopeParserTest.txt | 0
.../resources/logData/SourcefireParserTest.txt | 0
.../src/test/resources/logData/multi_elb_log.txt | 0
.../logData/multi_elb_with_errors_log.txt | 0
.../org/apache/metron/parsers/cef/adallom.cef | 0
.../org/apache/metron/parsers/cef/adallom.schema | 0
.../org/apache/metron/parsers/cef/cyberark.cef | 0
.../org/apache/metron/parsers/cef/cyberark.json | 0
.../org/apache/metron/parsers/cef/cyberark.schema | 0
.../org/apache/metron/parsers/cef/palo.cef | 0
.../org/apache/metron/parsers/cef/palo.schema | 0
.../org/apache/metron/parsers/cef/waf.cef | 0
.../org/apache/metron/parsers/cef/waf.schema | 0
.../src/test/resources/patterns/cisco_patterns | 0
.../metron-parsers}/README.md | 27 ++-
.../metron-parsing/metron-parsers/pom.xml | 218 +++++++++++++++++
.../metron-parsers/src/main/assembly/assembly.xml | 0
.../src/main/config/zookeeper/parsers/asa.json | 0
.../src/main/config/zookeeper/parsers/bro.json | 0
.../src/main/config/zookeeper/parsers/jsonMap.json | 0
.../config/zookeeper/parsers/jsonMapQuery.json | 0
.../zookeeper/parsers/jsonMapWrappedQuery.json | 0
.../src/main/config/zookeeper/parsers/snort.json | 0
.../src/main/config/zookeeper/parsers/squid.json | 0
.../main/config/zookeeper/parsers/syslog5424.json | 0
.../main/config/zookeeper/parsers/websphere.json | 0
.../src/main/config/zookeeper/parsers/yaf.json | 0
.../apache/metron/parsers/asa/BasicAsaParser.java | 0
.../apache/metron/parsers/bro/BasicBroParser.java | 0
.../org/apache/metron/parsers/bro/JSONCleaner.java | 0
.../org/apache/metron/parsers/cef/CEFParser.java | 0
.../metron/parsers/fireeye/BasicFireEyeParser.java | 0
.../apache/metron/parsers/ise/BasicIseParser.java | 0
.../org/apache/metron/parsers/ise/ISEParser.java | 0
.../org/apache/metron/parsers/ise/ISEParser.jj | 0
.../metron/parsers/ise/ISEParserConstants.java | 0
.../metron/parsers/ise/ISEParserTokenManager.java | 0
.../apache/metron/parsers/ise/JavaCharStream.java | 0
.../apache/metron/parsers/ise/ParseException.java | 0
.../java/org/apache/metron/parsers/ise/Token.java | 0
.../apache/metron/parsers/ise/TokenMgrError.java | 0
.../metron/parsers/lancope/BasicLancopeParser.java | 0
.../parsers/logstash/BasicLogstashParser.java | 7 +-
.../paloalto/BasicPaloAltoFirewallParser.java | 0
.../org/apache/metron/parsers/paloalto/README.md | 0
.../metron/parsers/snort/BasicSnortParser.java | 0
.../parsers/sourcefire/BasicSourcefireParser.java | 9 +-
.../metron/parsers/syslog/Syslog5424Parser.java | 17 +-
.../parsers/websphere/GrokWebSphereParser.java | 5 +-
.../src/main/resources/META-INF/LICENSE | 0
.../src/main/resources/META-INF/NOTICE | 2 +-
.../metron-parsers/src/main/resources/patterns/asa | 0
.../src/main/resources/patterns/fireeye | 0
.../src/main/resources/patterns/sourcefire | 0
.../src/main/resources/patterns/squid | 0
.../src/main/resources/patterns/websphere | 0
.../metron-parsers/src/main/resources/patterns/yaf | 0
.../org/apache/metron/filters/FiltersTest.java | 0
.../org/apache/metron/parsers/SnortParserTest.java | 0
.../org/apache/metron/parsers/SquidParserTest.java | 0
.../org/apache/metron/parsers/YafParserTest.java | 0
.../metron/parsers/asa/BasicAsaParserTest.java | 0
.../metron/parsers/bro/BasicBroParserTest.java | 0
.../apache/metron/parsers/cef/CEFParserTest.java | 0
.../parsers/fireeye/BasicFireEyeParserTest.java | 0
.../metron/parsers/ise/BasicIseParserTest.java | 0
.../parsers/lancope/BasicLancopeParserTest.java | 0
.../paloalto/BasicPaloAltoFirewallParserTest.java | 0
.../sourcefire/BasicSourcefireParserTest.java | 0
.../parsers/syslog/Syslog5424ParserTest.java | 0
.../parsers/websphere/GrokWebSphereParserTest.java | 5 +-
.../src/test/resources/TestSchemas/BroSchema.json | 0
.../src/test/resources/TestSchemas/IseSchema.json | 0
.../test/resources/TestSchemas/LancopeSchema.json | 0
.../src/test/resources/TestSchemas/PcapSchema.json | 0
.../resources/TestSchemas/SourcefireSchema.json | 0
.../resources/config/BasicFireEyeParserTest.config | 0
.../resources/config/BasicIseParserTest.config | 0
.../resources/config/BasicLancopeParserTest.config | 0
.../config/BasicPaloAltoFirewallParserTest.config | 0
.../config/BasicSourcefireParserTest.config | 0
.../src/test/resources/config/BroParserTest.config | 0
.../test/resources/logData/FireEyeParserTest.txt | 0
.../src/test/resources/logData/IseParserTest.txt | 0
.../test/resources/logData/LancopeParserTest.txt | 0
.../resources/logData/SourcefireParserTest.txt | 0
.../src/test/resources/logData/multi_elb_log.txt | 0
.../logData/multi_elb_with_errors_log.txt | 0
.../org/apache/metron/parsers/cef/adallom.cef | 0
.../org/apache/metron/parsers/cef/adallom.schema | 0
.../org/apache/metron/parsers/cef/cyberark.cef | 0
.../org/apache/metron/parsers/cef/cyberark.json | 0
.../org/apache/metron/parsers/cef/cyberark.schema | 0
.../org/apache/metron/parsers/cef/palo.cef | 0
.../org/apache/metron/parsers/cef/palo.schema | 0
.../org/apache/metron/parsers/cef/waf.cef | 0
.../org/apache/metron/parsers/cef/waf.schema | 0
.../src/test/resources/patterns/cisco_patterns | 0
.../metron-parsing/metron-parsing-storm/README.md | 148 ++++++++++++
.../metron-parsing/metron-parsing-storm/pom.xml | 264 +++++++++++++++++++++
.../src/main/assembly/assembly.xml | 0
.../org/apache/metron/parsers/bolt/ParserBolt.java | 3 +-
.../org/apache/metron/parsers/bolt/WriterBolt.java | 3 +-
.../apache/metron/parsers/bolt/WriterHandler.java | 0
.../parsers/topology/MergeAndShadeTransformer.java | 18 +-
.../parsers/topology/ParserTopologyBuilder.java | 0
.../metron/parsers/topology/ParserTopologyCLI.java | 6 +-
.../apache/metron/parsers/topology/config/Arg.java | 0
.../parsers/topology/config/ConfigHandlers.java | 8 +-
.../parsers/topology/config/ValueSupplier.java | 0
.../src/main/resources/META-INF/LICENSE | 0
.../src/main/resources/META-INF/NOTICE | 2 +-
.../src/main/scripts/start_parser_topology.sh | 2 +-
.../apache/metron/parsers/bolt/ParserBoltTest.java | 2 +-
.../apache/metron/parsers/bolt/WriterBoltTest.java | 0
.../integration/StormParserIntegrationTest.java} | 33 ++-
.../components/ParserTopologyComponent.java | 0
.../integration/validation/StormParserDriver.java} | 50 ++--
.../parsers/topology/ParserTopologyCLITest.java | 0
...SimpleHbaseEnrichmentWriterIntegrationTest.java | 21 +-
.../StormEnvelopedParserIntegrationTest.java | 52 ++++
.../integration/WriterBoltIntegrationTest.java | 0
.../src/test/resources/log4j.properties | 0
metron-platform/metron-parsing/pom.xml | 35 +++
.../main/java/org/apache/metron/TestConstants.java | 3 +-
.../apache/metron/test/utils/SampleDataUtils.java | 8 +-
metron-platform/metron-writer/README.md | 2 +-
metron-platform/pom.xml | 4 +-
pom.xml | 4 +-
site-book/bin/generate-md.sh | 6 +-
.../metron-platform/metron-parsers/index.html | 2 +-
238 files changed, 1234 insertions(+), 1004 deletions(-)
diff --git a/README.md b/README.md
index 95b6faf..3390e55 100644
--- a/README.md
+++ b/README.md
@@ -132,7 +132,7 @@ component and Apache Kafka as the unified data bus.
Some high level links to the relevant subparts of the architecture, for
more information:
-* [Parsers](metron-platform/metron-parsers) : Parsing data from kafka into the Metron data model and passing it downstream to Enrichment.
+* [Parsers](metron-platform/metron-parsers-common) : Parsing data from kafka into the Metron data model and passing it downstream to Enrichment.
* [Enrichment](metron-platform/metron-enrichment) : Enriching data post-parsing and providing the ability to tag a message as an alert and assign a risk triage level via a custom rule language.
* [Indexing](metron-platform/metron-indexing) : Indexing the data post-enrichment into HDFS, Elasticsearch or Solr.
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 5462a12..17453f5 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -54,6 +54,7 @@ junit:junit:jar:4.12:compile,Eclipse Public License 1.0,http://junit.org
junit:junit:jar:4.4:compile,Common Public License Version 1.0,http://junit.org
net.razorvine:pyrolite:jar:4.13:compile,MIT,https://github.com/irmen/Pyrolite
net.sf.jopt-simple:jopt-simple:jar:3.2:compile,The MIT License,http://jopt-simple.sourceforge.net
+net.sf.jopt-simple:jopt-simple:jar:4.6:compile,The MIT License,http://jopt-simple.sourceforge.net
net.sf.jopt-simple:jopt-simple:jar:4.9:compile,The MIT License,http://jopt-simple.sourceforge.net
net.sf.saxon:Saxon-HE:jar:9.5.1-5:compile,Mozilla Public License Version 2.0,http://www.saxonica.com/
org.abego.treelayout:org.abego.treelayout.core:jar:1.0.1:compile,BSD 3-Clause "New" or "Revised" License (BSD-3-Clause),http://code.google.com/p/treelayout/
@@ -181,6 +182,7 @@ com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:2.9.5:compile,A
com.fasterxml.woodstox:woodstox-core:jar:5.0.3:compile,ASLv2,https://github.com/FasterXML/woodstox
com.fasterxml:classmate:jar:1.3.1:compile,ASLv2,http://github.com/cowtowncoder/java-classmate
com.fasterxml:classmate:jar:1.3.4:compile,ASLv2,http://github.com/cowtowncoder/java-classmate
+com.google.code.gson:gson:jar:2.2.2:compile,The Apache Software License, Version 2.0,http://code.google.com/p/google-gson/
com.google.code.gson:gson:jar:2.2.4:compile,The Apache Software License, Version 2.0,http://code.google.com/p/google-gson/
com.google.code.gson:gson:jar:2.7:compile,The Apache Software License, Version 2.0,http://code.google.com/p/google-gson/
com.google.guava:guava:jar:11.0.2:compile,ASLv2,
diff --git a/metron-contrib/metron-docker/README.md b/metron-contrib/metron-docker/README.md
index 35364d4..761cc6e 100644
--- a/metron-contrib/metron-docker/README.md
+++ b/metron-contrib/metron-docker/README.md
@@ -122,7 +122,7 @@ Examples
### Deploy a new parser class
-After adding a new parser to metron-parsers, build Metron from the top level directory:
+After adding a new parser to metron-parsers-common, build Metron from the top level directory:
```
$ cd $METRON_HOME
$ mvn clean install -DskipTests
diff --git a/metron-contrib/metron-docker/compose/storm/Dockerfile b/metron-contrib/metron-docker/compose/storm/Dockerfile
index 78aeba4..8c90971 100644
--- a/metron-contrib/metron-docker/compose/storm/Dockerfile
+++ b/metron-contrib/metron-docker/compose/storm/Dockerfile
@@ -27,7 +27,7 @@ ADD ./enrichment /enrichment
ADD ./indexing /indexing
ADD ./elasticsearch /elasticsearch
RUN mkdir -p $METRON_HOME
-RUN tar -xzf /parser/metron-parsers-$METRON_VERSION-archive.tar.gz -C /usr/metron/$METRON_VERSION/
+RUN tar -xzf /parser/metron-parsing-storm-$METRON_VERSION-archive.tar.gz -C /usr/metron/$METRON_VERSION/
RUN tar -xzf /enrichment/metron-enrichment-$METRON_VERSION-archive.tar.gz -C /usr/metron/$METRON_VERSION/
RUN sed -i -e "s/kafka.zk=.*:/kafka.zk=kafkazk:/g" /usr/metron/$METRON_VERSION/config/enrichment.properties
diff --git a/metron-contrib/metron-docker/install-metron.sh b/metron-contrib/metron-docker/install-metron.sh
index adc95a0..b2fcaad 100755
--- a/metron-contrib/metron-docker/install-metron.sh
+++ b/metron-contrib/metron-docker/install-metron.sh
@@ -29,7 +29,8 @@ mkdir -p $METRON_DOCKER_ROOT/storm/elasticsearch/
echo Installing HBase dependencies
cp $METRON_PLATFORM_ROOT/metron-data-management/target/metron-data-management-$METRON_VERSION-archive.tar.gz $METRON_DOCKER_ROOT/hbase/data-management
echo Installing Storm dependencies
-cp $METRON_PLATFORM_ROOT/metron-parsers/target/metron-parsers-$METRON_VERSION-archive.tar.gz $METRON_DOCKER_ROOT/storm/parser/
+cp $METRON_PLATFORM_ROOT/metron-parsers/metron-parsers-common/target/metron-parsers-common-$METRON_VERSION-archive.tar.gz $METRON_DOCKER_ROOT/storm/parser/
+cp $METRON_PLATFORM_ROOT/metron-parsers/metron-parsing-storm/target/metron-parsing-storm-$METRON_VERSION-archive.tar.gz $METRON_DOCKER_ROOT/storm/parser/
cp $METRON_PLATFORM_ROOT/metron-enrichment/target/metron-enrichment-$METRON_VERSION-archive.tar.gz $METRON_DOCKER_ROOT/storm/enrichment/
cp $METRON_PLATFORM_ROOT/metron-indexing/target/metron-indexing-$METRON_VERSION-archive.tar.gz $METRON_DOCKER_ROOT/storm/indexing/
echo Installing Elasticsearch dependencies
diff --git a/metron-contrib/metron-docker/pom.xml b/metron-contrib/metron-docker/pom.xml
index 47c4fc3..3191ebc 100644
--- a/metron-contrib/metron-docker/pom.xml
+++ b/metron-contrib/metron-docker/pom.xml
@@ -63,7 +63,7 @@
<outputDirectory>${project.basedir}/compose/kafkazk/packages</outputDirectory>
<resources>
<resource>
- <directory>${project.parent.parent.basedir}/metron-platform/metron-parsers/target/</directory>
+ <directory>${project.parent.parent.basedir}/metron-platform/metron-parsing/metron-parsers-common/target/</directory>
<includes>
<include>*.tar.gz</include>
</includes>
@@ -81,7 +81,7 @@
<outputDirectory>${project.basedir}/compose/kafkazk/packages</outputDirectory>
<resources>
<resource>
- <directory>${project.parent.parent.basedir}/metron-platform/metron-enrichment/target/</directory>
+ <directory>${project.parent.parent.basedir}/metron-platform/metron-parsing/metron-enrichment/target/</directory>
<includes>
<include>*.tar.gz</include>
</includes>
@@ -153,7 +153,7 @@
<outputDirectory>${project.basedir}/compose/storm/parser</outputDirectory>
<resources>
<resource>
- <directory>${project.parent.parent.basedir}/metron-platform/metron-parsers/target/</directory>
+ <directory>${project.parent.parent.basedir}/metron-platform/metron-parsing/metron-parsers-common/target/</directory>
<includes>
<include>*.tar.gz</include>
</includes>
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index 97b5749..9516014 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -441,9 +441,15 @@
<name>metron-management</name>
</package>
<package>
+ <name>metron-parsers-common</name>
+ </package>
+ <package>
<name>metron-parsers</name>
</package>
<package>
+ <name>metron-parsing-storm</name>
+ </package>
+ <package>
<name>metron-enrichment</name>
</package>
<package>
diff --git a/metron-deployment/packaging/docker/deb-docker/pom.xml b/metron-deployment/packaging/docker/deb-docker/pom.xml
index 369ffee..19e9068 100644
--- a/metron-deployment/packaging/docker/deb-docker/pom.xml
+++ b/metron-deployment/packaging/docker/deb-docker/pom.xml
@@ -108,7 +108,19 @@
</includes>
</resource>
<resource>
- <directory>${metron_dir}/metron-platform/metron-parsers/target/</directory>
+ <directory>${metron_dir}/metron-platform/metron-parsing/metron-parsers-common/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-parsing/metron-parsers/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-parsing/metron-parsing-storm/target/</directory>
<includes>
<include>*.tar.gz</include>
</includes>
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 0d829d1..a16a863 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -44,7 +44,7 @@ License: ASL 2.0
Group: Applications/Internet
URL: %{url}
Source0: metron-common-%{full_version}-archive.tar.gz
-Source1: metron-parsers-%{full_version}-archive.tar.gz
+Source1: metron-parsers-common-%{full_version}-archive.tar.gz
Source2: metron-elasticsearch-%{full_version}-archive.tar.gz
Source3: metron-data-management-%{full_version}-archive.tar.gz
Source4: metron-solr-%{full_version}-archive.tar.gz
@@ -60,6 +60,8 @@ Source13: metron-alerts-%{full_version}-archive.tar.gz
Source14: metron-performance-%{full_version}-archive.tar.gz
Source15: metron-profiler-spark-%{full_version}-archive.tar.gz
Source16: metron-profiler-repl-%{full_version}-archive.tar.gz
+Source17: metron-parsing-storm-%{full_version}-archive.tar.gz
+Source18: metron-parsers-%{full_version}-archive.tar.gz
%description
Apache Metron provides a scalable advanced security analytics framework
@@ -99,6 +101,8 @@ tar -xzf %{SOURCE13} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE14} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE15} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE16} -C %{buildroot}%{metron_home}
+tar -xzf %{SOURCE17} -C %{buildroot}%{metron_home}
+tar -xzf %{SOURCE18} -C %{buildroot}%{metron_home}
install %{buildroot}%{metron_home}/bin/metron-management-ui %{buildroot}/etc/init.d/
install %{buildroot}%{metron_home}/bin/metron-alerts-ui %{buildroot}/etc/init.d/
@@ -133,29 +137,49 @@ This package installs the Metron common files %{metron_home}
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+%package parsers-common
+Summary: Metron Parser Common Files
+Group: Applications/Internet
+Provides: parsers-common = %{version}
+
+%description parsers-common
+This package installs the Metron Parser Common files
+
+%files parsers-common
+%defattr(-,root,root,755)
+%dir %{metron_root}
+%dir %{metron_home}
+%dir %{metron_home}/config
+%dir %{metron_home}/config/zookeeper
+%dir %{metron_home}/config/zookeeper/parsers
+%dir %{metron_home}/patterns
+%dir %{metron_home}/lib
+%{metron_home}/config/zookeeper/parsers/jsonMap.json
+%{metron_home}/config/zookeeper/parsers/jsonMapQuery.json
+%{metron_home}/config/zookeeper/parsers/jsonMapWrappedQuery.json
+%{metron_home}/patterns/common
+%attr(0644,root,root) %{metron_home}/lib/metron-parsers-common-%{full_version}-uber.jar
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
%package parsers
-Summary: Metron Parser Files
+Summary: Metron Bundled Parser Files
Group: Applications/Internet
Provides: parsers = %{version}
%description parsers
-This package installs the Metron Parser files
+This package installs the Metron Bundled Parser files
%files parsers
%defattr(-,root,root,755)
%dir %{metron_root}
%dir %{metron_home}
-%dir %{metron_home}/bin
%dir %{metron_home}/config
%dir %{metron_home}/config/zookeeper
%dir %{metron_home}/config/zookeeper/parsers
%dir %{metron_home}/patterns
%dir %{metron_home}/lib
-%{metron_home}/bin/start_parser_topology.sh
%{metron_home}/config/zookeeper/parsers/bro.json
-%{metron_home}/config/zookeeper/parsers/jsonMap.json
-%{metron_home}/config/zookeeper/parsers/jsonMapQuery.json
-%{metron_home}/config/zookeeper/parsers/jsonMapWrappedQuery.json
%{metron_home}/config/zookeeper/parsers/snort.json
%{metron_home}/config/zookeeper/parsers/squid.json
%{metron_home}/config/zookeeper/parsers/syslog5424.json
@@ -163,7 +187,6 @@ This package installs the Metron Parser files
%{metron_home}/config/zookeeper/parsers/yaf.json
%{metron_home}/config/zookeeper/parsers/asa.json
%{metron_home}/patterns/asa
-%{metron_home}/patterns/common
%{metron_home}/patterns/fireeye
%{metron_home}/patterns/sourcefire
%{metron_home}/patterns/squid
@@ -173,6 +196,25 @@ This package installs the Metron Parser files
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+%package parsing-storm
+Summary: Metron Parser Storm Files
+Group: Applications/Internet
+Provides: parsing-storm = %{version}
+
+%description parsing-storm
+This package installs the Metron Parser Storm files
+
+%files parsing-storm
+%defattr(-,root,root,755)
+%dir %{metron_root}
+%dir %{metron_home}
+%dir %{metron_home}/bin
+%dir %{metron_home}/lib
+%{metron_home}/bin/start_parser_topology.sh
+%attr(0644,root,root) %{metron_home}/lib/metron-parsing-storm-%{full_version}-uber.jar
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
%package elasticsearch
Summary: Metron Elasticsearch Files
Group: Applications/Internet
@@ -591,6 +633,8 @@ chkconfig --del metron-management-ui
chkconfig --del metron-alerts-ui
%changelog
+* Thu Nov 15 2018 Apache Metron <de...@metron.apache.org> - 0.7.0
+- Split metron-parsers into metron-parsing and submodules
* Wed Oct 31 2018 Apache Metron <de...@metron.apache.org> - 0.7.0
- Update files in Management UI from Angular upgrade
* Thu Aug 30 2018 Apache Metron <de...@metron.apache.org> - 0.7.0
diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml
index db5a0e6..e876652 100644
--- a/metron-deployment/packaging/docker/rpm-docker/pom.xml
+++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml
@@ -144,7 +144,19 @@
</includes>
</resource>
<resource>
- <directory>${metron_dir}/metron-platform/metron-parsers/target/</directory>
+ <directory>${metron_dir}/metron-platform/metron-parsing/metron-parsers/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-parsing/metron-parsers-common/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-parsing/metron-parsing-storm/target/</directory>
<includes>
<include>*.tar.gz</include>
</includes>
diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml
index 0d98b9f..67a6820 100644
--- a/metron-interface/metron-rest/pom.xml
+++ b/metron-interface/metron-rest/pom.xml
@@ -237,7 +237,7 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-parsers</artifactId>
+ <artifactId>metron-parsers-common</artifactId>
<version>${project.parent.version}</version>
<scope>provided</scope>
<exclusions>
@@ -267,6 +267,38 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parsers</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-profiler-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-statistics</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-writer</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/GrokConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/GrokConfig.java
index 1166e9c..6b0bc1b 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/GrokConfig.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/GrokConfig.java
@@ -30,7 +30,8 @@ public class GrokConfig {
@Bean
public Grok commonGrok() throws GrokException {
Grok grok = new Grok();
- grok.addPatternFromReader(new InputStreamReader(getClass().getResourceAsStream("/patterns/common")));
+ grok.addPatternFromReader(new InputStreamReader(getClass().getResourceAsStream(
+ "/patterns/common")));
return grok;
}
}
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
index 4a39635..11ab664 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
@@ -71,7 +71,8 @@ public class GrokServiceImpl implements GrokService {
throw new RestException("Grok statement is required");
}
Grok grok = new Grok();
- grok.addPatternFromReader(new InputStreamReader(getClass().getResourceAsStream("/patterns/common")));
+ grok.addPatternFromReader(new InputStreamReader(getClass().getResourceAsStream(
+ "/patterns/common")));
grok.addPatternFromReader(new StringReader(grokValidation.getStatement()));
String grokPattern = "%{" + grokValidation.getPatternLabel() + "}";
grok.compile(grokPattern);
diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
index b454b8e..6f07925 100644
--- a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
+++ b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
@@ -38,7 +38,8 @@ METRON_PID_FILE="${METRON_PID_FILE:-/var/run/metron/metron-rest.pid}"
PARSER_CONTRIB=${PARSER_CONTRIB:-$METRON_HOME/parser_contrib}
INDEXING_CONTRIB=${INDEXING_CONTRIB:-$METRON_HOME/indexing_contrib}
-PARSER_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar)
+METRON_PARSERS_PLATFORM="${METRON_PARSERS_PLATFORM:-storm}"
+PARSER_LIB=$(find $METRON_HOME/lib/ -name metron-parsing-${METRON_PARSERS_PLATFORM}*.jar)
echo "METRON_VERSION=${METRON_VERSION}"
echo "METRON_HOME=${METRON_HOME}"
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
index 8888eb0..b2f98fb 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
@@ -131,7 +131,7 @@ public class GrokControllerIntegrationTest {
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
.andExpect(jsonPath("$").isNotEmpty());
- String statement = FileUtils.readFileToString(new File("../../metron-platform/metron-parsers/src/main/resources/patterns/squid"));
+ String statement = FileUtils.readFileToString(new File("../../metron-platform/metron-parsing/metron-parsers/src/main/resources/patterns/squid"));
this.mockMvc.perform(get(grokUrl + "/get/statement?path=/patterns/squid").with(httpBasic(user,password)))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("text/plain;charset=UTF-8")))
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
index 8f7f40c..f308ba3 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
@@ -215,7 +215,8 @@ public class GrokServiceImplTest {
@Test
public void getStatementFromClasspathShouldReturnStatement() throws Exception {
- String expected = FileUtils.readFileToString(new File("../../metron-platform/metron-parsers/src/main/resources/patterns/squid"));
+ String expected = FileUtils.readFileToString(new File(
+ "../../metron-platform/metron-parsing/metron-parsers/src/main/resources/patterns/squid"));
assertEquals(expected, grokService.getStatementFromClasspath("/patterns/squid"));
}
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
index e6163c0..3e54e07 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
@@ -28,6 +28,7 @@ import org.apache.curator.framework.api.GetDataBuilder;
import org.apache.curator.framework.api.SetDataBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.metron.common.configuration.ConfigurationType;
+import org.apache.metron.common.configuration.ConfigurationType;
import org.apache.metron.common.configuration.ParserConfigurations;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
diff --git a/metron-platform/Performance-tuning-guide.md b/metron-platform/Performance-tuning-guide.md
index cb69cb8..ff83c80 100644
--- a/metron-platform/Performance-tuning-guide.md
+++ b/metron-platform/Performance-tuning-guide.md
@@ -65,7 +65,7 @@ one consumer in a given consumer group were able to read from that partition.
If you are using stellar field transformations in your sensors, by default, stellar expressions
are not cached. Sensors that use stellar field transformations by see a performance
boost by turning on caching via setting the `cacheConfig`
-[property](metron-parsers#parser_configuration).
+[property](metron-parsers-common#parser_configuration).
This is beneficial if your transformations:
* Are complex (e.g. `ENRICHMENT_GET` calls or other high latency calls)
@@ -170,7 +170,7 @@ using a Storm Flux file, a configuration properties file, and Ambari. Here is a
This is a mapping of the various performance tuning properties for parsers and how they are materialized.
-See more detail on starting parsers [here](https://github.com/apache/metron/blob/master/metron-platform/metron-parsers/README.md#starting-the-parser-topology)
+See more detail on starting parsers [here](https://github.com/apache/metron/blob/master/metron-platform/metron-parsing/metron-parsers-common/README.md#starting-the-parser-topology)
| Category | Management UI Property Name | JSON Config File Property Name | CLI Option | Storm Property Name | Notes |
|-----------------------------|--------------------------------------------|------------------------------------|------------------------------------------------------------------------------------------------|---------------------------------|-------------------------------------------------------------------------------|
@@ -427,7 +427,7 @@ though you could certainly do so if necessary. Notice that we only needed 1 work
-z $ZOOKEEPER \
```
-From the usage docs, here are the options we've used. The full reference can be found [here](../metron-platform/metron-parsers/README.md#Starting_the_Parser_Topology).
+From the usage docs, here are the options we've used. The full reference can be found [here](../metron-platform/metron-parsing/metron-parsers-common/README.md#Starting_the_Parser_Topology).
```
usage: start_parser_topology.sh
diff --git a/metron-platform/metron-common/README.md b/metron-platform/metron-common/README.md
index d68259a..20f0eef 100644
--- a/metron-platform/metron-common/README.md
+++ b/metron-platform/metron-common/README.md
@@ -79,30 +79,30 @@ This configuration is stored in zookeeper, but looks something like
Various parts of our stack uses the global config are documented throughout the Metron documentation,
but a convenient index is provided here:
-| Property Name | Subsystem | Type | Ambari Property |
-|---------------------------------------------------------------------------------------------------------------------|---------------|------------|------------------------------|
-| [`es.clustername`](../metron-elasticsearch#esclustername) | Indexing | String | `es_cluster_name` |
-| [`es.ip`](../metron-elasticsearch#esip) | Indexing | String | `es_hosts` |
-| [`es.port`](../metron-elasticsearch#esport) | Indexing | String | `es_port` |
-| [`es.date.format`](../metron-elasticsearch#esdateformat) | Indexing | String | `es_date_format` |
-| [`es.client.settings`](../metron-elasticsearch#esclientsettings) | Indexing | Object | N/A |
-| [`fieldValidations`](#validation-framework) | Parsing | Object | N/A |
-| [`parser.error.topic`](../metron-parsers#parsererrortopic) | Parsing | String | N/A |
-| [`stellar.function.paths`](../../metron-stellar/stellar-common#stellarfunctionpaths) | Stellar | CSV String | N/A |
-| [`stellar.function.resolver.includes`](../../metron-stellar/stellar-common#stellarfunctionresolverincludesexcludes) | Stellar | CSV String | N/A |
-| [`stellar.function.resolver.excludes`](../../metron-stellar/stellar-common#stellarfunctionresolverincludesexcludes) | Stellar | CSV String | N/A |
-| [`profiler.period.duration`](../../metron-analytics/metron-profiler#profilerperiodduration) | Profiler | Integer | `profiler_period_duration` |
-| [`profiler.period.duration.units`](../../metron-analytics/metron-profiler#profilerperioddurationunits) | Profiler | String | `profiler_period_units` |
-| [`profiler.writer.batchSize`](../../metron-analytics/metron-profiler/#profilerwriterbatchsize) | Profiler | Integer | N/A |
-| [`profiler.writer.batchTimeout`](../../metron-analytics/metron-profiler/#profilerwriterbatchtimeout) | Profiler | Integer | N/A |
-| [`update.hbase.table`](../metron-indexing#updatehbasetable) | REST/Indexing | String | `update_hbase_table` |
-| [`update.hbase.cf`](../metron-indexing#updatehbasecf) | REST/Indexing | String | `update_hbase_cf` |
-| [`geo.hdfs.file`](../metron-enrichment#geohdfsfile) | Enrichment | String | `geo_hdfs_file` |
-| [`enrichment.writer.batchSize`](../metron-enrichment#enrichmentwriterbatchsize) | Enrichment | Integer | N/A |
-| [`enrichment.writer.batchTimeout`](../metron-enrichment#enrichmentwriterbatchtimeout) | Enrichment | Integer | N/A |
-| [`geo.hdfs.file`](../metron-enrichment#geohdfsfile) | Enrichment | String | `geo_hdfs_file` |
-| [`source.type.field`](../../metron-interface/metron-alerts#sourcetypefield) | UI | String | `source_type_field` |
-| [`threat.triage.score.field`](../../metron-interface/metron-alerts#threattriagescorefield) | UI | String | `threat_triage_score_field` |
+| Property Name | Subsystem | Type | Ambari Property |
+|---------------------------------------------------------------------------------------------------------------------|---------------|------------|----------------------------|
+| [`es.clustername`](../metron-elasticsearch#esclustername) | Indexing | String | `es_cluster_name` |
+| [`es.ip`](../metron-elasticsearch#esip) | Indexing | String | `es_hosts` |
+| [`es.port`](../metron-elasticsearch#esport) | Indexing | String | `es_port` |
+| [`es.date.format`](../metron-elasticsearch#esdateformat) | Indexing | String | `es_date_format` |
+| [`es.client.settings`](../metron-elasticsearch#esclientsettings) | Indexing | Object | N/A |
+| [`fieldValidations`](#validation-framework) | Parsing | Object | N/A |
+| [`parser.error.topic`](../metron-parsers#parsererrortopic) | Parsing | String | N/A |
+| [`stellar.function.paths`](../../metron-stellar/stellar-common#stellarfunctionpaths) | Stellar | CSV String | N/A |
+| [`stellar.function.resolver.includes`](../../metron-stellar/stellar-common#stellarfunctionresolverincludesexcludes) | Stellar | CSV String | N/A |
+| [`stellar.function.resolver.excludes`](../../metron-stellar/stellar-common#stellarfunctionresolverincludesexcludes) | Stellar | CSV String | N/A |
+| [`profiler.period.duration`](../../metron-analytics/metron-profiler#profilerperiodduration) | Profiler | Integer | `profiler_period_duration` |
+| [`profiler.period.duration.units`](../../metron-analytics/metron-profiler#profilerperioddurationunits) | Profiler | String | `profiler_period_units` |
+| [`profiler.writer.batchSize`](../../metron-analytics/metron-profiler/#profilerwriterbatchsize) | Profiler | Integer | N/A |
+| [`profiler.writer.batchTimeout`](../../metron-analytics/metron-profiler/#profilerwriterbatchtimeout) | Profiler | Integer | N/A |
+| [`update.hbase.table`](../metron-indexing#updatehbasetable) | REST/Indexing | String | `update_hbase_table` |
+| [`update.hbase.cf`](../metron-indexing#updatehbasecf) | REST/Indexing | String | `update_hbase_cf` |
+| [`geo.hdfs.file`](../metron-enrichment#geohdfsfile) | Enrichment | String | `geo_hdfs_file` |
+| [`enrichment.writer.batchSize`](../metron-enrichment#enrichmentwriterbatchsize) | Enrichment | Integer | N/A |
+| [`enrichment.writer.batchTimeout`](../metron-enrichment#enrichmentwriterbatchtimeout) | Enrichment | Integer | N/A |
+| [`geo.hdfs.file`](../metron-enrichment#geohdfsfile) | Enrichment | String | `geo_hdfs_file` |
+| [`source.type.field`](../../metron-interface/metron-alerts#sourcetypefield) | UI | String | `source_type_field` |
+| [`threat.triage.score.field`](../../metron-interface/metron-alerts#threattriagescorefield) | UI | String | `threat_triage_score_field` |
## Note Configs in Ambari
If a field is managed via ambari, you should change the field via
@@ -440,4 +440,3 @@ Options:
-p DIRECTORY, --hdp_home=DIRECTORY
HDP home directory
```
-`
diff --git a/metron-platform/metron-common/src/main/scripts/stellar b/metron-platform/metron-common/src/main/scripts/stellar
index 690a9f0..a538e13 100644
--- a/metron-platform/metron-common/src/main/scripts/stellar
+++ b/metron-platform/metron-common/src/main/scripts/stellar
@@ -36,8 +36,9 @@ fi
# treat unset vars as an error; METRON_HOME
set -u
+export METRON_PARSERS_PLATFORM="${METRON_PARSERS_PLATFORM:-storm}"
export HBASE_CONFIGS=$(hbase classpath)
-export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar)
+export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers-${METRON_PARSERS_PLATFORM}*.jar)
export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar)
export PROFILER_LIB=$(find $METRON_HOME/lib/ -name metron-profiler-repl*.jar)
java $METRON_JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$PROFILER_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@"
diff --git a/metron-platform/metron-common/src/main/scripts/zk_load_configs.sh b/metron-platform/metron-common/src/main/scripts/zk_load_configs.sh
index 2380d13..b22bb4b 100755
--- a/metron-platform/metron-common/src/main/scripts/zk_load_configs.sh
+++ b/metron-platform/metron-common/src/main/scripts/zk_load_configs.sh
@@ -29,7 +29,8 @@ fi
export METRON_VERSION=${project.version}
export METRON_HOME=/usr/metron/$METRON_VERSION
export CLASSNAME="org.apache.metron.common.cli.ConfigurationManager"
-export JAR=metron-parsers-$METRON_VERSION-uber.jar
+export JAR=metron-parsers-common-$METRON_VERSION-uber.jar
+# TODO export the Storm jar?
export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
CP=$METRON_HOME/lib/$JAR:${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath`
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsUtilsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsUtilsTest.java
index 8f8cb54..9379285 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsUtilsTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsUtilsTest.java
@@ -32,6 +32,7 @@ import org.apache.curator.test.TestingServer;
import org.apache.metron.TestConstants;
import org.apache.metron.common.utils.JSONUtils;
import org.junit.After;
+
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --git a/metron-platform/metron-management/pom.xml b/metron-platform/metron-management/pom.xml
index 844439d..85bbc23 100644
--- a/metron-platform/metron-management/pom.xml
+++ b/metron-platform/metron-management/pom.xml
@@ -52,6 +52,18 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
+ <artifactId>metron-parsers-common</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-writer</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
<artifactId>stellar-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SettingsLoader.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SettingsLoader.java
deleted file mode 100644
index 1c64c16..0000000
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SettingsLoader.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.parsers;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.commons.configuration.XMLConfiguration;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
-import org.json.simple.parser.ParseException;
-
-public class SettingsLoader {
-
- @SuppressWarnings("unchecked")
- public static JSONObject loadEnvironmentIdnetifier(String config_path)
- throws ConfigurationException {
- Configuration config = new PropertiesConfiguration(config_path);
-
- String customer = config.getString("customer.id", "unknown");
- String datacenter = config.getString("datacenter.id", "unknown");
- String instance = config.getString("instance.id", "unknown");
-
- JSONObject identifier = new JSONObject();
- identifier.put("customer", customer);
- identifier.put("datacenter", datacenter);
- identifier.put("instance", instance);
-
- return identifier;
- }
-
- @SuppressWarnings("unchecked")
- public static JSONObject loadTopologyIdnetifier(String config_path)
- throws ConfigurationException {
- Configuration config = new PropertiesConfiguration(config_path);
-
- String topology = config.getString("topology.id", "unknown");
- String instance = config.getString("instance.id", "unknown");
-
- JSONObject identifier = new JSONObject();
- identifier.put("topology", topology);
- identifier.put("topology_instance", instance);
-
- return identifier;
- }
-
-
- public static String generateTopologyName(JSONObject env, JSONObject topo) {
-
- return (env.get("customer") + "_" + env.get("datacenter") + "_"
- + env.get("instance") + "_" + topo.get("topology") + "_" + topo.get("topology_instance"));
- }
-
- @SuppressWarnings("unchecked")
- public static JSONObject generateAlertsIdentifier(JSONObject env, JSONObject topo)
- {
- JSONObject identifier = new JSONObject();
- identifier.put("environment", env);
- identifier.put("topology", topo);
-
- return identifier;
- }
-
- public static Map<String, JSONObject> loadRegexAlerts(String config_path)
- throws ConfigurationException, ParseException {
- XMLConfiguration alert_rules = new XMLConfiguration();
- alert_rules.setDelimiterParsingDisabled(true);
- alert_rules.load(config_path);
-
- //int number_of_rules = alert_rules.getList("rule.pattern").size();
-
- String[] patterns = alert_rules.getStringArray("rule.pattern");
- String[] alerts = alert_rules.getStringArray("rule.alert");
-
- JSONParser pr = new JSONParser();
- Map<String, JSONObject> rules = new HashMap<String, JSONObject>();
-
- for (int i = 0; i < patterns.length; i++)
- rules.put(patterns[i], (JSONObject) pr.parse(alerts[i]));
-
- return rules;
- }
-
- public static Map<String, JSONObject> loadKnownHosts(String config_path)
- throws ConfigurationException, ParseException {
- Configuration hosts = new PropertiesConfiguration(config_path);
-
- Iterator<String> keys = hosts.getKeys();
- Map<String, JSONObject> known_hosts = new HashMap<String, JSONObject>();
- JSONParser parser = new JSONParser();
-
- while (keys.hasNext()) {
- String key = keys.next().trim();
- JSONArray value = (JSONArray) parser.parse(hosts.getProperty(key)
- .toString());
- known_hosts.put(key, (JSONObject) value.get(0));
- }
-
- return known_hosts;
- }
-
- public static void printConfigOptions(PropertiesConfiguration config, String path_fragment)
- {
- Iterator<String> itr = config.getKeys();
-
- while(itr.hasNext())
- {
- String key = itr.next();
-
- if(key.contains(path_fragment))
- {
-
- System.out.println("[Metron] Key: " + key + " -> " + config.getString(key));
- }
- }
-
- }
-
- public static void printOptionalSettings(Map<String, String> settings)
- {
- for(String setting: settings.keySet())
- {
- System.out.println("[Metron] Optional Setting: " + setting + " -> " +settings.get(setting));
- }
-
- }
-
- public static Map<String, String> getConfigOptions(PropertiesConfiguration config, String path_fragment)
- {
- Iterator<String> itr = config.getKeys();
- Map<String, String> settings = new HashMap<String, String>();
-
- while(itr.hasNext())
- {
- String key = itr.next();
-
- if(key.contains(path_fragment))
- {
- String tmp_key = key.replace(path_fragment, "");
- settings.put(tmp_key, config.getString(key));
- }
- }
-
- return settings;
- }
-}
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/AsaIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/AsaIntegrationTest.java
deleted file mode 100644
index 3ff07ad..0000000
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/AsaIntegrationTest.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.parsers.integration;
-
-import org.apache.metron.parsers.integration.validation.SampleDataValidation;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class AsaIntegrationTest extends ParserIntegrationTest {
- @Override
- String getSensorType() {
- return "asa";
- }
-
- @Override
- List<ParserValidation> getValidations() {
- return new ArrayList<ParserValidation>() {{
- add(new SampleDataValidation());
- }};
- }
-}
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/BroIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/BroIntegrationTest.java
deleted file mode 100644
index c9cfe0a..0000000
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/BroIntegrationTest.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.parsers.integration;
-
-import org.apache.metron.parsers.integration.validation.SampleDataValidation;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class BroIntegrationTest extends ParserIntegrationTest {
- @Override
- String getSensorType() {
- return "bro";
- }
-
- @Override
- List<ParserValidation> getValidations() {
- return new ArrayList<ParserValidation>() {{
- add(new SampleDataValidation());
- }};
- }
-}
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapIntegrationTest.java
deleted file mode 100644
index 13a7835..0000000
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapIntegrationTest.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.parsers.integration;
-
-import org.apache.metron.parsers.integration.validation.SampleDataValidation;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class JSONMapIntegrationTest extends ParserIntegrationTest {
- @Override
- String getSensorType() {
- return "jsonMap";
- }
-
- @Override
- List<ParserValidation> getValidations() {
- return new ArrayList<ParserValidation>() {{
- add(new SampleDataValidation());
- }};
- }
-}
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapQueryIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapQueryIntegrationTest.java
deleted file mode 100644
index cec6242..0000000
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapQueryIntegrationTest.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.parsers.integration;
-
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.metron.parsers.integration.validation.SampleDataValidation;
-
-public class JSONMapQueryIntegrationTest extends ParserIntegrationTest {
- @Override
- String getSensorType() {
- return "jsonMapQuery";
- }
-
- @Override
- List<ParserValidation> getValidations() {
- return new ArrayList<ParserValidation>() {{
- add(new SampleDataValidation());
- }};
- }
-}
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/SnortIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/SnortIntegrationTest.java
deleted file mode 100644
index 244dd89..0000000
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/SnortIntegrationTest.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.parsers.integration;
-
-import org.apache.metron.parsers.integration.validation.SampleDataValidation;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class SnortIntegrationTest extends ParserIntegrationTest {
- @Override
- String getSensorType() {
- return "snort";
- }
-
- @Override
- List<ParserValidation> getValidations() {
- return new ArrayList<ParserValidation>() {{
- add(new SampleDataValidation());
- }};
- }
-}
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/SquidIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/SquidIntegrationTest.java
deleted file mode 100644
index 9c03e81..0000000
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/SquidIntegrationTest.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.parsers.integration;
-
-import org.apache.metron.parsers.integration.validation.SampleDataValidation;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class SquidIntegrationTest extends ParserIntegrationTest {
- @Override
- String getSensorType() {
- return "squid";
- }
-
- @Override
- List<ParserValidation> getValidations() {
- return new ArrayList<ParserValidation>() {{
- add(new SampleDataValidation());
- }};
- }
-}
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/Syslog5424ParserIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/Syslog5424ParserIntegrationTest.java
deleted file mode 100644
index c9c47ce..0000000
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/Syslog5424ParserIntegrationTest.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.parsers.integration;
-
-import org.apache.metron.parsers.integration.validation.SampleDataValidation;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class Syslog5424ParserIntegrationTest extends ParserIntegrationTest {
- @Override
- String getSensorType() {
- return "syslog5424";
- }
-
- @Override
- List<ParserValidation> getValidations() {
- return new ArrayList<ParserValidation>() {{
- add(new SampleDataValidation());
- }};
- }
-}
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/WebSphereIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/WebSphereIntegrationTest.java
deleted file mode 100644
index 5dc0ac6..0000000
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/WebSphereIntegrationTest.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.parsers.integration;
-
-import org.apache.metron.parsers.integration.validation.SampleDataValidation;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class WebSphereIntegrationTest extends ParserIntegrationTest {
-
- @Override
- public String getSensorType() {
- return "websphere";
- }
-
- @Override
- List<ParserValidation> getValidations() {
- return new ArrayList<ParserValidation>() {{
- add(new SampleDataValidation());
- }};
- }
-
-}
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/YafIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/YafIntegrationTest.java
deleted file mode 100644
index 9930c2c..0000000
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/YafIntegrationTest.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.parsers.integration;
-
-import org.apache.metron.parsers.integration.validation.SampleDataValidation;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class YafIntegrationTest extends ParserIntegrationTest {
- @Override
- String getSensorType() {
- return "yaf";
- }
-
- @Override
- List<ParserValidation> getValidations() {
- return new ArrayList<ParserValidation>() {{
- add(new SampleDataValidation());
- }};
- }
-}
diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsing/README.md
similarity index 76%
rename from metron-platform/metron-parsers/README.md
rename to metron-platform/metron-parsing/README.md
index 5aff84a..76b6168 100644
--- a/metron-platform/metron-parsers/README.md
+++ b/metron-platform/metron-parsing/README.md
@@ -151,7 +151,7 @@ messages or marking messages as invalid.
### Invalidation Errors
There are two reasons a message will be marked as invalid:
-* Fail [global validation](../metron-common#validation-framework)
+* Fail [global validation](../../metron-common#validation-framework)
* Fail the parser's validate function (generally that means to not have a `timestamp` field or a `original_string` field.
Those messages which are marked as invalid are sent to the error queue
@@ -174,19 +174,12 @@ through.
-Data flows through the parser bolt via kafka and into the `enrichments`
+Data flows through the parser via kafka and into the `enrichments`
topology in kafka. Errors are collected with the context of the error
(e.g. stacktrace) and original message causing the error and sent to an
`error` queue. Invalid messages as determined by global validation
functions are also treated as errors and sent to an `error` queue.
-Multiple sensors can be aggregated into a single Storm topology. When this is done, there will be
-multiple Kafka spouts, but only a single parser bolt which will handle delegating to the correct
-parser as needed. There are some constraints around this, in particular regarding some configuration.
-Additionally, all sensors must flow to the same error topic. The Kafka topic is retrieved from the input Tuple itself.
-
-A worked example of this can be found in the [Parser Chaining use case](../../use-cases/parser_chaining/README.md#aggregated-parsers-with-parser-chaining).
-
## Message Format
All Metron messages follow a specific format in order to ingest a message. If a message does not conform to this format it will be dropped and put onto an error queue for further examination. The message must be of a JSON format and must have a JSON tag message like so:
@@ -274,19 +267,9 @@ then it is assumed to be a regex and will match any topic matching the pattern (
* `batchTimeout` : The timeout after which a batch will be flushed even if batchSize has not been met. Optional.
If unspecified, or set to `0`, it defaults to a system-determined duration which is a fraction of the Storm
parameter `topology.message.timeout.secs`. Ignored if batchSize is `1`, since this disables batching.
- * The kafka writer can be configured within the parser config as well. (This is all configured a priori, but this is convenient for overriding the settings). See [here](../metron-writer/README.md#kafka-writer)
+ * The kafka writer can be configured within the parser config as well. (This is all configured a priori, but this is convenient for overriding the settings). See [here](../../metron-writer/README.md#kafka-writer)
* `fieldTransformations` : An array of complex objects representing the transformations to be done on the message generated from the parser before writing out to the kafka topic.
-* `spoutParallelism` : The kafka spout parallelism (default to `1`). This can be overridden on the command line, and if there are multiple sensors should be in a comma separated list in the same order as the sensors.
-* `spoutNumTasks` : The number of tasks for the spout (default to `1`). This can be overridden on the command line, and if there are multiple sensors should be in a comma separated list in the same order as the sensors.
-* `parserParallelism` : The parser bolt parallelism (default to `1`). If there are multiple sensors, the last one's configuration will be used. This can be overridden on the command line.
-* `parserNumTasks` : The number of tasks for the parser bolt (default to `1`). If there are multiple sensors, the last one's configuration will be used. This can be overridden on the command line.
-* `errorWriterParallelism` : The error writer bolt parallelism (default to `1`). This can be overridden on the command line.
-* `errorWriterNumTasks` : The number of tasks for the error writer bolt (default to `1`). This can be overridden on the command line.
-* `numWorkers` : The number of workers to use in the topology (default is the storm default of `1`).
-* `numAckers` : The number of acker executors to use in the topology (default is the storm default of `1`).
-* `spoutConfig` : A map representing a custom spout config (this is a map). If there are multiple sensors, the configs will be merged with the last specified taking precedence. This can be overridden on the command line.
* `securityProtocol` : The security protocol to use for reading from kafka (this is a string). This can be overridden on the command line and also specified in the spout config via the `security.protocol` key. If both are specified, then they are merged and the CLI will take precedence. If multiple sensors are used, any non "PLAINTEXT" value will be used.
-* `stormConfig` : The storm config to use (this is a map). This can be overridden on the command line. If both are specified, they are merged with CLI properties taking precedence.
* `cacheConfig` : Cache config for stellar field transformations. This configures a least frequently used cache. This is a map with the following keys. If not explicitly configured (the default), then no cache will be used.
* `stellar.cache.maxSize` - The maximum number of elements in the cache. Default is to not use a cache.
* `stellar.cache.maxTimeRetain` - The maximum amount of time an element is kept in the cache (in minutes). Default is to not use a cache.
@@ -307,6 +290,9 @@ transformation which can be done to a message. This transformation can
* Add new fields given the values of existing fields of a message
* Remove existing fields of a message
+For platform specific configs, see the README of the appropriate project. This would include settings such as parallelism of individual components and general configuration.
+* [Storm](metron-parsing-storm/README.md#parser-configuration)
+
### Metadata
Metadata is a useful thing to send to Metron and use during enrichment or threat intelligence.
@@ -441,7 +427,7 @@ This transformation would transform `{ "protocol" : 6, "source.type" : "bro", ..
into `{ "protocol" : "TCP", "source.type" : "bro", ...}`
* `STELLAR` : This transformation executes a set of transformations
- expressed as [Stellar Language](../metron-common) statements.
+ expressed as [Stellar Language](../../metron-common) statements.
* `RENAME` : This transformation allows users to rename a set of fields. Specifically,
the config is presumed to be the mapping. The keys to the config are the existing field names
@@ -598,7 +584,7 @@ are defined in the Parser Config (defined above) JSON file in Zookeeper.
### Java Parser Adapters
Java parser adapters are intended for higher-velocity topologies and are not easily changed or extended. As the adoption of Metron continues we plan on extending our library of Java adapters to process more log formats. As of this moment the Java adapters included with Metron are:
-* org.apache.metron.parsers.ise.BasicIseParser : Parse ISE messages
+* BasicIseParser : Parse ISE messages
* org.apache.metron.parsers.bro.BasicBroParser : Parse Bro messages
* org.apache.metron.parsers.sourcefire.BasicSourcefireParser : Parse Sourcefire messages
* org.apache.metron.parsers.lancope.BasicLancopeParser : Parse Lancope messages
@@ -617,92 +603,17 @@ For more information on the Grok project please refer to the following link:
https://github.com/thekrakken/java-grok
-# Starting the Parser Topology
-
-Starting a particular parser topology on a running Metron deployment is
-as easy as running the `start_parser_topology.sh` script located in
-`$METRON_HOME/bin`. This utility will allow you to configure and start
-the running topology assuming that the sensor specific parser configuration
-exists within zookeeper.
-
-The usage for `start_parser_topology.sh` is as follows:
-
-```
-usage: start_parser_topology.sh
- -e,--extra_topology_options <JSON_FILE> Extra options in the form
- of a JSON file with a map
- for content.
- -esc,--extra_kafka_spout_config <JSON_FILE> Extra spout config options
- in the form of a JSON file
- with a map for content.
- Possible keys are:
- retryDelayMaxMs,retryDelay
- Multiplier,retryInitialDel
- ayMs,stateUpdateIntervalMs
- ,bufferSizeBytes,fetchMaxW
- ait,fetchSizeBytes,maxOffs
- etBehind,metricsTimeBucket
- SizeInSecs,socketTimeoutMs
- -ewnt,--error_writer_num_tasks <NUM_TASKS> Error Writer Num Tasks
- -ewp,--error_writer_p <PARALLELISM_HINT> Error Writer Parallelism
- Hint
- -h,--help This screen
- -iwnt,--invalid_writer_num_tasks <NUM_TASKS> Invalid Writer Num Tasks
- -iwp,--invalid_writer_p <PARALLELISM_HINT> Invalid Message Writer Parallelism Hint
- -k,--kafka <BROKER_URL> Kafka Broker URL
- -ksp,--kafka_security_protocol <SECURITY_PROTOCOL> Kafka Security Protocol
- -mt,--message_timeout <TIMEOUT_IN_SECS> Message Timeout in Seconds
- -mtp,--max_task_parallelism <MAX_TASK> Max task parallelism
- -na,--num_ackers <NUM_ACKERS> Number of Ackers
- -nw,--num_workers <NUM_WORKERS> Number of Workers
- -ot,--output_topic <KAFKA_TOPIC> Output Kafka Topic
- -pnt,--parser_num_tasks <NUM_TASKS> Parser Num Tasks
- -pp,--parser_p <PARALLELISM_HINT> Parser Parallelism Hint
- -s,--sensor <SENSOR_TYPE> Sensor Type
- -snt,--spout_num_tasks <NUM_TASKS> Spout Num Tasks
- -sp,--spout_p <SPOUT_PARALLELISM_HINT> Spout Parallelism Hint
- -t,--test <TEST> Run in Test Mode
- -z,--zk <ZK_QUORUM> Zookeeper Quroum URL
- (zk1:2181,zk2:2181,...
-```
-
-## The `--extra_kafka_spout_config` Option
-These options are intended to configure the Storm Kafka Spout more completely. These options can be
-specified in a JSON file containing a map associating the kafka spout configuration parameter to a value.
-The range of values possible to configure are:
-* `spout.pollTimeoutMs` - Specifies the time, in milliseconds, spent waiting in poll if data is not available. Default is 2s
-* `spout.firstPollOffsetStrategy` - Sets the offset used by the Kafka spout in the first poll to Kafka broker upon process start. One of
- * `EARLIEST`
- * `LATEST`
- * `UNCOMMITTED_EARLIEST` - Last uncommitted and if offsets aren't found, defaults to earliest. NOTE: This is the default.
- * `UNCOMMITTED_LATEST` - Last uncommitted and if offsets aren't found, defaults to latest.
-* `spout.offsetCommitPeriodMs` - Specifies the period, in milliseconds, the offset commit task is periodically called. Default is 15s.
-* `spout.maxUncommittedOffsets` - Defines the max number of polled offsets (records) that can be pending commit, before another poll can take place. Once this limit is reached, no more offsets (records) can be polled until the next successful commit(s) sets the number of pending offsets bellow the threshold. The default is 10,000,000.
-* `spout.maxRetries` - Defines the max number of retrials in case of tuple failure. The default is to retry forever, which means that no new records are committed until the previous polled records have been acked. This guarantees at once delivery of all the previously polled records. By specifying a finite value for maxRetries, the user decides to sacrifice guarantee of delivery for the previous polled records in favor of processing more records.
-* Any of the configs in the Consumer API for [Kafka 0.10.x](http://kafka.apache.org/0100/documentation.html#newconsumerconfigs)
-
-For instance, creating a JSON file which will set the offsets to `UNCOMMITTED_EARLIEST`
-```
-{
- "spout.firstPollOffsetStrategy" : "UNCOMMITTED_EARLIEST"
-}
-```
-
-This would be loaded by passing the file as argument to `--extra_kafka_spout_config`
+# Starting the Parser
-## The `--extra_topology_options` Option
+Starting a particular parser on a running Metron deployment is dependent on the platform being run on. Please see the appropriate platform-specific README.
+* [Storm](metron-parsing-storm/README.md#starting-the-parser-topology)
-These options are intended to be Storm configuration options and will live in
-a JSON file which will be loaded into the Storm config. For instance, if you wanted to set a storm property on
-the config called `topology.ticks.tuple.freq.secs` to 1000 and `storm.local.dir` to `/opt/my/path`
-you could create a file called `custom_config.json` containing
-```
-{
- "topology.ticks.tuple.freq.secs" : 1000,
- "storm.local.dir" : "/opt/my/path"
-}
-```
-and pass `--extra_topology_options custom_config.json` to `start_parser_topology.sh`.
+For all platforms, you will need to provide
+* Zookeeper Quorum
+* Kafka Broker URL
+* Sensor type
+* Output topic
+* Kakfa Security Protocol (Optional)
# Notes on Performance Tuning
@@ -711,13 +622,16 @@ are a few knobs to tune to get the most out of your system.
When using aggregated parsers, it's highly recommended to aggregate parsers with similar velocity and parser complexity together.
+Platform specific notes can be found in the appropriate README
+* [Storm](metron-parsing-storm/README.md)
+
# Notes on Adding a New Sensor
In order to allow for meta alerts to be queries alongside regular alerts in Elasticsearch 2.x,
it is necessary to add an additional field to the templates and mapping for existing sensors.
-Please see a description of the steps necessary to make this change in the metron-elasticsearch [Using Metron with Elasticsearch 2.x](../../metron-platform/metron-elasticsearch#using-metron-with-elasticsearch-2x)
+Please see a description of the steps necessary to make this change in the metron-elasticsearch [Using Metron with Elasticsearch 2.x](../../metron-elasticsearch#using-metron-with-elasticsearch-2x)
-If Solr is selected as the real-time store, it is also necessary to add additional fields. See the [Solr](../metron-indexing#solr) section in metron-indexing for more details.
+If Solr is selected as the real-time store, it is also necessary to add additional fields. See the [Solr](../../metron-indexing#solr) section in metron-indexing for more details.
## Kafka Queue
The kafka queue associated with your parser is a collection point for
@@ -725,28 +639,6 @@ all of the data sent to your parser. As such, make sure that the number of part
the kafka topic is sufficient to handle the throughput that you expect
from your parser topology.
-## Parser Topology
-The enrichment topology as started by the `$METRON_HOME/bin/start_parser_topology.sh`
-script uses a default of one executor per bolt. In a real production system, this should
-be customized by modifying the arguments sent to this utility.
-* Topology Wide
- * `--num_workers` : The number of workers for the topology
- * `--num_ackers` : The number of ackers for the topology
-* The Kafka Spout
- * `--spout_num_tasks` : The number of tasks for the spout
- * `--spout_p` : The parallelism hint for the spout
- * Ensure that the spout has enough parallelism so that it can dedicate a worker per partition in your kafka topic.
-* The Parser Bolt
- * `--parser_num_tasks` : The number of tasks for the parser bolt
- * `--parser_p` : The parallelism hint for the spout
- * This is bolt that gets the most processing, so ensure that it is configured with sufficient parallelism to match your throughput expectations.
-* The Error Message Writer Bolt
- * `--error_writer_num_tasks` : The number of tasks for the error writer bolt
- * `--error_writer_p` : The parallelism hint for the error writer bolt
-
-Finally, if workers and executors are new to you, the following might be of use to you:
-* [Understanding the Parallelism of a Storm Topology](http://www.michael-noll.com/blog/2012/10/16/understanding-the-parallelism-of-a-storm-topology/)
-
## JSON Path
> "JSONPath expressions always refer to a JSON structure in the same way as XPath expression are used in combination with an XML document."
diff --git a/metron-platform/metron-parsers/3rdPartyParser.md b/metron-platform/metron-parsing/metron-parsers-common/3rdPartyParser.md
similarity index 99%
rename from metron-platform/metron-parsers/3rdPartyParser.md
rename to metron-platform/metron-parsing/metron-parsers-common/3rdPartyParser.md
index 0c2409e..04cbc73 100644
--- a/metron-platform/metron-parsers/3rdPartyParser.md
+++ b/metron-platform/metron-parsing/metron-parsers-common/3rdPartyParser.md
@@ -87,7 +87,7 @@ up the maven project:
-->
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-parsers</artifactId>
+ <artifactId>metron-parsers-common</artifactId>
<version>${metron_version}</version>
<scope>provided</scope>
</dependency>
diff --git a/metron-platform/metron-parsers/ParserChaining.md b/metron-platform/metron-parsing/metron-parsers-common/ParserChaining.md
similarity index 97%
rename from metron-platform/metron-parsers/ParserChaining.md
rename to metron-platform/metron-parsing/metron-parsers-common/ParserChaining.md
index 4cab74c..9f08add 100644
--- a/metron-platform/metron-parsers/ParserChaining.md
+++ b/metron-platform/metron-parsing/metron-parsers-common/ParserChaining.md
@@ -31,7 +31,7 @@ Multi-pass parser involves the following requirements:
# High Level Solution
-
+
At a high level, we continue to maintain the architectural invariant of
a 1-1 relationship between logical sensors and storm topologies.
@@ -147,7 +147,7 @@ Together with routing, we have the complete solution to chain parsers which can:
Together this enables a directed acyclic graph of parsers to handle single or multi-layer parsing.
### Example
-For a complete example, look at the [parser chaining use-case](../../use-cases/parser_chaining), however for a simple example the following should suffice.
+For a complete example, look at the [parser chaining use-case](../../../use-cases/parser_chaining), however for a simple example the following should suffice.
If I want to configure a CSV parser to parse data which has 3 columns `f1`, `f2` and `f3` and is
held in a field called `payload` inside of a JSON Map, I can do so like
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/README.md b/metron-platform/metron-parsing/metron-parsers-common/README.md
similarity index 65%
copy from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/README.md
copy to metron-platform/metron-parsing/metron-parsers-common/README.md
index b6a1230..0c5cf23 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/README.md
+++ b/metron-platform/metron-parsing/metron-parsers-common/README.md
@@ -15,18 +15,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
-# BasicPaloAltoFirewallParser
-## Introduction
-The parser is able to parse logs generated by Palo Alto firewall devices powered by Pan OS. The supported log format is CSV.
-
-The supported log types and versions
-
-| Log type | Pan OS version |
-|----------|----------------|
-| Traffic | 6.0, 6.1, 7.0, 8.0 |
-| Threat | 6.0, 6.1, 7.0, 8.0 |
-| Config | 6.1, 7.0, 8.0 |
-| System | 6.1, 7.0, 8.0 |
+# metron-parsers-common
+metron-parsers-common contains the necessary components to write and test MessageParsers, along with some of the supporting utilities. The most basic parsers for extensions are included.
+The included parsers are
+* Grok Parser
+* JSONMapParser
+* CSVParser
+More details on these parsers and the overall architecture can be found in the metron-parsing [README](..#README.md)
diff --git a/metron-platform/metron-parsers/message-parser-implementation-notes.md b/metron-platform/metron-parsing/metron-parsers-common/message-parser-implementation-notes.md
similarity index 100%
rename from metron-platform/metron-parsers/message-parser-implementation-notes.md
rename to metron-platform/metron-parsing/metron-parsers-common/message-parser-implementation-notes.md
diff --git a/metron-platform/metron-parsers/parser-testing.md b/metron-platform/metron-parsing/metron-parsers-common/parser-testing.md
similarity index 57%
rename from metron-platform/metron-parsers/parser-testing.md
rename to metron-platform/metron-parsing/metron-parsers-common/parser-testing.md
index 3d670b5..29cfa88 100644
--- a/metron-platform/metron-parsers/parser-testing.md
+++ b/metron-platform/metron-parsing/metron-parsers-common/parser-testing.md
@@ -35,23 +35,15 @@ test that unit of functionality.
## The Integration Test
-Integration tests are more structured. The intent is that the parser that
-you have implemented can be driven successfully from `org.apache.metron.parsers.bolt.ParserBolt`.
+Integration tests are more structured. The intent is that the parser that you have implemented can be driven successfully from the appropriate driver, e.g. `org.apache.metron.parsers.bolt.ParserBolt` for Storm.
-The procedure for creating a new test is as follows:
-* Create an integration test that extends `org.apache.metron.parsers.integration.ParserIntegrationTest`
- * Override `getSensorType()` to return the sensor type to be used in the test (referred to as `${sensor_type}` at times)
- * Override `getValidations()` to indicate how you want the output of the parser to be validated (more on validations later)
- * Optionally `readSensorConfig(String sensorType)` to read the sensor config
- * By default, we will pull this from `metron-parsers/src/main/config/zookeeper/parsers/${sensor_type}`. Override if you want to provide your own
- * Optionally `readGlobalConfig()` to return the global config
- * By default, we will pull this from `metron-integration-test/src/main/config/zookeeper/global.json)`. Override if you want to provide your own
-* Place sample input data in `metron-integration-test/src/main/sample/data/${sensor_type}/raw`
- * It should be one line per input record.
-* Place expected output based on sample data in `metron-integration-test/src/main/sample/data/${sensor_type}/parsed`
- * Line `k` in the expected data should match with line `k`
+To add a new test, just add it to the list of sensorTypes in `org.apache.metron.parsers.integration.ParserIntegrationTest`.
-The way these tests function is by creating a `ParserBolt` instance with your specified global configuration and
+To setup the tests for a new platform, extend `ParserIntegrationTest`, e.g. as in `org.apache.metron.parsers.integration.StormParserIntegrationTests`. This should be a parameterized test, so that each sensorType gets its own test. Use `StormParserIntegrationTests` as a template for the new platform's class. The test method should just setup the appropriate `ParserDriver` implementation, and simply call back into the parent to run the test.
+
+Customized versions of the tests can be added by extending `ParserIntegrationTest` and performing additional setup or validations as needed.
+
+The way these tests function is by running the `ParserDriver` instance with your specified global configuration and
sensor configuration. It will then send your specified sample input data in line-by-line. It will then
perform some basic sanity validation:
* Ensure no errors were logged
@@ -66,22 +58,3 @@ conforms (excluding timestamp).
If you have special validations required, you may implement your own and return an instance of that in the `getValidations()`
method of your Integration Test.
-
-### Sample Integration Test
-
-A sample integration test for the `snort` parser is as follows:
-```
-public class SnortIntegrationTest extends ParserIntegrationTest {
- @Override
- String getSensorType() {
- return "snort";
- }
-
- @Override
- List<ParserValidation> getValidations() {
- return new ArrayList<ParserValidation>() {{
- add(new SampleDataValidation());
- }};
- }
-}
-```
diff --git a/metron-platform/metron-parsers/parser_arch.png b/metron-platform/metron-parsing/metron-parsers-common/parser_arch.png
similarity index 100%
rename from metron-platform/metron-parsers/parser_arch.png
rename to metron-platform/metron-parsing/metron-parsers-common/parser_arch.png
diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsing/metron-parsers-common/pom.xml
similarity index 86%
rename from metron-platform/metron-parsers/pom.xml
rename to metron-platform/metron-parsing/metron-parsers-common/pom.xml
index e4914b6..617366a 100644
--- a/metron-platform/metron-parsers/pom.xml
+++ b/metron-platform/metron-parsing/metron-parsers-common/pom.xml
@@ -17,11 +17,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-platform</artifactId>
+ <artifactId>metron-parsing</artifactId>
<version>0.7.0</version>
</parent>
- <artifactId>metron-parsers</artifactId>
- <name>metron-parsers</name>
+ <artifactId>metron-parsers-common</artifactId>
+ <name>metron-parsers-common</name>
+ <description>Metron common code for building parsers</description>
<url>https://metron.apache.org/</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -124,40 +125,6 @@
</exclusions>
</dependency>
<dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-rename-hack</artifactId>
- <version>${global_storm_version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- <version>${global_storm_version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-over-slf4j</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-slf4j-impl</artifactId>
- <groupId>org.apache.logging.log4j</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>${global_log4j_core_version}</version>
@@ -176,11 +143,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${global_guava_version}</version>
- </dependency>
- <dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${global_jackson_version}</version>
@@ -240,12 +202,6 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-test-utilities</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
<artifactId>metron-integration-test</artifactId>
<version>${project.parent.version}</version>
<scope>test</scope>
@@ -261,11 +217,6 @@
<artifactId>json-path</artifactId>
<version>2.3.0</version>
</dependency>
- <dependency>
- <groupId>com.github.palindromicity</groupId>
- <artifactId>simple-syslog-5424</artifactId>
- <version>${global_simple_syslog_version}</version>
- </dependency>
</dependencies>
<build>
<plugins>
@@ -320,8 +271,6 @@
</relocations>
<artifactSet>
<excludes>
- <exclude>storm:storm-core:*</exclude>
- <exclude>storm:storm-lib:*</exclude>
<exclude>org.slf4j.impl*</exclude>
<exclude>org.slf4j:slf4j-log4j*</exclude>
</excludes>
diff --git a/metron-platform/metron-parsers/src/main/assembly/assembly.xml b/metron-platform/metron-parsing/metron-parsers-common/src/main/assembly/assembly.xml
similarity index 100%
copy from metron-platform/metron-parsers/src/main/assembly/assembly.xml
copy to metron-platform/metron-parsing/metron-parsers-common/src/main/assembly/assembly.xml
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMap.json b/metron-platform/metron-parsing/metron-parsers-common/src/main/config/zookeeper/parsers/jsonMap.json
similarity index 100%
copy from metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMap.json
copy to metron-platform/metron-parsing/metron-parsers-common/src/main/config/zookeeper/parsers/jsonMap.json
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMapQuery.json b/metron-platform/metron-parsing/metron-parsers-common/src/main/config/zookeeper/parsers/jsonMapQuery.json
similarity index 100%
copy from metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMapQuery.json
copy to metron-platform/metron-parsing/metron-parsers-common/src/main/config/zookeeper/parsers/jsonMapQuery.json
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMapWrappedQuery.json b/metron-platform/metron-parsing/metron-parsers-common/src/main/config/zookeeper/parsers/jsonMapWrappedQuery.json
similarity index 100%
copy from metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMapWrappedQuery.json
copy to metron-platform/metron-parsing/metron-parsers-common/src/main/config/zookeeper/parsers/jsonMapWrappedQuery.json
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/BasicParser.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/BasicParser.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/BasicParser.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/BasicParser.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/DefaultMessageParserResult.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/DefaultMessageParserResult.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/DefaultMessageParserResult.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/DefaultMessageParserResult.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/DefaultParserRunnerResults.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/DefaultParserRunnerResults.java
similarity index 99%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/DefaultParserRunnerResults.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/DefaultParserRunnerResults.java
index 79a9b5d..4e9d6b7 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/DefaultParserRunnerResults.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/DefaultParserRunnerResults.java
@@ -68,4 +68,4 @@ public class DefaultParserRunnerResults implements ParserRunnerResults<JSONObjec
result = 31 * result + (errors != null ? errors.hashCode() : 0);
return result;
}
-}
+}
\ No newline at end of file
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/GrokParser.java
similarity index 99%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/GrokParser.java
index 6bdfb81..f64b4af 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/GrokParser.java
@@ -44,7 +44,6 @@ import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParseException.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/ParseException.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParseException.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/ParseException.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserComponent.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/ParserComponent.java
similarity index 97%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserComponent.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/ParserComponent.java
index eb5ff9f..c040acb 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserComponent.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/ParserComponent.java
@@ -15,14 +15,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.metron.parsers.topology;
+package org.apache.metron.parsers;
+import java.io.Serializable;
import org.apache.metron.parsers.interfaces.MessageFilter;
import org.apache.metron.parsers.interfaces.MessageParser;
import org.json.simple.JSONObject;
-import java.io.Serializable;
-
public class ParserComponent implements Serializable {
private static final long serialVersionUID = 7880346740026374665L;
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunner.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/ParserRunner.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunner.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/ParserRunner.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunnerImpl.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/ParserRunnerImpl.java
similarity index 99%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunnerImpl.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/ParserRunnerImpl.java
index a986db7..df3ed1c 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunnerImpl.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/ParserRunnerImpl.java
@@ -17,6 +17,20 @@
*/
package org.apache.metron.parsers;
+import java.io.Serializable;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.metron.common.Constants;
import org.apache.metron.common.configuration.FieldTransformer;
@@ -30,27 +44,11 @@ import org.apache.metron.parsers.filters.Filters;
import org.apache.metron.parsers.interfaces.MessageFilter;
import org.apache.metron.parsers.interfaces.MessageParser;
import org.apache.metron.parsers.interfaces.MessageParserResult;
-import org.apache.metron.parsers.topology.ParserComponent;
import org.apache.metron.stellar.dsl.Context;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.Serializable;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.UUID;
-import java.util.function.Consumer;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-
/**
* The default implemention of a ParserRunner.
*/
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunnerResults.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/ParserRunnerResults.java
similarity index 99%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunnerResults.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/ParserRunnerResults.java
index 7ca853c..63abd0e 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunnerResults.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/ParserRunnerResults.java
@@ -30,4 +30,4 @@ public interface ParserRunnerResults<T> {
List<T> getMessages();
List<MetronError> getErrors();
-}
+}
\ No newline at end of file
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/csv/CSVParser.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/csv/CSVParser.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/AbstractMessageFilter.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/filters/AbstractMessageFilter.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/AbstractMessageFilter.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/filters/AbstractMessageFilter.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/Filters.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/filters/Filters.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/Filters.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/filters/Filters.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/StellarFilter.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/filters/StellarFilter.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/StellarFilter.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/filters/StellarFilter.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/Configurable.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/interfaces/Configurable.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/Configurable.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/interfaces/Configurable.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParserResult.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/interfaces/MessageParserResult.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParserResult.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/interfaces/MessageParserResult.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/DateUtils.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/utils/DateUtils.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/DateUtils.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/utils/DateUtils.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/GrokUtils.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/utils/GrokUtils.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/GrokUtils.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/utils/GrokUtils.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/ParserUtils.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/utils/ParserUtils.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/ParserUtils.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/utils/ParserUtils.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/SyslogUtils.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/utils/SyslogUtils.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/SyslogUtils.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/utils/SyslogUtils.java
diff --git a/metron-platform/metron-parsers/src/main/resources/META-INF/LICENSE b/metron-platform/metron-parsing/metron-parsers-common/src/main/resources/META-INF/LICENSE
similarity index 100%
copy from metron-platform/metron-parsers/src/main/resources/META-INF/LICENSE
copy to metron-platform/metron-parsing/metron-parsers-common/src/main/resources/META-INF/LICENSE
diff --git a/metron-platform/metron-parsers/src/main/resources/META-INF/NOTICE b/metron-platform/metron-parsing/metron-parsers-common/src/main/resources/META-INF/NOTICE
similarity index 98%
copy from metron-platform/metron-parsers/src/main/resources/META-INF/NOTICE
copy to metron-platform/metron-parsing/metron-parsers-common/src/main/resources/META-INF/NOTICE
index d9212cd..456a20a 100644
--- a/metron-platform/metron-parsers/src/main/resources/META-INF/NOTICE
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/main/resources/META-INF/NOTICE
@@ -1,5 +1,5 @@
-metron-parsers
+metron-parsers-common
Copyright 2006-2016 The Apache Software Foundation
This product includes software developed at
diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/common b/metron-platform/metron-parsing/metron-parsers-common/src/main/resources/patterns/common
similarity index 100%
rename from metron-platform/metron-parsers/src/main/resources/patterns/common
rename to metron-platform/metron-parsing/metron-parsers-common/src/main/resources/patterns/common
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/filters/FiltersTest.java
similarity index 100%
copy from metron-platform/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/filters/FiltersTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/AbstractParserConfigTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/AbstractParserConfigTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/AbstractParserConfigTest.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/AbstractParserConfigTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/GrokParserTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/GrokParserTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MessageParserTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MessageParserTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MessageParserTest.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MessageParserTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
similarity index 98%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
index e24a39d..8478bd4 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
@@ -126,7 +126,7 @@ public class MultiLineGrokParserTest {
public String getMultiLine() { return "true";}
public String getGrokPath() {
- return "../metron-integration-test/src/main/sample/patterns/test";
+ return "../../metron-integration-test/src/main/sample/patterns/test";
}
public String getGrokPatternLabel() {
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineWithErrorsGrokParserTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MultiLineWithErrorsGrokParserTest.java
similarity index 98%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineWithErrorsGrokParserTest.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MultiLineWithErrorsGrokParserTest.java
index 8ab8246..4a235bf 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineWithErrorsGrokParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MultiLineWithErrorsGrokParserTest.java
@@ -121,7 +121,7 @@ public class MultiLineWithErrorsGrokParserTest {
}
public String getGrokPath() {
- return "../metron-integration-test/src/main/sample/patterns/test";
+ return "../../metron-integration-test/src/main/sample/patterns/test";
}
public String getGrokPatternLabel() {
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
similarity index 99%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
index 5f05b24..29a625d 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
@@ -17,6 +17,23 @@
*/
package org.apache.metron.parsers;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Optional;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.Constants;
import org.apache.metron.common.configuration.ParserConfigurations;
@@ -25,13 +42,12 @@ import org.apache.metron.common.error.MetronError;
import org.apache.metron.common.message.metadata.RawMessage;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.common.utils.ReflectionUtils;
+import org.apache.metron.parsers.ParserRunnerImpl.ProcessResult;
import org.apache.metron.parsers.filters.Filters;
import org.apache.metron.parsers.filters.StellarFilter;
import org.apache.metron.parsers.interfaces.MessageFilter;
import org.apache.metron.parsers.interfaces.MessageParser;
import org.apache.metron.parsers.interfaces.MessageParserResult;
-import org.apache.metron.parsers.topology.ParserComponent;
-import org.apache.metron.parsers.ParserRunnerImpl.ProcessResult;
import org.apache.metron.stellar.dsl.Context;
import org.json.simple.JSONObject;
import org.junit.Assert;
@@ -43,24 +59,6 @@ import org.junit.runner.RunWith;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Optional;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
-import static org.powermock.api.mockito.PowerMockito.mockStatic;
-import static org.powermock.api.mockito.PowerMockito.when;
-
@RunWith(PowerMockRunner.class)
@PrepareForTest({ParserRunnerImpl.class, ReflectionUtils.class, Filters.class})
public class ParserRunnerImplTest {
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java
similarity index 97%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java
index 35e07f8..23ef72d 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java
@@ -73,7 +73,7 @@ public class SampleGrokParserTest extends GrokParserTest {
@Override
public String getGrokPath() {
- return "../metron-integration-test/src/main/sample/patterns/test";
+ return "../../metron-integration-test/src/main/sample/patterns/test";
}
@Override
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/EnvelopedParserIntegrationTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/EnvelopedParserIntegrationTest.java
similarity index 79%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/EnvelopedParserIntegrationTest.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/EnvelopedParserIntegrationTest.java
index c017415..3044061 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/EnvelopedParserIntegrationTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/EnvelopedParserIntegrationTest.java
@@ -18,20 +18,19 @@
package org.apache.metron.parsers.integration;
import com.google.common.collect.ImmutableList;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.Constants;
import org.apache.metron.common.message.metadata.MetadataUtil;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.integration.ProcessorResult;
+import org.apache.metron.parsers.integration.validation.ParserDriver;
import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-public class EnvelopedParserIntegrationTest {
+public abstract class EnvelopedParserIntegrationTest {
/**
* {
@@ -52,9 +51,7 @@ public class EnvelopedParserIntegrationTest {
@Multiline
public static String parserConfig_default;
- @Test
- public void testEnvelopedData() throws IOException {
- ParserDriver driver = new ParserDriver("test", parserConfig_default, "{}");
+ public void testEnvelopedData(ParserDriver driver) throws IOException {
Map<String, Object> inputRecord = new HashMap<String, Object>() {{
put(Constants.Fields.ORIGINAL.getName(), "real_original_string");
put("data", "field1_val,100");
@@ -91,9 +88,7 @@ public class EnvelopedParserIntegrationTest {
@Multiline
public static String parserConfig_withPrefix;
- @Test
- public void testEnvelopedData_withMetadataPrefix() throws IOException {
- ParserDriver driver = new ParserDriver("test", parserConfig_withPrefix, "{}");
+ public void testEnvelopedData_withMetadataPrefix(ParserDriver driver) throws IOException {
Map<String, Object> inputRecord = new HashMap<String, Object>() {{
put(Constants.Fields.ORIGINAL.getName(), "real_original_string");
put("data", "field1_val,100");
@@ -129,9 +124,7 @@ public class EnvelopedParserIntegrationTest {
@Multiline
public static String parserConfig_nomerge;
- @Test
- public void testEnvelopedData_noMergeMetadata() throws IOException {
- ParserDriver driver = new ParserDriver("test", parserConfig_nomerge, "{}");
+ public void testEnvelopedData_noMergeMetadata(ParserDriver driver) throws IOException {
Map<String, Object> inputRecord = new HashMap<String, Object>() {{
put(Constants.Fields.ORIGINAL.getName(), "real_original_string");
put("data", "field1_val,100");
@@ -182,28 +175,20 @@ public class EnvelopedParserIntegrationTest {
@Multiline
public static String cisco302020Config;
- @Test
- public void testCiscoPixEnvelopingCisco302020() throws Exception {
+ public void testCiscoPixEnvelopingCisco302020(ParserDriver syslogDriver, ParserDriver driver)
+ throws Exception {
byte[] envelopedData = null;
- {
- ParserDriver driver = new ParserDriver("ciscoPix", ciscoPixSyslogConfig, "{}");
- String inputRecord = "Mar 29 2004 09:54:18: %PIX-6-302005: Built UDP connection for faddr 198.207.223.240/53337 gaddr 10.0.0.187/53 laddr 192.168.0.2/53";
- ProcessorResult<List<byte[]>> results = driver.run(ImmutableList.of(inputRecord.getBytes()));
- Assert.assertFalse(results.failed());
- List<byte[]> resultList = results.getResult();
- envelopedData = resultList.get(0);
- }
- {
- ParserDriver driver = new ParserDriver("cisco302020", cisco302020Config, "{}");
- ProcessorResult<List<byte[]>> results = driver.run(ImmutableList.of(envelopedData));
- Assert.assertFalse(results.failed());
- List<byte[]> resultList = results.getResult();
- Assert.assertEquals(1, resultList.size());
- Map<String, Object> result = JSONUtils.INSTANCE.load(new String(resultList.get(0)), JSONUtils.MAP_SUPPLIER);
- Assert.assertEquals("UDP", result.get("protocol"));
- Assert.assertTrue((long)result.get("timestamp") > 1000 );
- }
-
+ String inputRecord = "Mar 29 2004 09:54:18: %PIX-6-302005: Built UDP connection for faddr 198.207.223.240/53337 gaddr 10.0.0.187/53 laddr 192.168.0.2/53";
+ ProcessorResult<List<byte[]>> syslogResult = syslogDriver.run(ImmutableList.of(inputRecord.getBytes()));
+ Assert.assertFalse(syslogResult.failed());
+ List<byte[]> syslogResultList = syslogResult.getResult();
+ envelopedData = syslogResultList.get(0);
+ ProcessorResult<List<byte[]>> results = driver.run(ImmutableList.of(envelopedData));
+ Assert.assertFalse(results.failed());
+ List<byte[]> resultList = results.getResult();
+ Assert.assertEquals(1, resultList.size());
+ Map<String, Object> result = JSONUtils.INSTANCE.load(new String(resultList.get(0)), JSONUtils.MAP_SUPPLIER);
+ Assert.assertEquals("UDP", result.get("protocol"));
+ Assert.assertTrue((long) result.get("timestamp") > 1000);
}
-
}
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
similarity index 71%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
index 6b00a2b..de0b984 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
@@ -17,39 +17,60 @@
*/
package org.apache.metron.parsers.integration;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.List;
import org.apache.metron.TestConstants;
import org.apache.metron.integration.BaseIntegrationTest;
import org.apache.metron.integration.ProcessorResult;
import org.apache.metron.integration.utils.TestUtils;
+import org.apache.metron.parsers.integration.validation.ParserDriver;
import org.apache.metron.test.TestDataType;
import org.apache.metron.test.utils.SampleDataUtils;
import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.util.List;
public abstract class ParserIntegrationTest extends BaseIntegrationTest {
+ // Contains the list of sensor types to be tested.
+ // See StormParserIntegrationTest for an example of how to use this in a Parameterized JUnit test
+ protected static List<String> sensorTypes = Arrays.asList(
+ "asa",
+ "bro",
+ "jsonMap",
+ "jsonMapQuery",
+ "jsonMapWrappedQuery",
+ "snort",
+ "squid",
+ "websphere",
+ "yaf",
+ "syslog5424"
+ );
+
protected List<byte[]> inputMessages;
protected String readGlobalConfig() throws IOException {
- File configsRoot = new File(TestConstants.SAMPLE_CONFIG_PATH);
+ File configsRoot = new File("../" + TestConstants.SAMPLE_CONFIG_PATH);
return new String(Files.readAllBytes(new File(configsRoot, "global.json").toPath()));
}
protected String readSensorConfig(String sensorType) throws IOException {
- File configsRoot = new File(TestConstants.PARSER_CONFIGS_PATH);
+ // First check the basic parsers dir.
+ File configsRoot = new File("../" + TestConstants.PARSER_COMMON_CONFIGS_PATH);
File parsersRoot = new File(configsRoot, "parsers");
+ System.out.println("Workspace: " + System.getProperty("user.dir"));
+ System.out.println("Parsers root: " + parsersRoot);
+ if (!Files.exists(new File(parsersRoot, sensorType + ".json").toPath())) {
+ // Use the main parsers configs
+ configsRoot = new File("../" + TestConstants.PARSER_CONFIGS_PATH);
+ parsersRoot = new File(configsRoot, "parsers");
+ }
return new String(Files.readAllBytes(new File(parsersRoot, sensorType + ".json").toPath()));
}
- @Test
- public void test() throws Exception {
- String sensorType = getSensorType();
- ParserDriver driver = new ParserDriver(sensorType, readSensorConfig(sensorType), readGlobalConfig());
- inputMessages = TestUtils.readSampleData(SampleDataUtils.getSampleDataPath(sensorType, TestDataType.RAW));
+ public void runTest(ParserDriver driver) throws Exception {
+ String sensorType = driver.getSensorType();
+ inputMessages = TestUtils.readSampleData(SampleDataUtils.getSampleDataPath("..", sensorType, TestDataType.RAW));
ProcessorResult<List<byte[]>> result = driver.run(inputMessages);
List<byte[]> outputMessages = result.getResult();
@@ -80,7 +101,5 @@ public abstract class ParserIntegrationTest extends BaseIntegrationTest {
}
}
- abstract String getSensorType();
abstract List<ParserValidation> getValidations();
-
}
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserValidation.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/ParserValidation.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserValidation.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/ParserValidation.java
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/validation/ParserDriver.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/validation/ParserDriver.java
new file mode 100644
index 0000000..4854429
--- /dev/null
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/validation/ParserDriver.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.integration.validation;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.HashSet;
+import java.util.List;
+import org.apache.metron.common.configuration.ParserConfigurations;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.integration.ProcessorResult;
+import org.apache.metron.parsers.ParserRunner;
+import org.apache.metron.parsers.ParserRunnerImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class ParserDriver implements Serializable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ParserDriver.class);
+
+ protected ParserConfigurations config;
+ protected String sensorType;
+ protected ParserRunner parserRunner;
+
+ public ParserDriver(String sensorType, String parserConfig, String globalConfig)
+ throws IOException {
+ SensorParserConfig sensorParserConfig = SensorParserConfig.fromBytes(parserConfig.getBytes());
+ this.sensorType = sensorType == null ? sensorParserConfig.getSensorTopic() : sensorType;
+ config = new ParserConfigurations();
+ config.updateSensorParserConfig(this.sensorType,
+ SensorParserConfig.fromBytes(parserConfig.getBytes()));
+ config.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER));
+
+ parserRunner = new ParserRunnerImpl(new HashSet<String>() {{
+ add(sensorType);
+ }});
+ }
+
+ abstract public ProcessorResult<List<byte[]>> run(Iterable<byte[]> in);
+
+ public String getSensorType() {
+ return sensorType;
+ }
+}
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/validation/SampleDataValidation.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/validation/SampleDataValidation.java
similarity index 96%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/validation/SampleDataValidation.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/validation/SampleDataValidation.java
index 1dff22f..adc246d 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/validation/SampleDataValidation.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/validation/SampleDataValidation.java
@@ -35,7 +35,7 @@ public class SampleDataValidation implements ParserValidation {
@Override
public void validate(String sensorType, List<byte[]> actualMessages) throws Exception {
- List<byte[]> expectedMessages = TestUtils.readSampleData(SampleDataUtils.getSampleDataPath(sensorType, TestDataType.PARSED));
+ List<byte[]> expectedMessages = TestUtils.readSampleData(SampleDataUtils.getSampleDataPath("..", sensorType, TestDataType.PARSED));
Assert.assertEquals(expectedMessages.size(), actualMessages.size());
for (int i = 0; i < actualMessages.size(); i++) {
String expectedMessage = new String(expectedMessages.get(i));
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserQueryTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/json/JSONMapParserQueryTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserQueryTest.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/json/JSONMapParserQueryTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserWrappedQueryTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/json/JSONMapParserWrappedQueryTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserWrappedQueryTest.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/json/JSONMapParserWrappedQueryTest.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/README.md b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/paloalto/README.md
similarity index 100%
copy from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/README.md
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/paloalto/README.md
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/regex/RegularExpressionsParser.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/regex/RegularExpressionsParser.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/regex/RegularExpressionsParser.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/regex/RegularExpressionsParser.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/regex/RegularExpressionsParserTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/regex/RegularExpressionsParserTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/regex/RegularExpressionsParserTest.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/regex/RegularExpressionsParserTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/utils/ParserUtilsTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/utils/ParserUtilsTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/utils/ParserUtilsTest.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/utils/ParserUtilsTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/utils/SyslogUtilsTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/utils/SyslogUtilsTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/utils/SyslogUtilsTest.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/utils/SyslogUtilsTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
rename to metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
diff --git a/metron-platform/metron-parsers/src/test/resources/TestSchemas/BroSchema.json b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/TestSchemas/BroSchema.json
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/TestSchemas/BroSchema.json
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/TestSchemas/BroSchema.json
diff --git a/metron-platform/metron-parsers/src/test/resources/TestSchemas/IseSchema.json b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/TestSchemas/IseSchema.json
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/TestSchemas/IseSchema.json
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/TestSchemas/IseSchema.json
diff --git a/metron-platform/metron-parsers/src/test/resources/TestSchemas/LancopeSchema.json b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/TestSchemas/LancopeSchema.json
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/TestSchemas/LancopeSchema.json
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/TestSchemas/LancopeSchema.json
diff --git a/metron-platform/metron-parsers/src/test/resources/TestSchemas/PcapSchema.json b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/TestSchemas/PcapSchema.json
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/TestSchemas/PcapSchema.json
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/TestSchemas/PcapSchema.json
diff --git a/metron-platform/metron-parsers/src/test/resources/TestSchemas/SourcefireSchema.json b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/TestSchemas/SourcefireSchema.json
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/TestSchemas/SourcefireSchema.json
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/TestSchemas/SourcefireSchema.json
diff --git a/metron-platform/metron-parsers/src/test/resources/config/BasicFireEyeParserTest.config b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/config/BasicFireEyeParserTest.config
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/config/BasicFireEyeParserTest.config
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/config/BasicFireEyeParserTest.config
diff --git a/metron-platform/metron-parsers/src/test/resources/config/BasicIseParserTest.config b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/config/BasicIseParserTest.config
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/config/BasicIseParserTest.config
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/config/BasicIseParserTest.config
diff --git a/metron-platform/metron-parsers/src/test/resources/config/BasicLancopeParserTest.config b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/config/BasicLancopeParserTest.config
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/config/BasicLancopeParserTest.config
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/config/BasicLancopeParserTest.config
diff --git a/metron-platform/metron-parsers/src/test/resources/config/BasicPaloAltoFirewallParserTest.config b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/config/BasicPaloAltoFirewallParserTest.config
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/config/BasicPaloAltoFirewallParserTest.config
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/config/BasicPaloAltoFirewallParserTest.config
diff --git a/metron-platform/metron-parsers/src/test/resources/config/BasicSourcefireParserTest.config b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/config/BasicSourcefireParserTest.config
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/config/BasicSourcefireParserTest.config
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/config/BasicSourcefireParserTest.config
diff --git a/metron-platform/metron-parsers/src/test/resources/config/BroParserTest.config b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/config/BroParserTest.config
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/config/BroParserTest.config
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/config/BroParserTest.config
diff --git a/metron-platform/metron-parsers/src/test/resources/logData/FireEyeParserTest.txt b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/logData/FireEyeParserTest.txt
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/logData/FireEyeParserTest.txt
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/logData/FireEyeParserTest.txt
diff --git a/metron-platform/metron-parsers/src/test/resources/logData/IseParserTest.txt b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/logData/IseParserTest.txt
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/logData/IseParserTest.txt
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/logData/IseParserTest.txt
diff --git a/metron-platform/metron-parsers/src/test/resources/logData/LancopeParserTest.txt b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/logData/LancopeParserTest.txt
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/logData/LancopeParserTest.txt
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/logData/LancopeParserTest.txt
diff --git a/metron-platform/metron-parsers/src/test/resources/logData/SourcefireParserTest.txt b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/logData/SourcefireParserTest.txt
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/logData/SourcefireParserTest.txt
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/logData/SourcefireParserTest.txt
diff --git a/metron-platform/metron-parsers/src/test/resources/logData/multi_elb_log.txt b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/logData/multi_elb_log.txt
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/logData/multi_elb_log.txt
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/logData/multi_elb_log.txt
diff --git a/metron-platform/metron-parsers/src/test/resources/logData/multi_elb_with_errors_log.txt b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/logData/multi_elb_with_errors_log.txt
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/logData/multi_elb_with_errors_log.txt
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/logData/multi_elb_with_errors_log.txt
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.cef b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/org/apache/metron/parsers/cef/adallom.cef
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.cef
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/org/apache/metron/parsers/cef/adallom.cef
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.schema b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/org/apache/metron/parsers/cef/adallom.schema
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.schema
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/org/apache/metron/parsers/cef/adallom.schema
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.json b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/org/apache/metron/parsers/cef/cyberark.json
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.json
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/org/apache/metron/parsers/cef/cyberark.json
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.cef b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/org/apache/metron/parsers/cef/palo.cef
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.cef
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/org/apache/metron/parsers/cef/palo.cef
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.schema b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/org/apache/metron/parsers/cef/palo.schema
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.schema
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/org/apache/metron/parsers/cef/palo.schema
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.cef b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/org/apache/metron/parsers/cef/waf.cef
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.cef
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/org/apache/metron/parsers/cef/waf.cef
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.schema b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/org/apache/metron/parsers/cef/waf.schema
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.schema
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/org/apache/metron/parsers/cef/waf.schema
diff --git a/metron-platform/metron-parsers/src/test/resources/patterns/cisco_patterns b/metron-platform/metron-parsing/metron-parsers-common/src/test/resources/patterns/cisco_patterns
similarity index 100%
copy from metron-platform/metron-parsers/src/test/resources/patterns/cisco_patterns
copy to metron-platform/metron-parsing/metron-parsers-common/src/test/resources/patterns/cisco_patterns
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/README.md b/metron-platform/metron-parsing/metron-parsers/README.md
similarity index 63%
copy from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/README.md
copy to metron-platform/metron-parsing/metron-parsers/README.md
index b6a1230..98e0094 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/README.md
+++ b/metron-platform/metron-parsing/metron-parsers/README.md
@@ -15,18 +15,23 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
-# BasicPaloAltoFirewallParser
-## Introduction
-The parser is able to parse logs generated by Palo Alto firewall devices powered by Pan OS. The supported log format is CSV.
+# metron-parsers
-The supported log types and versions
-
-| Log type | Pan OS version |
-|----------|----------------|
-| Traffic | 6.0, 6.1, 7.0, 8.0 |
-| Threat | 6.0, 6.1, 7.0, 8.0 |
-| Config | 6.1, 7.0, 8.0 |
-| System | 6.1, 7.0, 8.0 |
+metron-parsers contains the included parsers that aren't our basic parsers.
+The included parsers are:
+* Asa
+* CEF
+* FireEye
+* ISE
+* Lancope
+* Logstash
+* PaloAlto
+* Snort
+* Sourcefire
+* Syslog
+* Websphere
+The basic parsers and their details can be found at [README](../metron-parsers-common#README.md).
+More details on these parsers and the overall architecture can be found in the metron-parsing [README](..#README.md).
\ No newline at end of file
diff --git a/metron-platform/metron-parsing/metron-parsers/pom.xml b/metron-platform/metron-parsing/metron-parsers/pom.xml
new file mode 100644
index 0000000..d8b6825
--- /dev/null
+++ b/metron-platform/metron-parsing/metron-parsers/pom.xml
@@ -0,0 +1,218 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parsing</artifactId>
+ <version>0.7.0</version>
+ </parent>
+ <artifactId>metron-parsers</artifactId>
+ <name>metron-parsers</name>
+ <description>Metron bundled parsers</description>
+ <url>https://metron.apache.org/</url>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ </properties>
+ <dependencies>
+ <!-- Metron Dependencies -->
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parsers-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+
+ <!-- 3rd party dependencies -->
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${global_guava_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.thekraken</groupId>
+ <artifactId>grok</artifactId>
+ <version>0.1.0</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>commons-lang3</artifactId>
+ <groupId>org.apache.commons</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.github.palindromicity</groupId>
+ <artifactId>simple-syslog-5424</artifactId>
+ <version>${global_simple_syslog_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parsers-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <!-- Test dependencies -->
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parsers-common</artifactId>
+ <version>${project.parent.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-test-utilities</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.github.fge</groupId>
+ <artifactId>json-schema-validator</artifactId>
+ <version>2.2.6</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>${global_jar_version}</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${global_shade_version}</version>
+ <configuration>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <artifactSet>
+ <excludes>
+ <exclude>*slf4j*</exclude>
+ </excludes>
+ </artifactSet>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>org.apache.metron.jackson</shadedPattern>
+ </relocation>
+ </relocations>
+ <artifactSet>
+ <excludes>
+ <exclude>org.slf4j.impl*</exclude>
+ <exclude>org.slf4j:slf4j-log4j*</exclude>
+ </excludes>
+ </artifactSet>
+ <transformers>
+ <transformer implementation="org.atteo.classindex.ClassIndexTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resources>
+ <resource>.yaml</resource>
+ <resource>LICENSE.txt</resource>
+ <resource>ASL2.0</resource>
+ <resource>NOTICE.txt</resource>
+ </resources>
+ </transformer>
+ <!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE -->
+ <!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
+ <addHeader>false</addHeader>
+ <projectName>${project.name}</projectName>
+ </transformer-->
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass></mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.atteo.classindex</groupId>
+ <artifactId>classindex-transformer</artifactId>
+ <version>${global_classindex_version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id> <!-- this is used for inheritance merges -->
+ <phase>package</phase> <!-- bind to the packaging phase -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ <resource>
+ <directory>src/main/patterns</directory>
+ </resource>
+ <resource>
+ <directory>src/test/resources</directory>
+ </resource>
+ </resources>
+ </build>
+</project>
diff --git a/metron-platform/metron-parsers/src/main/assembly/assembly.xml b/metron-platform/metron-parsing/metron-parsers/src/main/assembly/assembly.xml
similarity index 100%
copy from metron-platform/metron-parsers/src/main/assembly/assembly.xml
copy to metron-platform/metron-parsing/metron-parsers/src/main/assembly/assembly.xml
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/asa.json b/metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/asa.json
similarity index 100%
rename from metron-platform/metron-parsers/src/main/config/zookeeper/parsers/asa.json
rename to metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/asa.json
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/bro.json b/metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/bro.json
similarity index 100%
rename from metron-platform/metron-parsers/src/main/config/zookeeper/parsers/bro.json
rename to metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/bro.json
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMap.json b/metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/jsonMap.json
similarity index 100%
rename from metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMap.json
rename to metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/jsonMap.json
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMapQuery.json b/metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/jsonMapQuery.json
similarity index 100%
rename from metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMapQuery.json
rename to metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/jsonMapQuery.json
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMapWrappedQuery.json b/metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/jsonMapWrappedQuery.json
similarity index 100%
rename from metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMapWrappedQuery.json
rename to metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/jsonMapWrappedQuery.json
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/snort.json b/metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/snort.json
similarity index 100%
rename from metron-platform/metron-parsers/src/main/config/zookeeper/parsers/snort.json
rename to metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/snort.json
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/squid.json b/metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/squid.json
similarity index 100%
rename from metron-platform/metron-parsers/src/main/config/zookeeper/parsers/squid.json
rename to metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/squid.json
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/syslog5424.json b/metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/syslog5424.json
similarity index 100%
rename from metron-platform/metron-parsers/src/main/config/zookeeper/parsers/syslog5424.json
rename to metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/syslog5424.json
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json b/metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/websphere.json
similarity index 100%
rename from metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json
rename to metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/websphere.json
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/yaf.json b/metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/yaf.json
similarity index 100%
rename from metron-platform/metron-parsers/src/main/config/zookeeper/parsers/yaf.json
rename to metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/yaf.json
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ISEParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ISEParser.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ISEParser.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ISEParser.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ISEParser.jj b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ISEParser.jj
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ISEParser.jj
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ISEParser.jj
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ISEParserConstants.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ISEParserConstants.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ISEParserConstants.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ISEParserConstants.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ISEParserTokenManager.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ISEParserTokenManager.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ISEParserTokenManager.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ISEParserTokenManager.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ParseException.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ParseException.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ParseException.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ParseException.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/Token.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/Token.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/Token.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/Token.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java
similarity index 99%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java
index 2f5310c..f10b976 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java
@@ -17,14 +17,13 @@
*/
package org.apache.metron.parsers.logstash;
-import org.apache.metron.parsers.BasicParser;
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
-
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.metron.parsers.BasicParser;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
public class BasicLogstashParser extends BasicParser {
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/README.md b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/README.md
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/README.md
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/README.md
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java
similarity index 99%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java
index 91faca2..d7ead05 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java
@@ -18,16 +18,15 @@
package org.apache.metron.parsers.sourcefire;
-import org.apache.metron.parsers.BasicParser;
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.metron.parsers.BasicParser;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@SuppressWarnings("serial")
public class BasicSourcefireParser extends BasicParser {
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
similarity index 98%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
index 5b62e85..77ebd18 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
@@ -23,15 +23,6 @@ import com.github.palindromicity.syslog.NilPolicy;
import com.github.palindromicity.syslog.SyslogParser;
import com.github.palindromicity.syslog.SyslogParserBuilder;
import com.github.palindromicity.syslog.dsl.SyslogFieldKeys;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.metron.parsers.BasicParser;
-import org.apache.metron.parsers.DefaultMessageParserResult;
-import org.apache.metron.parsers.interfaces.MessageParser;
-import org.apache.metron.parsers.interfaces.MessageParserResult;
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
@@ -41,12 +32,18 @@ import java.lang.invoke.MethodHandles;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.metron.parsers.DefaultMessageParserResult;
+import org.apache.metron.parsers.interfaces.MessageParser;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java
similarity index 99%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java
rename to metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java
index a58e0c9..44a98aa 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java
@@ -18,12 +18,11 @@
package org.apache.metron.parsers.websphere;
-import org.apache.metron.parsers.GrokParser;
-import org.json.simple.JSONObject;
-
import java.text.ParseException;
import java.util.Calendar;
import java.util.Iterator;
+import org.apache.metron.parsers.GrokParser;
+import org.json.simple.JSONObject;
public class GrokWebSphereParser extends GrokParser {
diff --git a/metron-platform/metron-parsers/src/main/resources/META-INF/LICENSE b/metron-platform/metron-parsing/metron-parsers/src/main/resources/META-INF/LICENSE
similarity index 100%
copy from metron-platform/metron-parsers/src/main/resources/META-INF/LICENSE
copy to metron-platform/metron-parsing/metron-parsers/src/main/resources/META-INF/LICENSE
diff --git a/metron-platform/metron-parsers/src/main/resources/META-INF/NOTICE b/metron-platform/metron-parsing/metron-parsers/src/main/resources/META-INF/NOTICE
similarity index 98%
copy from metron-platform/metron-parsers/src/main/resources/META-INF/NOTICE
copy to metron-platform/metron-parsing/metron-parsers/src/main/resources/META-INF/NOTICE
index d9212cd..456a20a 100644
--- a/metron-platform/metron-parsers/src/main/resources/META-INF/NOTICE
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/resources/META-INF/NOTICE
@@ -1,5 +1,5 @@
-metron-parsers
+metron-parsers-common
Copyright 2006-2016 The Apache Software Foundation
This product includes software developed at
diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/asa b/metron-platform/metron-parsing/metron-parsers/src/main/resources/patterns/asa
similarity index 100%
rename from metron-platform/metron-parsers/src/main/resources/patterns/asa
rename to metron-platform/metron-parsing/metron-parsers/src/main/resources/patterns/asa
diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/fireeye b/metron-platform/metron-parsing/metron-parsers/src/main/resources/patterns/fireeye
similarity index 100%
rename from metron-platform/metron-parsers/src/main/resources/patterns/fireeye
rename to metron-platform/metron-parsing/metron-parsers/src/main/resources/patterns/fireeye
diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/sourcefire b/metron-platform/metron-parsing/metron-parsers/src/main/resources/patterns/sourcefire
similarity index 100%
rename from metron-platform/metron-parsers/src/main/resources/patterns/sourcefire
rename to metron-platform/metron-parsing/metron-parsers/src/main/resources/patterns/sourcefire
diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/squid b/metron-platform/metron-parsing/metron-parsers/src/main/resources/patterns/squid
similarity index 100%
rename from metron-platform/metron-parsers/src/main/resources/patterns/squid
rename to metron-platform/metron-parsing/metron-parsers/src/main/resources/patterns/squid
diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/websphere b/metron-platform/metron-parsing/metron-parsers/src/main/resources/patterns/websphere
similarity index 100%
rename from metron-platform/metron-parsers/src/main/resources/patterns/websphere
rename to metron-platform/metron-parsing/metron-parsers/src/main/resources/patterns/websphere
diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/yaf b/metron-platform/metron-parsing/metron-parsers/src/main/resources/patterns/yaf
similarity index 100%
rename from metron-platform/metron-parsers/src/main/resources/patterns/yaf
rename to metron-platform/metron-parsing/metron-parsers/src/main/resources/patterns/yaf
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java
rename to metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java
rename to metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java
rename to metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java
rename to metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/asa/BasicAsaParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/asa/BasicAsaParserTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/asa/BasicAsaParserTest.java
rename to metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/asa/BasicAsaParserTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java
rename to metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
rename to metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java
rename to metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java
rename to metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java
rename to metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java
rename to metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java
rename to metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
rename to metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java
similarity index 98%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java
rename to metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java
index eb447d0..81a22be 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java
@@ -26,10 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
-import org.apache.log4j.Level;
-import org.apache.metron.parsers.GrokParser;
import org.apache.metron.parsers.interfaces.MessageParserResult;
-import org.apache.metron.test.utils.UnitTestHelper;
import org.json.simple.JSONObject;
import org.junit.Assert;
import org.junit.Before;
@@ -43,7 +40,7 @@ public class GrokWebSphereParserTest {
@Before
public void setup() {
parserConfig = new HashMap<>();
- parserConfig.put("grokPath", "../metron-parsers/src/main/resources/patterns/websphere");
+ parserConfig.put("grokPath", "src/main/resources/patterns/websphere");
parserConfig.put("patternLabel", "WEBSPHERE");
parserConfig.put("timestampField", "timestamp_string");
parserConfig.put("dateFormat", "yyyy MMM dd HH:mm:ss");
diff --git a/metron-platform/metron-parsers/src/test/resources/TestSchemas/BroSchema.json b/metron-platform/metron-parsing/metron-parsers/src/test/resources/TestSchemas/BroSchema.json
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/TestSchemas/BroSchema.json
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/TestSchemas/BroSchema.json
diff --git a/metron-platform/metron-parsers/src/test/resources/TestSchemas/IseSchema.json b/metron-platform/metron-parsing/metron-parsers/src/test/resources/TestSchemas/IseSchema.json
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/TestSchemas/IseSchema.json
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/TestSchemas/IseSchema.json
diff --git a/metron-platform/metron-parsers/src/test/resources/TestSchemas/LancopeSchema.json b/metron-platform/metron-parsing/metron-parsers/src/test/resources/TestSchemas/LancopeSchema.json
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/TestSchemas/LancopeSchema.json
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/TestSchemas/LancopeSchema.json
diff --git a/metron-platform/metron-parsers/src/test/resources/TestSchemas/PcapSchema.json b/metron-platform/metron-parsing/metron-parsers/src/test/resources/TestSchemas/PcapSchema.json
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/TestSchemas/PcapSchema.json
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/TestSchemas/PcapSchema.json
diff --git a/metron-platform/metron-parsers/src/test/resources/TestSchemas/SourcefireSchema.json b/metron-platform/metron-parsing/metron-parsers/src/test/resources/TestSchemas/SourcefireSchema.json
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/TestSchemas/SourcefireSchema.json
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/TestSchemas/SourcefireSchema.json
diff --git a/metron-platform/metron-parsers/src/test/resources/config/BasicFireEyeParserTest.config b/metron-platform/metron-parsing/metron-parsers/src/test/resources/config/BasicFireEyeParserTest.config
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/config/BasicFireEyeParserTest.config
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/config/BasicFireEyeParserTest.config
diff --git a/metron-platform/metron-parsers/src/test/resources/config/BasicIseParserTest.config b/metron-platform/metron-parsing/metron-parsers/src/test/resources/config/BasicIseParserTest.config
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/config/BasicIseParserTest.config
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/config/BasicIseParserTest.config
diff --git a/metron-platform/metron-parsers/src/test/resources/config/BasicLancopeParserTest.config b/metron-platform/metron-parsing/metron-parsers/src/test/resources/config/BasicLancopeParserTest.config
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/config/BasicLancopeParserTest.config
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/config/BasicLancopeParserTest.config
diff --git a/metron-platform/metron-parsers/src/test/resources/config/BasicPaloAltoFirewallParserTest.config b/metron-platform/metron-parsing/metron-parsers/src/test/resources/config/BasicPaloAltoFirewallParserTest.config
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/config/BasicPaloAltoFirewallParserTest.config
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/config/BasicPaloAltoFirewallParserTest.config
diff --git a/metron-platform/metron-parsers/src/test/resources/config/BasicSourcefireParserTest.config b/metron-platform/metron-parsing/metron-parsers/src/test/resources/config/BasicSourcefireParserTest.config
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/config/BasicSourcefireParserTest.config
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/config/BasicSourcefireParserTest.config
diff --git a/metron-platform/metron-parsers/src/test/resources/config/BroParserTest.config b/metron-platform/metron-parsing/metron-parsers/src/test/resources/config/BroParserTest.config
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/config/BroParserTest.config
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/config/BroParserTest.config
diff --git a/metron-platform/metron-parsers/src/test/resources/logData/FireEyeParserTest.txt b/metron-platform/metron-parsing/metron-parsers/src/test/resources/logData/FireEyeParserTest.txt
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/logData/FireEyeParserTest.txt
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/logData/FireEyeParserTest.txt
diff --git a/metron-platform/metron-parsers/src/test/resources/logData/IseParserTest.txt b/metron-platform/metron-parsing/metron-parsers/src/test/resources/logData/IseParserTest.txt
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/logData/IseParserTest.txt
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/logData/IseParserTest.txt
diff --git a/metron-platform/metron-parsers/src/test/resources/logData/LancopeParserTest.txt b/metron-platform/metron-parsing/metron-parsers/src/test/resources/logData/LancopeParserTest.txt
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/logData/LancopeParserTest.txt
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/logData/LancopeParserTest.txt
diff --git a/metron-platform/metron-parsers/src/test/resources/logData/SourcefireParserTest.txt b/metron-platform/metron-parsing/metron-parsers/src/test/resources/logData/SourcefireParserTest.txt
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/logData/SourcefireParserTest.txt
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/logData/SourcefireParserTest.txt
diff --git a/metron-platform/metron-parsers/src/test/resources/logData/multi_elb_log.txt b/metron-platform/metron-parsing/metron-parsers/src/test/resources/logData/multi_elb_log.txt
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/logData/multi_elb_log.txt
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/logData/multi_elb_log.txt
diff --git a/metron-platform/metron-parsers/src/test/resources/logData/multi_elb_with_errors_log.txt b/metron-platform/metron-parsing/metron-parsers/src/test/resources/logData/multi_elb_with_errors_log.txt
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/logData/multi_elb_with_errors_log.txt
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/logData/multi_elb_with_errors_log.txt
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.cef b/metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.cef
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.cef
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.cef
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.schema b/metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.schema
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.schema
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.schema
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef b/metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.json b/metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.json
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.json
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.json
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema b/metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.cef b/metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.cef
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.cef
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.cef
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.schema b/metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.schema
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.schema
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.schema
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.cef b/metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.cef
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.cef
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.cef
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.schema b/metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.schema
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.schema
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.schema
diff --git a/metron-platform/metron-parsers/src/test/resources/patterns/cisco_patterns b/metron-platform/metron-parsing/metron-parsers/src/test/resources/patterns/cisco_patterns
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/patterns/cisco_patterns
rename to metron-platform/metron-parsing/metron-parsers/src/test/resources/patterns/cisco_patterns
diff --git a/metron-platform/metron-parsing/metron-parsing-storm/README.md b/metron-platform/metron-parsing/metron-parsing-storm/README.md
new file mode 100644
index 0000000..69a09f4
--- /dev/null
+++ b/metron-platform/metron-parsing/metron-parsing-storm/README.md
@@ -0,0 +1,148 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# Parsers
+
+## Introduction
+Metron's parsers can be run in Storm topologies, complete with their own set of configuration options (e.g. parallelism). A script is provided to deploy a parser as a Storm topologoy.
+
+## Parser Configuration
+
+* `spoutParallelism` : The kafka spout parallelism (default to `1`). This can be overridden on the command line, and if there are multiple sensors should be in a comma separated list in the same order as the sensors.
+* `spoutNumTasks` : The number of tasks for the spout (default to `1`). This can be overridden on the command line, and if there are multiple sensors should be in a comma separated list in the same order as the sensors.
+* `parserParallelism` : The parser bolt parallelism (default to `1`). If there are multiple sensors, the last one's configuration will be used. This can be overridden on the command line.
+* `parserNumTasks` : The number of tasks for the parser bolt (default to `1`). If there are multiple sensors, the last one's configuration will be used. This can be overridden on the command line.
+* `errorWriterParallelism` : The error writer bolt parallelism (default to `1`). This can be overridden on the command line.
+* `errorWriterNumTasks` : The number of tasks for the error writer bolt (default to `1`). This can be overridden on the command line.
+* `numWorkers` : The number of workers to use in the topology (default is the storm default of `1`).
+* `numAckers` : The number of acker executors to use in the topology (default is the storm default of `1`).
+* `spoutConfig` : A map representing a custom spout config (this is a map). If there are multiple sensors, the configs will be merged with the last specified taking precedence. This can be overridden on the command line.
+* `stormConfig` : The storm config to use (this is a map). This can be overridden on the command line. If both are specified, they are merged with CLI properties taking precedence.
+
+# Starting the Parser Topology
+
+Starting a particular parser topology on a running Metron deployment is
+as easy as running the `start_parser_topology.sh` script located in
+`$METRON_HOME/bin`. This utility will allow you to configure and start
+the running topology assuming that the sensor specific parser configuration
+exists within zookeeper.
+
+The usage for `start_parser_topology.sh` is as follows:
+
+```
+usage: start_parser_topology.sh
+ -e,--extra_topology_options <JSON_FILE> Extra options in the form
+ of a JSON file with a map
+ for content.
+ -esc,--extra_kafka_spout_config <JSON_FILE> Extra spout config options
+ in the form of a JSON file
+ with a map for content.
+ Possible keys are:
+ retryDelayMaxMs,retryDelay
+ Multiplier,retryInitialDel
+ ayMs,stateUpdateIntervalMs
+ ,bufferSizeBytes,fetchMaxW
+ ait,fetchSizeBytes,maxOffs
+ etBehind,metricsTimeBucket
+ SizeInSecs,socketTimeoutMs
+ -ewnt,--error_writer_num_tasks <NUM_TASKS> Error Writer Num Tasks
+ -ewp,--error_writer_p <PARALLELISM_HINT> Error Writer Parallelism
+ Hint
+ -h,--help This screen
+ -iwnt,--invalid_writer_num_tasks <NUM_TASKS> Invalid Writer Num Tasks
+ -iwp,--invalid_writer_p <PARALLELISM_HINT> Invalid Message Writer Parallelism Hint
+ -k,--kafka <BROKER_URL> Kafka Broker URL
+ -ksp,--kafka_security_protocol <SECURITY_PROTOCOL> Kafka Security Protocol
+ -mt,--message_timeout <TIMEOUT_IN_SECS> Message Timeout in Seconds
+ -mtp,--max_task_parallelism <MAX_TASK> Max task parallelism
+ -na,--num_ackers <NUM_ACKERS> Number of Ackers
+ -nw,--num_workers <NUM_WORKERS> Number of Workers
+ -ot,--output_topic <KAFKA_TOPIC> Output Kafka Topic
+ -pnt,--parser_num_tasks <NUM_TASKS> Parser Num Tasks
+ -pp,--parser_p <PARALLELISM_HINT> Parser Parallelism Hint
+ -s,--sensor <SENSOR_TYPE> Sensor Type
+ -snt,--spout_num_tasks <NUM_TASKS> Spout Num Tasks
+ -sp,--spout_p <SPOUT_PARALLELISM_HINT> Spout Parallelism Hint
+ -t,--test <TEST> Run in Test Mode
+ -z,--zk <ZK_QUORUM> Zookeeper Quroum URL
+ (zk1:2181,zk2:2181,...
+```
+
+## The `--extra_kafka_spout_config` Option
+These options are intended to configure the Storm Kafka Spout more completely. These options can be
+specified in a JSON file containing a map associating the kafka spout configuration parameter to a value.
+The range of values possible to configure are:
+* `spout.pollTimeoutMs` - Specifies the time, in milliseconds, spent waiting in poll if data is not available. Default is 2s
+* `spout.firstPollOffsetStrategy` - Sets the offset used by the Kafka spout in the first poll to Kafka broker upon process start. One of
+ * `EARLIEST`
+ * `LATEST`
+ * `UNCOMMITTED_EARLIEST` - Last uncommitted and if offsets aren't found, defaults to earliest. NOTE: This is the default.
+ * `UNCOMMITTED_LATEST` - Last uncommitted and if offsets aren't found, defaults to latest.
+* `spout.offsetCommitPeriodMs` - Specifies the period, in milliseconds, the offset commit task is periodically called. Default is 15s.
+* `spout.maxUncommittedOffsets` - Defines the max number of polled offsets (records) that can be pending commit, before another poll can take place. Once this limit is reached, no more offsets (records) can be polled until the next successful commit(s) sets the number of pending offsets bellow the threshold. The default is 10,000,000.
+* `spout.maxRetries` - Defines the max number of retrials in case of tuple failure. The default is to retry forever, which means that no new records are committed until the previous polled records have been acked. This guarantees at once delivery of all the previously polled records. By specifying a finite value for maxRetries, the user decides to sacrifice guarantee of delivery for the previous polled records in favor of processing more records.
+* Any of the configs in the Consumer API for [Kafka 0.10.x](http://kafka.apache.org/0100/documentation.html#newconsumerconfigs)
+
+For instance, creating a JSON file which will set the offsets to `UNCOMMITTED_EARLIEST`
+```
+{
+ "spout.firstPollOffsetStrategy" : "UNCOMMITTED_EARLIEST"
+}
+```
+
+This would be loaded by passing the file as argument to `--extra_kafka_spout_config`
+
+## The `--extra_topology_options` Option
+
+These options are intended to be Storm configuration options and will live in
+a JSON file which will be loaded into the Storm config. For instance, if you wanted to set a storm property on
+the config called `topology.ticks.tuple.freq.secs` to 1000 and `storm.local.dir` to `/opt/my/path`
+you could create a file called `custom_config.json` containing
+```
+{
+ "topology.ticks.tuple.freq.secs" : 1000,
+ "storm.local.dir" : "/opt/my/path"
+}
+```
+and pass `--extra_topology_options custom_config.json` to `start_parser_topology.sh`.
+
+## Parser Topology
+The enrichment topology as started by the `$METRON_HOME/bin/start_parser_topology.sh`
+script uses a default of one executor per bolt. In a real production system, this should
+be customized by modifying the arguments sent to this utility.
+* Topology Wide
+ * `--num_workers` : The number of workers for the topology
+ * `--num_ackers` : The number of ackers for the topology
+* The Kafka Spout
+ * `--spout_num_tasks` : The number of tasks for the spout
+ * `--spout_p` : The parallelism hint for the spout
+ * Ensure that the spout has enough parallelism so that it can dedicate a worker per partition in your kafka topic.
+* The Parser Bolt
+ * `--parser_num_tasks` : The number of tasks for the parser bolt
+ * `--parser_p` : The parallelism hint for the spout
+ * This is bolt that gets the most processing, so ensure that it is configured with sufficient parallelism to match your throughput expectations.
+* The Error Message Writer Bolt
+ * `--error_writer_num_tasks` : The number of tasks for the error writer bolt
+ * `--error_writer_p` : The parallelism hint for the error writer bolt
+
+Finally, if workers and executors are new to you, the following might be of use to you:
+* [Understanding the Parallelism of a Storm Topology](http://www.michael-noll.com/blog/2012/10/16/understanding-the-parallelism-of-a-storm-topology/)
+
+## Parser Aggregation
+For performance reasons, multiple sensors can be aggregated into a single Storm topology. When this is done, there will be multiple Kafka spouts, but only a single parser bolt which will handle delegating to the correct parser as needed. There are some constraints around this, in particular regarding some configuration. Additionally, all sensors must flow to the same error topic. The Kafka topic is retrieved from the input Tuple itself.
+
+A worked example of this can be found in the [Parser Chaining use case](../../../use-cases/parser_chaining/README.md#aggregated-parsers-with-parser-chaining).
diff --git a/metron-platform/metron-parsing/metron-parsing-storm/pom.xml b/metron-platform/metron-parsing/metron-parsing-storm/pom.xml
new file mode 100644
index 0000000..50228b3
--- /dev/null
+++ b/metron-platform/metron-parsing/metron-parsing-storm/pom.xml
@@ -0,0 +1,264 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parsing</artifactId>
+ <version>0.7.0</version>
+ </parent>
+ <artifactId>metron-parsing-storm</artifactId>
+ <name>metron-parsing-storm</name>
+ <description>Metron code for running parsers on Apache Storm</description>
+ <url>https://metron.apache.org/</url>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ </properties>
+ <dependencies>
+ <!-- Metron dependencies -->
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parsers-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parsers</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+
+ <!-- Storm dependencies -->
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-rename-hack</artifactId>
+ <version>${global_storm_version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ <version>${global_storm_version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parsers-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-integration-test</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-enrichment</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-hbase</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${global_kafka_version}</version>
+ <classifier>test</classifier>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>${global_log4j_core_version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>${global_log4j_core_version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>${global_jar_version}</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${global_shade_version}</version>
+ <configuration>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <artifactSet>
+ <excludes>
+ <exclude>*slf4j*</exclude>
+ </excludes>
+ </artifactSet>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>org.apache.metron.jackson</shadedPattern>
+ </relocation>
+ </relocations>
+ <artifactSet>
+ <excludes>
+ <exclude>storm:storm-core:*</exclude>
+ <exclude>storm:storm-lib:*</exclude>
+ <exclude>org.slf4j.impl*</exclude>
+ <exclude>org.slf4j:slf4j-log4j*</exclude>
+ </excludes>
+ </artifactSet>
+ <transformers>
+ <transformer implementation="org.atteo.classindex.ClassIndexTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resources>
+ <resource>.yaml</resource>
+ <resource>LICENSE.txt</resource>
+ <resource>ASL2.0</resource>
+ <resource>NOTICE.txt</resource>
+ </resources>
+ </transformer>
+ <!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE -->
+ <!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
+ <addHeader>false</addHeader>
+ <projectName>${project.name}</projectName>
+ </transformer-->
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass></mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.atteo.classindex</groupId>
+ <artifactId>classindex-transformer</artifactId>
+ <version>${global_classindex_version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id> <!-- this is used for inheritance merges -->
+ <phase>package</phase> <!-- bind to the packaging phase -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ <resource>
+ <directory>src/main/patterns</directory>
+ </resource>
+ <resource>
+ <directory>src/test/resources</directory>
+ </resource>
+ </resources>
+ </build>
+</project>
diff --git a/metron-platform/metron-parsers/src/main/assembly/assembly.xml b/metron-platform/metron-parsing/metron-parsing-storm/src/main/assembly/assembly.xml
similarity index 100%
rename from metron-platform/metron-parsers/src/main/assembly/assembly.xml
rename to metron-platform/metron-parsing/metron-parsing-storm/src/main/assembly/assembly.xml
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
similarity index 99%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
rename to metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
index a9ee305..d5f8d1e 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
+++ b/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
@@ -18,6 +18,7 @@
package org.apache.metron.parsers.bolt;
+import com.github.benmanes.caffeine.cache.Cache;
import java.io.Serializable;
import java.lang.invoke.MethodHandles;
import java.util.Collections;
@@ -25,8 +26,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.function.Function;
-
-import com.github.benmanes.caffeine.cache.Cache;
import org.apache.metron.common.Constants;
import org.apache.metron.common.bolt.ConfiguredParserBolt;
import org.apache.metron.common.configuration.ParserConfigurations;
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterBolt.java b/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/bolt/WriterBolt.java
similarity index 99%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterBolt.java
rename to metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/bolt/WriterBolt.java
index fdfceda..abf3e46 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterBolt.java
+++ b/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/bolt/WriterBolt.java
@@ -19,6 +19,7 @@
package org.apache.metron.parsers.bolt;
import java.util.Collections;
+import java.util.Map;
import org.apache.metron.common.Constants;
import org.apache.metron.common.configuration.ParserConfigurations;
import org.apache.metron.common.error.MetronError;
@@ -32,8 +33,6 @@ import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Tuple;
import org.json.simple.JSONObject;
-import java.util.Map;
-
public class WriterBolt extends BaseRichBolt {
private WriterHandler handler;
private ParserConfigurations configuration;
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java b/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java
rename to metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/MergeAndShadeTransformer.java b/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/MergeAndShadeTransformer.java
similarity index 94%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/MergeAndShadeTransformer.java
rename to metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/MergeAndShadeTransformer.java
index eaadf71..eb7e2da 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/MergeAndShadeTransformer.java
+++ b/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/MergeAndShadeTransformer.java
@@ -18,18 +18,24 @@
package org.apache.metron.parsers.topology;
import com.google.common.base.Splitter;
-import org.apache.storm.daemon.JarTransformer;
-import org.apache.storm.hack.StormShadeTransformer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.*;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.lang.invoke.MethodHandles;
import java.util.HashSet;
import java.util.Set;
import java.util.jar.JarEntry;
import java.util.jar.JarInputStream;
import java.util.jar.JarOutputStream;
+import org.apache.storm.daemon.JarTransformer;
+import org.apache.storm.hack.StormShadeTransformer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This is a storm jar transformer that will add in additional jars pulled from an
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java b/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
rename to metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java b/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java
similarity index 99%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java
rename to metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java
index eb39f89..d142279 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java
+++ b/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java
@@ -351,7 +351,8 @@ public class ParserTopologyCLI {
if(ParserOptions.SPOUT_PARALLELISM.has(cmd)) {
// Handle the case where there's only one and we can default reasonably
if( parserConfigs.size() == 1) {
- return Collections.singletonList(Integer.parseInt(ParserOptions.SPOUT_PARALLELISM.get(cmd, "1")));
+ return Collections.singletonList(Integer.parseInt(
+ ParserOptions.SPOUT_PARALLELISM.get(cmd, "1")));
}
// Handle the multiple explicitly passed spout parallelism's case.
@@ -380,7 +381,8 @@ public class ParserTopologyCLI {
if(ParserOptions.SPOUT_NUM_TASKS.has(cmd)) {
// Handle the case where there's only one and we can default reasonably
if( parserConfigs.size() == 1) {
- return Collections.singletonList(Integer.parseInt(ParserOptions.SPOUT_NUM_TASKS.get(cmd, "1")));
+ return Collections.singletonList(Integer.parseInt(
+ ParserOptions.SPOUT_NUM_TASKS.get(cmd, "1")));
}
// Handle the multiple explicitly passed spout parallelism's case.
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/Arg.java b/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/config/Arg.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/Arg.java
rename to metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/config/Arg.java
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java b/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java
similarity index 99%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java
rename to metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java
index 8d98922..ccb07bf 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java
+++ b/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java
@@ -18,14 +18,12 @@
package org.apache.metron.parsers.topology.config;
-import org.apache.storm.Config;
-import org.apache.commons.io.FileUtils;
-import org.apache.metron.common.utils.JSONUtils;
-
import java.io.File;
import java.io.IOException;
-import java.util.Map;
import java.util.function.Function;
+import org.apache.commons.io.FileUtils;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.storm.Config;
public class ConfigHandlers {
public static class SetNumWorkersHandler implements Function<Arg, Config> {
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ValueSupplier.java b/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/config/ValueSupplier.java
similarity index 100%
rename from metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ValueSupplier.java
rename to metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/config/ValueSupplier.java
diff --git a/metron-platform/metron-parsers/src/main/resources/META-INF/LICENSE b/metron-platform/metron-parsing/metron-parsing-storm/src/main/resources/META-INF/LICENSE
similarity index 100%
rename from metron-platform/metron-parsers/src/main/resources/META-INF/LICENSE
rename to metron-platform/metron-parsing/metron-parsing-storm/src/main/resources/META-INF/LICENSE
diff --git a/metron-platform/metron-parsers/src/main/resources/META-INF/NOTICE b/metron-platform/metron-parsing/metron-parsing-storm/src/main/resources/META-INF/NOTICE
similarity index 98%
rename from metron-platform/metron-parsers/src/main/resources/META-INF/NOTICE
rename to metron-platform/metron-parsing/metron-parsing-storm/src/main/resources/META-INF/NOTICE
index d9212cd..c773ab7 100644
--- a/metron-platform/metron-parsers/src/main/resources/META-INF/NOTICE
+++ b/metron-platform/metron-parsing/metron-parsing-storm/src/main/resources/META-INF/NOTICE
@@ -1,5 +1,5 @@
-metron-parsers
+metron-parsing-storm
Copyright 2006-2016 The Apache Software Foundation
This product includes software developed at
diff --git a/metron-platform/metron-parsers/src/main/scripts/start_parser_topology.sh b/metron-platform/metron-parsing/metron-parsing-storm/src/main/scripts/start_parser_topology.sh
similarity index 97%
rename from metron-platform/metron-parsers/src/main/scripts/start_parser_topology.sh
rename to metron-platform/metron-parsing/metron-parsing-storm/src/main/scripts/start_parser_topology.sh
index bbbeaff..db777dc 100755
--- a/metron-platform/metron-parsers/src/main/scripts/start_parser_topology.sh
+++ b/metron-platform/metron-parsing/metron-parsing-storm/src/main/scripts/start_parser_topology.sh
@@ -18,7 +18,7 @@
#
METRON_VERSION=${project.version}
METRON_HOME=/usr/metron/$METRON_VERSION
-TOPOLOGY_JAR=metron-parsers-$METRON_VERSION-uber.jar
+TOPOLOGY_JAR=metron-parsing-storm-$METRON_VERSION-uber.jar
PARSER_CONTRIB=${PARSER_CONTRIB:-$METRON_HOME/parser_contrib}
EXTRA_ARGS="$@"
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
rename to metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
index 9f58d1c..0166a63 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
+++ b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
@@ -25,8 +25,8 @@ import org.apache.metron.common.error.MetronError;
import org.apache.metron.common.message.MessageGetStrategy;
import org.apache.metron.common.message.metadata.RawMessage;
import org.apache.metron.parsers.DefaultParserRunnerResults;
-import org.apache.metron.parsers.ParserRunnerResults;
import org.apache.metron.parsers.ParserRunnerImpl;
+import org.apache.metron.parsers.ParserRunnerResults;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration;
import org.apache.metron.test.bolt.BaseBoltTest;
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/WriterBoltTest.java b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/bolt/WriterBoltTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/WriterBoltTest.java
rename to metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/bolt/WriterBoltTest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapWrappedQueryIntegrationTest.java b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/StormParserIntegrationTest.java
similarity index 56%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapWrappedQueryIntegrationTest.java
rename to metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/StormParserIntegrationTest.java
index 569e175..03d55bb 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapWrappedQueryIntegrationTest.java
+++ b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/StormParserIntegrationTest.java
@@ -1,4 +1,4 @@
-/**
+package org.apache.metron.parsers.integration;/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,17 +15,34 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.metron.parsers.integration;
-
-import org.apache.metron.parsers.integration.validation.SampleDataValidation;
import java.util.ArrayList;
import java.util.List;
+import org.apache.metron.parsers.integration.validation.ParserDriver;
+import org.apache.metron.parsers.integration.validation.SampleDataValidation;
+import org.apache.metron.parsers.integration.validation.StormParserDriver;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
-public class JSONMapWrappedQueryIntegrationTest extends ParserIntegrationTest {
- @Override
- String getSensorType() {
- return "jsonMapWrappedQuery";
+@RunWith(Parameterized.class)
+public class StormParserIntegrationTest extends ParserIntegrationTest {
+
+ @Parameters(name = "{index}: sensorType={0}")
+ public static Iterable<String> data() {
+ return sensorTypes;
+ }
+
+ @Parameter
+ public String sensorType;
+
+ @Test
+ public void test() throws Exception {
+ ParserDriver driver = new StormParserDriver(sensorType, readSensorConfig(sensorType),
+ readGlobalConfig());
+ runTest(driver);
}
@Override
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java
rename to metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/validation/StormParserDriver.java
similarity index 76%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
rename to metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/validation/StormParserDriver.java
index 31d87a7..dfc454f 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
+++ b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/validation/StormParserDriver.java
@@ -15,28 +15,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.metron.parsers.integration;
+package org.apache.metron.parsers.integration.validation;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
-import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashSet;
import java.util.List;
-
import org.apache.commons.lang.SerializationUtils;
import org.apache.metron.common.configuration.ParserConfigurations;
-import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.common.writer.MessageWriter;
import org.apache.metron.integration.ProcessorResult;
-import org.apache.metron.parsers.ParserRunner;
-import org.apache.metron.parsers.ParserRunnerImpl;
import org.apache.metron.parsers.bolt.ParserBolt;
import org.apache.metron.parsers.bolt.WriterHandler;
import org.apache.storm.task.OutputCollector;
@@ -45,9 +38,11 @@ import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class ParserDriver implements Serializable {
- private static final Logger LOG = LoggerFactory.getLogger(ParserBolt.class);
- public static class CollectingWriter implements MessageWriter<JSONObject>{
+public class StormParserDriver extends ParserDriver {
+ private static final Logger LOG = LoggerFactory.getLogger(StormParserDriver.class);
+
+ public static class CollectingWriter implements MessageWriter<JSONObject> {
+
List<byte[]> output;
public CollectingWriter(List<byte[]> output) {
this.output = output;
@@ -109,21 +104,23 @@ public class ParserDriver implements Serializable {
}
}
-
- private ParserConfigurations config;
- private String sensorType;
- private ParserRunner parserRunner;
-
- public ParserDriver(String sensorType, String parserConfig, String globalConfig) throws IOException {
- SensorParserConfig sensorParserConfig = SensorParserConfig.fromBytes(parserConfig.getBytes());
- this.sensorType = sensorType == null ? sensorParserConfig.getSensorTopic() : sensorType;
- config = new ParserConfigurations();
- config.updateSensorParserConfig(this.sensorType, SensorParserConfig.fromBytes(parserConfig.getBytes()));
- config.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER));
-
- parserRunner = new ParserRunnerImpl(new HashSet<String>() {{
- add(sensorType);
- }});
+//
+// private ParserConfigurations config;
+// private String sensorType;
+// private ParserRunner parserRunner;
+//
+// public ParserDriver(String sensorType, String parserConfig, String globalConfig) throws IOException {
+// SensorParserConfig sensorParserConfig = SensorParserConfig.fromBytes(parserConfig.getBytes());
+// this.sensorType = sensorType == null ? sensorParserConfig.getSensorTopic() : sensorType;
+// config = new ParserConfigurations();
+// config.updateSensorParserConfig(this.sensorType, SensorParserConfig.fromBytes(parserConfig.getBytes()));
+// config.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER));
+//
+// parserRunner = new ParserRunnerImpl(new HashSet<String>() {{
+// add(sensorType);
+// }});
+ public StormParserDriver(String sensorType, String parserConfig, String globalConfig) throws IOException {
+ super(sensorType, parserConfig, globalConfig);
}
public ProcessorResult<List<byte[]>> run(Iterable<byte[]> in) {
@@ -144,5 +141,4 @@ public class ParserDriver implements Serializable {
when(ret.getBinary(eq(0))).thenReturn(record);
return ret;
}
-
}
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/topology/ParserTopologyCLITest.java b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/topology/ParserTopologyCLITest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/topology/ParserTopologyCLITest.java
rename to metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/topology/ParserTopologyCLITest.java
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
similarity index 97%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
rename to metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
index 788df2d..467429c 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
+++ b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
@@ -19,7 +19,15 @@
package org.apache.metron.writers.integration;
import com.google.common.collect.ImmutableList;
+import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
@@ -45,15 +53,6 @@ import org.apache.metron.parsers.integration.components.ParserTopologyComponent;
import org.junit.Assert;
import org.junit.Test;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-
public class SimpleHbaseEnrichmentWriterIntegrationTest extends BaseIntegrationTest {
/**
@@ -102,9 +101,11 @@ public class SimpleHbaseEnrichmentWriterIntegrationTest extends BaseIntegrationT
SensorParserConfig parserConfig = JSONUtils.INSTANCE.load(parserConfigJSON, SensorParserConfig.class);
+ System.out.println("Workspace: " + System.getProperty("user.dir"));
+ System.out.println("Configs path: ../" + TestConstants.SAMPLE_CONFIG_PATH);
ConfigUploadComponent configUploadComponent = new ConfigUploadComponent()
.withTopologyProperties(topologyProperties)
- .withGlobalConfigsPath(TestConstants.SAMPLE_CONFIG_PATH)
+ .withGlobalConfigsPath("../" + TestConstants.SAMPLE_CONFIG_PATH)
.withParserSensorConfig(sensorType, parserConfig);
ParserTopologyComponent parserTopologyComponent = new ParserTopologyComponent.Builder()
diff --git a/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/StormEnvelopedParserIntegrationTest.java b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/StormEnvelopedParserIntegrationTest.java
new file mode 100644
index 0000000..4baa353
--- /dev/null
+++ b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/StormEnvelopedParserIntegrationTest.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.writers.integration;
+
+import java.io.IOException;
+import org.apache.metron.parsers.integration.EnvelopedParserIntegrationTest;
+import org.apache.metron.parsers.integration.validation.ParserDriver;
+import org.apache.metron.parsers.integration.validation.StormParserDriver;
+import org.junit.Test;
+
+public class StormEnvelopedParserIntegrationTest extends EnvelopedParserIntegrationTest {
+
+ @Test
+ public void testEnvelopedData() throws IOException {
+ ParserDriver driver = new StormParserDriver("test", parserConfig_default, "{}");
+ super.testEnvelopedData(driver);
+ }
+
+ @Test
+ public void testEnvelopedData_withMetadataPrefix() throws IOException {
+ ParserDriver driver = new StormParserDriver("test", parserConfig_withPrefix, "{}");
+ super.testEnvelopedData_withMetadataPrefix(driver);
+ }
+
+ @Test
+ public void testEnvelopedData_noMergeMetadata() throws IOException {
+ ParserDriver driver = new StormParserDriver("test", parserConfig_nomerge, "{}");
+ super.testEnvelopedData_noMergeMetadata(driver);
+ }
+
+ @Test
+ public void testCiscoPixEnvelopingCisco302020() throws Exception {
+ ParserDriver syslogDriver = new StormParserDriver("ciscoPix", ciscoPixSyslogConfig, "{}");
+ ParserDriver driver = new StormParserDriver("cisco302020", cisco302020Config, "{}");
+ super.testCiscoPixEnvelopingCisco302020(syslogDriver, driver);
+ }
+}
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
similarity index 100%
rename from metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
rename to metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
diff --git a/metron-platform/metron-parsers/src/test/resources/log4j.properties b/metron-platform/metron-parsing/metron-parsing-storm/src/test/resources/log4j.properties
similarity index 100%
rename from metron-platform/metron-parsers/src/test/resources/log4j.properties
rename to metron-platform/metron-parsing/metron-parsing-storm/src/test/resources/log4j.properties
diff --git a/metron-platform/metron-parsing/pom.xml b/metron-platform/metron-parsing/pom.xml
new file mode 100644
index 0000000..f349551
--- /dev/null
+++ b/metron-platform/metron-parsing/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>metron-parsing</artifactId>
+ <packaging>pom</packaging>
+ <name>metron-parsing</name>
+ <parent>
+ <artifactId>metron-platform</artifactId>
+ <groupId>org.apache.metron</groupId>
+ <version>0.7.0</version>
+ </parent>
+ <description>Metron parsers and platform support</description>
+ <url>https://metron.apache.org/</url>
+ <scm>
+ <connection>scm:git:https://gitbox.apache.org/repos/asf/metron.git</connection>
+ <developerConnection>scm:git:https://gitbox.apache.org/repos/asf/metron.git</developerConnection>
+ <tag>HEAD</tag>
+ <url>https://gitbox.apache.org/repos/asf/metron.git</url>
+ </scm>
+
+ <licenses>
+ <license>
+ <name>The Apache Software License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ </license>
+ </licenses>
+ <modules>
+ <module>metron-parsers</module>
+ <module>metron-parsers-common</module>
+ <module>metron-parsing-storm</module>
+ </modules>
+</project>
\ No newline at end of file
diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/TestConstants.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/TestConstants.java
index 2b39f9d..53e49db 100644
--- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/TestConstants.java
+++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/TestConstants.java
@@ -20,7 +20,8 @@ package org.apache.metron;
public class TestConstants {
public final static String SAMPLE_CONFIG_PATH = "../metron-integration-test/src/main/config/zookeeper/";
- public final static String PARSER_CONFIGS_PATH = "../metron-parsers/src/main/config/zookeeper/";
+ public final static String PARSER_CONFIGS_PATH = "../metron-parsing/metron-parsers/src/main/config/zookeeper/";
+ public final static String PARSER_COMMON_CONFIGS_PATH = "../metron-parsing/metron-parsers-common/src/main/config/zookeeper/";
public final static String ENRICHMENTS_CONFIGS_PATH = "../metron-enrichment/src/main/config/zookeeper/";
public final static String SAMPLE_DATA_PATH = "../metron-integration-test/src/main/sample/data/";
public final static String SAMPLE_DATA_INPUT_PATH = "../metron-integration-test/src/main/sample/data/yaf/raw/";
diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/SampleDataUtils.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/SampleDataUtils.java
index 0e3e4e6..bc2561b 100644
--- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/SampleDataUtils.java
+++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/SampleDataUtils.java
@@ -25,8 +25,8 @@ import java.io.FileNotFoundException;
public class SampleDataUtils {
- public static String getSampleDataPath(String sensorType, TestDataType testDataType) throws FileNotFoundException {
- File sensorSampleDataPath = new File(TestConstants.SAMPLE_DATA_PATH, sensorType);
+ public static String getSampleDataPath(String pathPrefix, String sensorType, TestDataType testDataType) throws FileNotFoundException {
+ File sensorSampleDataPath = new File(pathPrefix + "/" + TestConstants.SAMPLE_DATA_PATH, sensorType);
if (sensorSampleDataPath.exists() && sensorSampleDataPath.isDirectory()) {
File sampleDataPath = new File(sensorSampleDataPath, testDataType.getDirectoryName());
if (sampleDataPath.exists() && sampleDataPath.isDirectory()) {
@@ -38,4 +38,8 @@ public class SampleDataUtils {
}
throw new FileNotFoundException("Could not find data in " + TestConstants.SAMPLE_DATA_PATH + sensorType + "/" + testDataType.getDirectoryName());
}
+
+ public static String getSampleDataPath(String sensorType, TestDataType testDataType) throws FileNotFoundException {
+ return getSampleDataPath("", sensorType, testDataType);
+ }
}
diff --git a/metron-platform/metron-writer/README.md b/metron-platform/metron-writer/README.md
index 21cdbca..bbec39b 100644
--- a/metron-platform/metron-writer/README.md
+++ b/metron-platform/metron-writer/README.md
@@ -29,7 +29,7 @@ the topic for the message.
The configuration for this writer is held in the individual Sensor
Configurations:
* [Enrichment](../metron-enrichment/README.md#sensor-enrichment-configuration) under the `config` element
-* [Parsers](../metron-parsers/README.md#parser-configuration) in the `parserConfig` element
+* [Parsers](../metron-parsers-common/README.md#parser-configuration) in the `parserConfig` element
* Profiler - Unsupported currently
In each of these, the kafka writer can be configured via a map which has
diff --git a/metron-platform/pom.xml b/metron-platform/pom.xml
index 0eab085..c793c4c 100644
--- a/metron-platform/pom.xml
+++ b/metron-platform/pom.xml
@@ -45,7 +45,6 @@
<module>metron-common</module>
<module>metron-enrichment</module>
<module>metron-solr</module>
- <module>metron-parsers</module>
<module>metron-job</module>
<module>metron-pcap-backend</module>
<module>metron-data-management</module>
@@ -61,7 +60,8 @@
<module>metron-storm-kafka</module>
<module>metron-storm-kafka-override</module>
<module>metron-zookeeper</module>
- </modules>
+ <module>metron-parsing</module>
+ </modules>
<dependencies>
<dependency>
<groupId>org.slf4j</groupId>
diff --git a/pom.xml b/pom.xml
index 081adde..c352813 100644
--- a/pom.xml
+++ b/pom.xml
@@ -360,8 +360,8 @@
<exclude>checkstyle.xml</exclude>
<!-- Alerts UI e2e test data -->
<exclude>e2e/mock-data/alerts_ui_e2e_index.data</exclude>
- <!-- retry files generated by Ansible during a failure -->
- <exclude>**/*.retry</exclude>
+ <!-- retry files generated by Ansible during a failure -->
+ <exclude>**/*.retry</exclude>
</excludes>
</configuration>
</plugin>
diff --git a/site-book/bin/generate-md.sh b/site-book/bin/generate-md.sh
index 966ee06..60549f8 100755
--- a/site-book/bin/generate-md.sh
+++ b/site-book/bin/generate-md.sh
@@ -64,7 +64,7 @@ RESOURCE_LIST=(
metron-deployment/readme-images/enable-kerberos-started.png
metron-deployment/readme-images/enable-kerberos.png
metron-platform/metron-job/metron-job_state_statechart_diagram.svg
- metron-platform/metron-parsers/parser_arch.png
+ metron-platform/metron-parsing/metron-parsers-common/parser_arch.png
metron-platform/metron-indexing/indexing_arch.png
metron-platform/metron-enrichment/enrichment_arch.png
metron-analytics/metron-maas-service/maas_arch.png
@@ -96,8 +96,8 @@ HREF_REWRITE_LIST=(
metron-platform/metron-enrichment/README.md 's#(enrichment_arch.png)#(../../images/enrichment_arch.png)#g'
metron-platform/metron-indexing/README.md 's#(indexing_arch.png)#(../../images/indexing_arch.png)#g'
metron-platform/metron-job/README.md 's#(metron-job_state_statechart_diagram.svg)#(../../images/metron-job_state_statechart_diagram.svg)#g'
- metron-platform/metron-parsers/README.md 's#(parser_arch.png)#(../../images/parser_arch.png)#g'
- metron-platform/metron-parsers/ParserChaining.md 's#(../../use-cases/parser_chaining/message_routing_high_level.svg)#(../../images/message_routing_high_level.svg)#g'
+ metron-platform/metron-parsing/metron-parsers-common/README.md 's#(parser_arch.png)#(../../images/parser_arch.png)#g'
+ metron-platform/metron-parsing/metron-parsers-common/ParserChaining.md 's#(../../use-cases/parser_chaining/message_routing_high_level.svg)#(../../images/message_routing_high_level.svg)#g'
metron-analytics/metron-maas-service/README.md 's#(maas_arch.png)#(../../images/maas_arch.png)#g'
metron-contrib/metron-performance/README.md 's#(performance_measurement.png)#(../../images/performance_measurement.png)#g'
use-cases/forensic_clustering/README.md 's#(find_alerts.png)#(../../images/find_alerts.png)#g'
diff --git a/site/current-book/metron-platform/metron-parsers/index.html b/site/current-book/metron-platform/metron-parsers/index.html
index 15440a1..06cfb47 100644
--- a/site/current-book/metron-platform/metron-parsers/index.html
+++ b/site/current-book/metron-platform/metron-parsers/index.html
@@ -665,7 +665,7 @@ HH:mm:ss', MAP_GET(dc, dc2tz, 'UTC') )"
<p>Java parser adapters are intended for higher-velocity topologies and are not easily changed or extended. As the adoption of Metron continues we plan on extending our library of Java adapters to process more log formats. As of this moment the Java adapters included with Metron are:</p>
<ul>
-<li>org.apache.metron.parsers.ise.BasicIseParser : Parse ISE messages</li>
+<li>BasicIseParser : Parse ISE messages</li>
<li>org.apache.metron.parsers.bro.BasicBroParser : Parse Bro messages</li>
<li>org.apache.metron.parsers.sourcefire.BasicSourcefireParser : Parse Sourcefire messages</li>
<li>org.apache.metron.parsers.lancope.BasicLancopeParser : Parse Lancope messages</li>