You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2019/08/23 21:42:20 UTC
[metron] branch master updated: METRON-2149 Shaded jar classifier
is not consistent (merrimanr) closes apache/metron#1436
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new e6ca3c7 METRON-2149 Shaded jar classifier is not consistent (merrimanr) closes apache/metron#1436
e6ca3c7 is described below
commit e6ca3c71d8120516afcf816be2cda81093088b72
Author: merrimanr <me...@gmail.com>
AuthorDate: Fri Aug 23 16:42:01 2019 -0500
METRON-2149 Shaded jar classifier is not consistent (merrimanr) closes apache/metron#1436
---
metron-analytics/metron-maas-common/pom.xml | 1 +
metron-analytics/metron-maas-service/pom.xml | 6 ++
metron-analytics/metron-profiler-client/pom.xml | 8 ++
.../src/main/assembly/assembly.xml | 2 +-
metron-analytics/metron-profiler-common/pom.xml | 6 ++
metron-analytics/metron-profiler-repl/pom.xml | 8 ++
.../src/main/assembly/assembly.xml | 2 +-
metron-analytics/metron-profiler-spark/pom.xml | 3 +
.../src/main/assembly/assembly.xml | 2 +-
.../src/main/scripts/start_batch_profiler.sh | 4 +-
metron-analytics/metron-profiler-storm/pom.xml | 8 ++
.../src/main/scripts/start_profiler_topology.sh | 3 +-
metron-analytics/metron-statistics/pom.xml | 3 +
metron-contrib/metron-performance/pom.xml | 8 ++
.../src/main/assembly/assembly.xml | 2 +-
.../src/main/scripts/load_tool.sh | 2 +-
.../common-services/METRON/CURRENT/metainfo.xml | 3 +
.../packaging/docker/rpm-docker/SPECS/metron.spec | 35 +++++++--
.../packaging/docker/rpm-docker/pom.xml | 6 ++
metron-interface/metron-rest/pom.xml | 22 ++++++
.../metron-rest/src/main/assembly/assembly.xml | 2 +-
.../metron-rest/src/main/scripts/metron-rest.sh | 2 +
metron-platform/elasticsearch-shaded/pom.xml | 2 +
.../metron-common-storm/pom.xml | 6 ++
metron-platform/metron-common/pom.xml | 51 +++++++------
.../metron-common/src/main/assembly/assembly.xml | 2 +-
.../metron-common/src/main/scripts/stellar | 5 +-
.../src/main/scripts/zk_load_configs.sh | 3 +-
metron-platform/metron-data-management/pom.xml | 89 ++++++++++++++--------
.../src/main/assembly/assembly.xml | 2 +-
.../src/main/scripts/flatfile_loader.sh | 7 +-
.../src/main/scripts/flatfile_summarizer.sh | 7 +-
.../src/main/scripts/maxmind_enrichment_load.sh | 10 +--
.../src/main/scripts/prune_hdfs_files.sh | 2 +-
.../src/main/scripts/threatintel_bulk_prune.sh | 2 +-
.../src/main/scripts/threatintel_taxii_load.sh | 2 +-
.../metron-elasticsearch-common/pom.xml | 27 ++++++-
.../metron-elasticsearch-storm/pom.xml | 19 +++++
.../main/scripts/start_elasticsearch_topology.sh | 5 +-
.../metron-enrichment-common/pom.xml | 1 +
.../metron-enrichment-storm/pom.xml | 6 ++
.../src/main/scripts/start_enrichment_topology.sh | 3 +-
metron-platform/metron-hbase-client/pom.xml | 2 +
metron-platform/metron-hbase-server/pom.xml | 6 ++
.../metron-indexing/metron-indexing-common/pom.xml | 11 +--
.../metron-indexing/metron-indexing-storm/pom.xml | 6 ++
.../src/main/scripts/start_hdfs_topology.sh | 3 +-
metron-platform/metron-integration-test/pom.xml | 1 +
metron-platform/metron-management/pom.xml | 3 +
.../src/main/assembly/assembly.xml | 2 +-
.../metron-parsing/metron-parsers-common/pom.xml | 6 ++
.../metron-parsing/metron-parsers/pom.xml | 6 ++
.../metron-parsing/metron-parsing-storm/pom.xml | 6 ++
.../src/main/scripts/start_parser_topology.sh | 8 +-
metron-platform/metron-pcap-backend/pom.xml | 40 ++++++----
.../src/main/assembly/assembly.xml | 2 +-
.../src/main/scripts/pcap_inspector.sh | 2 +-
.../src/main/scripts/pcap_query.sh | 2 +-
.../src/main/scripts/start_pcap_topology.sh | 6 +-
metron-platform/metron-pcap/pom.xml | 14 ++++
.../metron-solr/metron-solr-common/pom.xml | 6 ++
.../metron-solr/metron-solr-storm/pom.xml | 6 ++
.../src/main/scripts/start_solr_topology.sh | 3 +-
.../metron-storm-kafka-override/pom.xml | 2 +
metron-platform/metron-storm-kafka/pom.xml | 1 +
.../metron-writer/metron-writer-common/pom.xml | 45 ++++++-----
.../metron-writer/metron-writer-storm/pom.xml | 6 ++
metron-stellar/stellar-common/pom.xml | 10 ++-
.../stellar-common/src/main/assembly/assembly.xml | 12 ---
.../src/main/scripts/deployed/stellar | 4 +-
metron-stellar/stellar-zeppelin/pom.xml | 1 +
71 files changed, 444 insertions(+), 167 deletions(-)
diff --git a/metron-analytics/metron-maas-common/pom.xml b/metron-analytics/metron-maas-common/pom.xml
index 3d89800..b404111 100644
--- a/metron-analytics/metron-maas-common/pom.xml
+++ b/metron-analytics/metron-maas-common/pom.xml
@@ -43,6 +43,7 @@
<artifactId>hadoop-auth</artifactId>
</exclusion>
</exclusions>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
diff --git a/metron-analytics/metron-maas-service/pom.xml b/metron-analytics/metron-maas-service/pom.xml
index 2bb06fe..a1671b5 100644
--- a/metron-analytics/metron-maas-service/pom.xml
+++ b/metron-analytics/metron-maas-service/pom.xml
@@ -44,6 +44,12 @@
<version>${project.parent.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>de.javakaffee</groupId>
<artifactId>kryo-serializers</artifactId>
<version>${global_kryo_serializers_version}</version>
diff --git a/metron-analytics/metron-profiler-client/pom.xml b/metron-analytics/metron-profiler-client/pom.xml
index 04d9740..876e1d6 100644
--- a/metron-analytics/metron-profiler-client/pom.xml
+++ b/metron-analytics/metron-profiler-client/pom.xml
@@ -59,6 +59,12 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava_version}</version>
@@ -143,6 +149,8 @@
<artifactId>maven-shade-plugin</artifactId>
<version>${global_shade_version}</version>
<configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
<createDependencyReducedPom>false</createDependencyReducedPom>
<artifactSet>
<excludes>
diff --git a/metron-analytics/metron-profiler-client/src/main/assembly/assembly.xml b/metron-analytics/metron-profiler-client/src/main/assembly/assembly.xml
index b0ab070..86c27e5 100644
--- a/metron-analytics/metron-profiler-client/src/main/assembly/assembly.xml
+++ b/metron-analytics/metron-profiler-client/src/main/assembly/assembly.xml
@@ -28,7 +28,7 @@
<fileSet>
<directory>${project.basedir}/target</directory>
<includes>
- <include>${project.artifactId}-${project.version}.jar</include>
+ <include>${project.artifactId}-${project.version}-uber.jar</include>
</includes>
<outputDirectory>lib</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
diff --git a/metron-analytics/metron-profiler-common/pom.xml b/metron-analytics/metron-profiler-common/pom.xml
index bd2e5b6..13d5abe 100644
--- a/metron-analytics/metron-profiler-common/pom.xml
+++ b/metron-analytics/metron-profiler-common/pom.xml
@@ -48,6 +48,12 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${global_hbase_version}</version>
diff --git a/metron-analytics/metron-profiler-repl/pom.xml b/metron-analytics/metron-profiler-repl/pom.xml
index 86e917d..f3714b8 100644
--- a/metron-analytics/metron-profiler-repl/pom.xml
+++ b/metron-analytics/metron-profiler-repl/pom.xml
@@ -77,6 +77,12 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>${global_log4j_core_version}</version>
@@ -96,6 +102,8 @@
<artifactId>maven-shade-plugin</artifactId>
<version>${global_shade_version}</version>
<configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
<createDependencyReducedPom>true</createDependencyReducedPom>
</configuration>
<executions>
diff --git a/metron-analytics/metron-profiler-repl/src/main/assembly/assembly.xml b/metron-analytics/metron-profiler-repl/src/main/assembly/assembly.xml
index fa6f0c1..6f883d3 100644
--- a/metron-analytics/metron-profiler-repl/src/main/assembly/assembly.xml
+++ b/metron-analytics/metron-profiler-repl/src/main/assembly/assembly.xml
@@ -27,7 +27,7 @@
<fileSet>
<directory>${project.basedir}/target</directory>
<includes>
- <include>${project.artifactId}-${project.version}.jar</include>
+ <include>${project.artifactId}-${project.version}-uber.jar</include>
</includes>
<outputDirectory>lib</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
diff --git a/metron-analytics/metron-profiler-spark/pom.xml b/metron-analytics/metron-profiler-spark/pom.xml
index b046cc4..a087663 100644
--- a/metron-analytics/metron-profiler-spark/pom.xml
+++ b/metron-analytics/metron-profiler-spark/pom.xml
@@ -63,6 +63,7 @@
<groupId>org.apache.metron</groupId>
<artifactId>stellar-common</artifactId>
<version>${project.parent.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
@@ -158,6 +159,8 @@
<artifactId>maven-shade-plugin</artifactId>
<version>${global_shade_version}</version>
<configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
<createDependencyReducedPom>true</createDependencyReducedPom>
</configuration>
<executions>
diff --git a/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml b/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml
index 02f97eb..cb707be 100644
--- a/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml
+++ b/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml
@@ -51,7 +51,7 @@
<fileSet>
<directory>${project.basedir}/target</directory>
<includes>
- <include>${project.artifactId}-${project.version}.jar</include>
+ <include>${project.artifactId}-${project.version}-uber.jar</include>
</includes>
<outputDirectory>lib</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
diff --git a/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh b/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh
index 33d9aa4..134af6e 100644
--- a/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh
+++ b/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh
@@ -18,7 +18,8 @@
#
METRON_VERSION=${project.version}
METRON_HOME=/usr/metron/${METRON_VERSION}
-PROFILER_JAR=${METRON_HOME}/lib/${project.artifactId}-${METRON_VERSION}.jar
+PROFILER_JAR=${METRON_HOME}/lib/${project.artifactId}-${METRON_VERSION}-uber.jar
+STELLAR_JAR=${METRON_HOME}/lib/stellar-common-$METRON_VERSION-uber.jar
MAIN_CLASS=org.apache.metron.profiler.spark.cli.BatchProfilerCLI
PROFILER_PROPS=${PROFILER_PROPS:-"${METRON_HOME}/config/batch-profiler.properties"}
SPARK_HOME=${SPARK_HOME:-"/usr/hdp/current/spark2-client"}
@@ -39,6 +40,7 @@ fi
${SPARK_HOME}/bin/spark-submit \
--class ${MAIN_CLASS} \
+ --jars ${STELLAR_JAR} \
--properties-file ${PROFILER_PROPS} \
${PROFILER_JAR} \
--config ${PROFILER_PROPS} \
diff --git a/metron-analytics/metron-profiler-storm/pom.xml b/metron-analytics/metron-profiler-storm/pom.xml
index 09bd238..2ec108a 100644
--- a/metron-analytics/metron-profiler-storm/pom.xml
+++ b/metron-analytics/metron-profiler-storm/pom.xml
@@ -166,6 +166,12 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>com.esotericsoftware</groupId>
<artifactId>kryo-shaded</artifactId>
<version>${global_kryo_version}</version>
@@ -325,6 +331,8 @@
<artifactId>maven-shade-plugin</artifactId>
<version>${global_shade_version}</version>
<configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
<createDependencyReducedPom>true</createDependencyReducedPom>
</configuration>
<executions>
diff --git a/metron-analytics/metron-profiler-storm/src/main/scripts/start_profiler_topology.sh b/metron-analytics/metron-profiler-storm/src/main/scripts/start_profiler_topology.sh
index 6ec78f5..0a0ea4e 100644
--- a/metron-analytics/metron-profiler-storm/src/main/scripts/start_profiler_topology.sh
+++ b/metron-analytics/metron-profiler-storm/src/main/scripts/start_profiler_topology.sh
@@ -19,4 +19,5 @@
METRON_VERSION=${project.version}
METRON_HOME=/usr/metron/$METRON_VERSION
TOPOLOGY_JAR=${project.artifactId}-$METRON_VERSION-uber.jar
-storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/profiler/remote.yaml --filter $METRON_HOME/config/profiler.properties
+STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar
+storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/profiler/remote.yaml --filter $METRON_HOME/config/profiler.properties --jars "$METRON_HOME/lib/$STELLAR_JAR"
diff --git a/metron-analytics/metron-statistics/pom.xml b/metron-analytics/metron-statistics/pom.xml
index 2a4b9ce..4befad0 100644
--- a/metron-analytics/metron-statistics/pom.xml
+++ b/metron-analytics/metron-statistics/pom.xml
@@ -41,6 +41,7 @@
<groupId>org.apache.metron</groupId>
<artifactId>stellar-common</artifactId>
<version>${project.parent.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@@ -65,6 +66,8 @@
<artifactId>maven-shade-plugin</artifactId>
<version>${global_shade_version}</version>
<configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
<createDependencyReducedPom>true</createDependencyReducedPom>
</configuration>
<executions>
diff --git a/metron-contrib/metron-performance/pom.xml b/metron-contrib/metron-performance/pom.xml
index 0518fc8..40992ab 100644
--- a/metron-contrib/metron-performance/pom.xml
+++ b/metron-contrib/metron-performance/pom.xml
@@ -43,6 +43,12 @@
<scope>provided</scope>
</dependency>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.10</artifactId>
<version>${global_kafka_version}</version>
@@ -77,6 +83,8 @@
<goal>shade</goal>
</goals>
<configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
<filters>
<filter>
<artifact>*:*</artifact>
diff --git a/metron-contrib/metron-performance/src/main/assembly/assembly.xml b/metron-contrib/metron-performance/src/main/assembly/assembly.xml
index 3595284..0b5dfb8 100644
--- a/metron-contrib/metron-performance/src/main/assembly/assembly.xml
+++ b/metron-contrib/metron-performance/src/main/assembly/assembly.xml
@@ -33,7 +33,7 @@
<fileSet>
<directory>${project.basedir}/target</directory>
<includes>
- <include>${project.artifactId}-${project.version}.jar</include>
+ <include>${project.artifactId}-${project.version}-uber.jar</include>
</includes>
<outputDirectory>lib</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
diff --git a/metron-contrib/metron-performance/src/main/scripts/load_tool.sh b/metron-contrib/metron-performance/src/main/scripts/load_tool.sh
index 1b56a6e..da63197 100755
--- a/metron-contrib/metron-performance/src/main/scripts/load_tool.sh
+++ b/metron-contrib/metron-performance/src/main/scripts/load_tool.sh
@@ -30,7 +30,7 @@ fi
export METRON_VERSION=${project.version}
export METRON_HOME=/usr/metron/$METRON_VERSION
export CLASSNAME="org.apache.metron.performance.load.LoadGenerator"
-export GEN_JAR=${project.artifactId}-$METRON_VERSION.jar
+export GEN_JAR=${project.artifactId}-$METRON_VERSION-uber.jar
export PARSERS_JAR=metron-parsers-$METRON_VERSION-uber.jar
java -cp $METRON_HOME/lib/$GEN_JAR:$METRON_HOME/lib/$PARSERS_JAR $CLASSNAME "$@"
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index 417c225..473c860 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -508,6 +508,9 @@
<package>
<name>metron-performance</name>
</package>
+ <package>
+ <name>stellar-common</name>
+ </package>
</packages>
</osSpecific>
<osSpecific>
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index e12f6cd..673ace5 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -66,6 +66,7 @@ Source19: metron-profiler-repl-%{full_version}-archive.tar.gz
Source20: metron-parsing-storm-%{full_version}-archive.tar.gz
Source21: metron-parsers-%{full_version}-archive.tar.gz
Source22: metron-hbase-server-%{full_version}-archive.tar.gz
+Source23: stellar-common-%{full_version}-archive.tar.gz
%description
Apache Metron provides a scalable advanced security analytics framework
@@ -111,6 +112,7 @@ tar -xzf %{SOURCE19} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE20} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE21} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE22} -C %{buildroot}%{metron_home}
+tar -xzf %{SOURCE23} -C %{buildroot}%{metron_home}
install %{buildroot}%{metron_home}/bin/metron-management-ui %{buildroot}/etc/init.d/
install %{buildroot}%{metron_home}/bin/metron-alerts-ui %{buildroot}/etc/init.d/
@@ -142,7 +144,7 @@ This package installs the Metron common files %{metron_home}
%{metron_home}/bin/cluster_info.py
%{metron_home}/bin/tgt_renew.py
%{metron_home}/config/zookeeper/global.json
-%attr(0644,root,root) %{metron_home}/lib/metron-common-%{full_version}.jar
+%attr(0644,root,root) %{metron_home}/lib/metron-common-%{full_version}-uber.jar
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -262,7 +264,7 @@ This package installs performance tools useful for Metron
%dir %{metron_home}/bin
%dir %{metron_home}/lib
%{metron_home}/bin/load_tool.sh
-%attr(0644,root,root) %{metron_home}/lib/metron-performance-%{full_version}.jar
+%attr(0644,root,root) %{metron_home}/lib/metron-performance-%{full_version}-uber.jar
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -288,7 +290,7 @@ This package installs the Metron Parser files
%{metron_home}/bin/prune_hdfs_files.sh
%{metron_home}/bin/threatintel_bulk_prune.sh
%{metron_home}/bin/threatintel_taxii_load.sh
-%attr(0644,root,root) %{metron_home}/lib/metron-data-management-%{full_version}.jar
+%attr(0644,root,root) %{metron_home}/lib/metron-data-management-%{full_version}-uber.jar
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -455,7 +457,7 @@ This package installs the Metron Management Library
%defattr(-,root,root,755)
%dir %{metron_root}
%dir %{metron_home}/lib
-%attr(0644,root,root) %{metron_home}/lib/metron-management-%{full_version}.jar
+%attr(0644,root,root) %{metron_home}/lib/metron-management-%{full_version}-uber.jar
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -484,7 +486,7 @@ This package installs the Metron PCAP files %{metron_home}
%{metron_home}/bin/pcap_zeppelin_run.sh
%{metron_home}/flux/pcap/remote.yaml
%{metron_home}/config/zeppelin/metron/metron-pcap.json
-%attr(0644,root,root) %{metron_home}/lib/metron-pcap-backend-%{full_version}.jar
+%attr(0644,root,root) %{metron_home}/lib/metron-pcap-backend-%{full_version}-uber.jar
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -539,7 +541,7 @@ This package installs the Metron Rest %{metron_home}
%{metron_home}/bin/metron-rest.sh
%{metron_home}/bin/pcap_to_pdml.sh
%{metron_home}/bin/install_metron_knox.sh
-%attr(0644,root,root) %{metron_home}/lib/metron-rest-%{full_version}.jar
+%attr(0644,root,root) %{metron_home}/lib/metron-rest-%{full_version}-uber.jar
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -676,7 +678,7 @@ This package installs the Metron Profiler for Spark %{metron_home}
%dir %{metron_home}/bin
%{metron_home}/bin/start_batch_profiler.sh
%dir %{metron_home}/lib
-%attr(0644,root,root) %{metron_home}/lib/metron-profiler-spark-%{full_version}.jar
+%attr(0644,root,root) %{metron_home}/lib/metron-profiler-spark-%{full_version}-uber.jar
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -693,7 +695,7 @@ This package installs the Metron Profiler for the Stellar REPL %{metron_home}
%dir %{metron_root}
%dir %{metron_home}
%dir %{metron_home}/lib
-%attr(0644,root,root) %{metron_home}/lib/metron-profiler-repl-%{full_version}.jar
+%attr(0644,root,root) %{metron_home}/lib/metron-profiler-repl-%{full_version}-uber.jar
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -716,6 +718,23 @@ This package installs the Metron HBase Server files
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+%package stellar-common
+Summary: Stellar Common Files
+Group: Applications/Internet
+Provides: stellar-common = %{version}
+
+%description stellar-common
+This package installs the Stellar Common files
+
+%files stellar-common
+%defattr(-,root,root,755)
+%dir %{metron_root}
+%dir %{metron_home}
+%dir %{metron_home}/lib
+%attr(0644,root,root) %{metron_home}/lib/stellar-common-%{full_version}-uber.jar
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
%post config
chkconfig --add metron-management-ui
chkconfig --add metron-alerts-ui
diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml
index 81e0c10..63c0402 100644
--- a/metron-deployment/packaging/docker/rpm-docker/pom.xml
+++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml
@@ -245,6 +245,12 @@
<include>*.tar.gz</include>
</includes>
</resource>
+ <resource>
+ <directory>${metron_dir}/metron-stellar/stellar-common/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
</resources>
</configuration>
</execution>
diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml
index 84269c2..c67a5d5 100644
--- a/metron-interface/metron-rest/pom.xml
+++ b/metron-interface/metron-rest/pom.xml
@@ -191,6 +191,12 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
@@ -200,6 +206,10 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -211,12 +221,22 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-app</artifactId>
<version>${global_hadoop_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -533,6 +553,8 @@
</dependency>
</dependencies>
<configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
<keepDependenciesWithProvidedScope>true</keepDependenciesWithProvidedScope>
<createDependencyReducedPom>true</createDependencyReducedPom>
<filters>
diff --git a/metron-interface/metron-rest/src/main/assembly/assembly.xml b/metron-interface/metron-rest/src/main/assembly/assembly.xml
index e0e4f14..ee0fe02 100644
--- a/metron-interface/metron-rest/src/main/assembly/assembly.xml
+++ b/metron-interface/metron-rest/src/main/assembly/assembly.xml
@@ -45,7 +45,7 @@
<fileSet>
<directory>${project.basedir}/target</directory>
<includes>
- <include>${project.artifactId}-${project.version}.jar</include>
+ <include>${project.artifactId}-${project.version}-uber.jar</include>
</includes>
<outputDirectory>lib</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
index 19d0a33..87aee10 100644
--- a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
+++ b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
@@ -40,6 +40,7 @@ PARSER_CONTRIB=${PARSER_CONTRIB:-$METRON_HOME/parser_contrib}
INDEXING_CONTRIB=${INDEXING_CONTRIB:-$METRON_HOME/indexing_contrib}
METRON_PARSERS_PLATFORM="${METRON_PARSERS_PLATFORM:-storm}"
PARSER_LIB=$(find $METRON_HOME/lib/ -name metron-parsing-${METRON_PARSERS_PLATFORM}*.jar)
+STELLAR_LIB=$METRON_HOME/lib/stellar-common-$METRON_VERSION-uber.jar
echo "METRON_VERSION=${METRON_VERSION}"
echo "METRON_HOME=${METRON_HOME}"
@@ -61,6 +62,7 @@ rest_jar_pattern="${METRON_HOME}/lib/metron-rest*.jar"
rest_files=( ${rest_jar_pattern} )
echo "Default metron-rest jar is: ${rest_files[0]}"
METRON_REST_CLASSPATH+=":${rest_files[0]}"
+METRON_REST_CLASSPATH+=":$STELLAR_LIB"
METRON_REST_CLASSPATH+=":$PARSER_LIB"
if [ -d "$PARSER_CONTRIB" ]; then
diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml
index a1ebe50..8651e05 100644
--- a/metron-platform/elasticsearch-shaded/pom.xml
+++ b/metron-platform/elasticsearch-shaded/pom.xml
@@ -103,6 +103,8 @@
<artifactId>maven-shade-plugin</artifactId>
<version>${global_shade_version}</version>
<configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
<createDependencyReducedPom>true</createDependencyReducedPom>
</configuration>
<executions>
diff --git a/metron-platform/metron-common-streaming/metron-common-storm/pom.xml b/metron-platform/metron-common-streaming/metron-common-storm/pom.xml
index 30371bd..27f3aa1 100644
--- a/metron-platform/metron-common-streaming/metron-common-storm/pom.xml
+++ b/metron-platform/metron-common-streaming/metron-common-storm/pom.xml
@@ -36,6 +36,12 @@
<version>${project.parent.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>${global_storm_version}</version>
diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml
index d215180..0bcf343 100644
--- a/metron-platform/metron-common/pom.xml
+++ b/metron-platform/metron-common/pom.xml
@@ -48,6 +48,7 @@
<groupId>org.apache.metron</groupId>
<artifactId>stellar-common</artifactId>
<version>${project.parent.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@@ -272,10 +273,10 @@
<artifactId>reflections</artifactId>
<version>0.9.10</version>
<exclusions>
- <exclusion>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>annotations</artifactId>
- </exclusion>
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>annotations</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -338,14 +339,14 @@
<artifactId>antlr4-maven-plugin</artifactId>
<version>${global_antlr_version}</version>
<configuration>
- <outputDirectory>${basedir}/src/main/java</outputDirectory>
+ <outputDirectory>${basedir}/src/main/java</outputDirectory>
</configuration>
<executions>
- <execution>
- <goals>
- <goal>antlr4</goal>
- </goals>
- </execution>
+ <execution>
+ <goals>
+ <goal>antlr4</goal>
+ </goals>
+ </execution>
</executions>
</plugin>
@@ -368,16 +369,18 @@
<goal>shade</goal>
</goals>
<configuration>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
@@ -390,13 +393,13 @@
</relocations>
<transformers>
<transformer
- implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
- <resources>
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resources>
<resource>.yaml</resource>
<resource>LICENSE.txt</resource>
<resource>ASL2.0</resource>
<resource>NOTICE.txt</resource>
- </resources>
+ </resources>
</transformer>
<!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE -->
<!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
@@ -404,7 +407,7 @@
<projectName>${project.name}</projectName>
</transformer-->
<transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass></mainClass>
diff --git a/metron-platform/metron-common/src/main/assembly/assembly.xml b/metron-platform/metron-common/src/main/assembly/assembly.xml
index e0e4f14..ee0fe02 100644
--- a/metron-platform/metron-common/src/main/assembly/assembly.xml
+++ b/metron-platform/metron-common/src/main/assembly/assembly.xml
@@ -45,7 +45,7 @@
<fileSet>
<directory>${project.basedir}/target</directory>
<includes>
- <include>${project.artifactId}-${project.version}.jar</include>
+ <include>${project.artifactId}-${project.version}-uber.jar</include>
</includes>
<outputDirectory>lib</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
diff --git a/metron-platform/metron-common/src/main/scripts/stellar b/metron-platform/metron-common/src/main/scripts/stellar
index 142cc29..6f9418b 100644
--- a/metron-platform/metron-common/src/main/scripts/stellar
+++ b/metron-platform/metron-common/src/main/scripts/stellar
@@ -38,7 +38,8 @@ set -u
export METRON_PARSERS_PLATFORM="${METRON_PARSERS_PLATFORM:-storm}"
export HBASE_CONFIGS=$(hbase classpath)
-export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsing-${METRON_PARSERS_PLATFORM}*.jar)
+export STELLAR_LIB=$(find $METRON_HOME/lib/ -name stellar-common*.jar)
+export PARSING_LIB=$(find $METRON_HOME/lib/ -name metron-parsing-${METRON_PARSERS_PLATFORM}*.jar)
export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar)
export PROFILER_LIB=$(find $METRON_HOME/lib/ -name metron-profiler-repl*.jar)
-java $METRON_JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$PROFILER_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@"
+java $METRON_JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$PARSING_LIB:$MANAGEMENT_LIB:$PROFILER_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@"
diff --git a/metron-platform/metron-common/src/main/scripts/zk_load_configs.sh b/metron-platform/metron-common/src/main/scripts/zk_load_configs.sh
index b22bb4b..3e8f787 100755
--- a/metron-platform/metron-common/src/main/scripts/zk_load_configs.sh
+++ b/metron-platform/metron-common/src/main/scripts/zk_load_configs.sh
@@ -30,9 +30,10 @@ export METRON_VERSION=${project.version}
export METRON_HOME=/usr/metron/$METRON_VERSION
export CLASSNAME="org.apache.metron.common.cli.ConfigurationManager"
export JAR=metron-parsers-common-$METRON_VERSION-uber.jar
+export STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar
# TODO export the Storm jar?
export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
-CP=$METRON_HOME/lib/$JAR:${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath`
+CP=$METRON_HOME/lib/$JAR:$METRON_HOME/lib/$STELLAR_JAR:${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath`
java $METRON_JVMFLAGS -cp $CP $CLASSNAME "$@"
diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml
index 3e67c59..d4fae01 100644
--- a/metron-platform/metron-data-management/pom.xml
+++ b/metron-platform/metron-data-management/pom.xml
@@ -33,9 +33,16 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>${guava_version}</version>
+ <version>${global_hbase_guava_version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>2.2.11</version>
@@ -181,10 +188,10 @@
<artifactId>hbase-common</artifactId>
<version>${global_hbase_version}</version>
<exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -237,7 +244,7 @@
<version>2.1.14</version>
<scope>test</scope>
</dependency>
- <dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${global_hadoop_version}</version>
@@ -262,14 +269,14 @@
<version>${global_hbase_version}</version>
<scope>test</scope>
<exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
@@ -367,17 +374,19 @@
<goal>shade</goal>
</goals>
<configuration>
- <createDependencyReducedPom>false</createDependencyReducedPom>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
@@ -387,16 +396,16 @@
<pattern>org.apache.http</pattern>
<shadedPattern>org.apache.metron.httpcore.dataload</shadedPattern>
</relocation>
- </relocations>
- <transformers>
+ </relocations>
+ <transformers>
<transformer
- implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
- <resources>
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resources>
<resource>.yaml</resource>
<resource>LICENSE.txt</resource>
<resource>ASL2.0</resource>
<resource>NOTICE.txt</resource>
- </resources>
+ </resources>
</transformer>
<!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE -->
<!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
@@ -448,8 +457,24 @@
</execution>
</executions>
</plugin>
- </plugins>
- <resources>
+
+ <!-- This plugin changes the integration tests in this module to use an external dependency jar for stellar-common. This is necessary because conflicting versions of
+ guava is required by code in both this module and the stellar-common module. -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${global_surefire_version}</version>
+ <configuration>
+ <additionalClasspathElements>
+ <additionalClasspathElement>${basedir}/../../metron-stellar/stellar-common/target/stellar-common-${project.parent.version}-uber.jar</additionalClasspathElement>
+ </additionalClasspathElements>
+ <classpathDependencyExcludes>
+ <classpathDependencyExclude>org.apache.metron:stellar-common</classpathDependencyExclude>
+ </classpathDependencyExcludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ <resources>
<resource>
<directory>src/main/resources</directory>
</resource>
diff --git a/metron-platform/metron-data-management/src/main/assembly/assembly.xml b/metron-platform/metron-data-management/src/main/assembly/assembly.xml
index 3595284..0b5dfb8 100644
--- a/metron-platform/metron-data-management/src/main/assembly/assembly.xml
+++ b/metron-platform/metron-data-management/src/main/assembly/assembly.xml
@@ -33,7 +33,7 @@
<fileSet>
<directory>${project.basedir}/target</directory>
<includes>
- <include>${project.artifactId}-${project.version}.jar</include>
+ <include>${project.artifactId}-${project.version}-uber.jar</include>
</includes>
<outputDirectory>lib</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
diff --git a/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh b/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh
index feccc33..5453bf4 100755
--- a/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh
+++ b/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh
@@ -30,15 +30,16 @@ fi
export METRON_VERSION=${project.version}
export METRON_HOME=/usr/metron/$METRON_VERSION
export CLASSNAME="org.apache.metron.dataloads.nonbulk.flatfile.SimpleEnrichmentFlatFileLoader"
-export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
+export DM_JAR=${project.artifactId}-$METRON_VERSION-uber.jar
+export STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar
export HBASE_CONF=${HBASE_CONF:-/etc/hbase/conf}
export HADOOP_OPTS="$HADOOP_OPTS $METRON_JVMFLAGS"
if [ $(which hadoop) ]
then
- export HADOOP_CLASSPATH="$HBASE_CONF"
+ export HADOOP_CLASSPATH="$METRON_HOME/lib/$DM_JAR:$HBASE_CONF:$METRON_HOME/lib/$STELLAR_JAR"
hadoop jar $METRON_HOME/lib/$DM_JAR $CLASSNAME "$@"
else
echo "Warning: Metron cannot find the hadoop client on this node. This means that loading via Map Reduce will NOT function."
- CP=$METRON_HOME/lib/$DM_JAR:$HBASE_CONF
+ CP=$METRON_HOME/lib/$DM_JAR:$HBASE_CONF:$METRON_HOME/lib/$STELLAR_JAR
java $METRON_JVMFLAGS -cp $CP $CLASSNAME "$@"
fi
diff --git a/metron-platform/metron-data-management/src/main/scripts/flatfile_summarizer.sh b/metron-platform/metron-data-management/src/main/scripts/flatfile_summarizer.sh
index 018d61a..1279ecc 100755
--- a/metron-platform/metron-data-management/src/main/scripts/flatfile_summarizer.sh
+++ b/metron-platform/metron-data-management/src/main/scripts/flatfile_summarizer.sh
@@ -30,12 +30,13 @@ fi
export METRON_VERSION=${project.version}
export METRON_HOME=/usr/metron/$METRON_VERSION
export CLASSNAME="org.apache.metron.dataloads.nonbulk.flatfile.SimpleFlatFileSummarizer"
-export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
+export DM_JAR=${project.artifactId}-$METRON_VERSION-uber.jar
+export STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar
export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
export HADOOP_OPTS="$HADOOP_OPTS $METRON_JVMFLAGS"
if [ $(which hadoop) ]
then
- HADOOP_CLASSPATH=${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath`
+ HADOOP_CLASSPATH=$METRON_HOME/lib/$DM_JAR:$METRON_HOME/lib/$STELLAR_JAR:${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath`
for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
if [ -f $jar ];then
LIBJARS="$jar,$LIBJARS"
@@ -45,7 +46,7 @@ then
hadoop jar $METRON_HOME/lib/$DM_JAR $CLASSNAME -libjars ${LIBJARS} "$@"
else
echo "Warning: Metron cannot find the hadoop client on this node. This means that loading via Map Reduce will NOT function."
- CP=$METRON_HOME/lib/$DM_JAR:/usr/metron/${METRON_VERSION}/lib/taxii-1.1.0.1.jar:`${HBASE_HOME}/bin/hbase classpath`
+ CP=$METRON_HOME/lib/$DM_JAR:$METRON_HOME/lib/$STELLAR_JAR:/usr/metron/${METRON_VERSION}/lib/taxii-1.1.0.1.jar:`${HBASE_HOME}/bin/hbase classpath`
java $METRON_JVMFLAGS -cp $CP $CLASSNAME "$@"
fi
diff --git a/metron-platform/metron-data-management/src/main/scripts/maxmind_enrichment_load.sh b/metron-platform/metron-data-management/src/main/scripts/maxmind_enrichment_load.sh
index cf43b14..0af400d 100644
--- a/metron-platform/metron-data-management/src/main/scripts/maxmind_enrichment_load.sh
+++ b/metron-platform/metron-data-management/src/main/scripts/maxmind_enrichment_load.sh
@@ -26,17 +26,17 @@ if [ -e /usr/libexec/bigtop-detect-javahome ]; then
elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
. /usr/lib/bigtop-utils/bigtop-detect-javahome
fi
-
+export METRON_VERSION=${project.version}
+export METRON_HOME=/usr/metron/$METRON_VERSION
+export DM_JAR=${project.artifactId}-$METRON_VERSION-uber.jar
+export STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar
export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
-HADOOP_CLASSPATH=${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath`
+HADOOP_CLASSPATH=${HBASE_HOME}/lib/hbase-server.jar:${METRON_HOME}/lib/${STELLAR_JAR}:`${HBASE_HOME}/bin/hbase classpath`
for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
if [ -f $jar ];then
LIBJARS="$jar,$LIBJARS"
fi
done
export HADOOP_CLASSPATH
-export METRON_VERSION=${project.version}
-export METRON_HOME=/usr/metron/$METRON_VERSION
-export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
export HADOOP_OPTS="$HADOOP_OPTS $METRON_JVMFLAGS"
hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.nonbulk.geo.MaxmindDbEnrichmentLoader "$@"
diff --git a/metron-platform/metron-data-management/src/main/scripts/prune_hdfs_files.sh b/metron-platform/metron-data-management/src/main/scripts/prune_hdfs_files.sh
index b8d9372..8670074 100644
--- a/metron-platform/metron-data-management/src/main/scripts/prune_hdfs_files.sh
+++ b/metron-platform/metron-data-management/src/main/scripts/prune_hdfs_files.sh
@@ -17,5 +17,5 @@
# limitations under the License.
#
-yarn jar /usr/metron/${project.version}/lib/${project.artifactId}-${project.version}.jar org.apache.metron.dataloads.bulk.HDFSDataPruner "$@"
+yarn jar /usr/metron/${project.version}/lib/${project.artifactId}-${project.version}-uber.jar org.apache.metron.dataloads.bulk.HDFSDataPruner "$@"
diff --git a/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_prune.sh b/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_prune.sh
index 6156027..47daa2f 100755
--- a/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_prune.sh
+++ b/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_prune.sh
@@ -36,5 +36,5 @@ done
export HADOOP_CLASSPATH
export METRON_VERSION=${project.version}
export METRON_HOME=/usr/metron/$METRON_VERSION
-export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
+export DM_JAR=${project.artifactId}-$METRON_VERSION-uber.jar
hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.bulk.LeastRecentlyUsedPruner -libjars ${LIBJARS} "$@"
diff --git a/metron-platform/metron-data-management/src/main/scripts/threatintel_taxii_load.sh b/metron-platform/metron-data-management/src/main/scripts/threatintel_taxii_load.sh
index 953a86e..15ca27a 100755
--- a/metron-platform/metron-data-management/src/main/scripts/threatintel_taxii_load.sh
+++ b/metron-platform/metron-data-management/src/main/scripts/threatintel_taxii_load.sh
@@ -30,7 +30,7 @@ fi
export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
export METRON_VERSION=${project.version}
export METRON_HOME=/usr/metron/$METRON_VERSION
-export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
+export DM_JAR=${project.artifactId}-$METRON_VERSION-uber.jar
CP=$METRON_HOME/lib/$DM_JAR:/usr/metron/$METRON_VERSION/lib/taxii-1.1.0.1.jar:`${HBASE_HOME}/bin/hbase classpath`
HADOOP_CLASSPATH=$(echo $CP )
diff --git a/metron-platform/metron-elasticsearch/metron-elasticsearch-common/pom.xml b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/pom.xml
index 190bcb2..15df035 100644
--- a/metron-platform/metron-elasticsearch/metron-elasticsearch-common/pom.xml
+++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/pom.xml
@@ -38,6 +38,12 @@
<groupId>org.apache.metron</groupId>
<artifactId>metron-common</artifactId>
<version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
@@ -62,9 +68,16 @@
</exclusions>
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava_version}</version>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
@@ -117,6 +130,14 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git a/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/pom.xml b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/pom.xml
index 1804098..1ba1e27 100644
--- a/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/pom.xml
+++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/pom.xml
@@ -40,6 +40,24 @@
<groupId>org.apache.metron</groupId>
<artifactId>metron-storm-kafka</artifactId>
<version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
@@ -110,6 +128,7 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+
</dependencies>
<build>
<plugins>
diff --git a/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/main/scripts/start_elasticsearch_topology.sh b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/main/scripts/start_elasticsearch_topology.sh
index 20ce23f..d3a29c0 100755
--- a/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/main/scripts/start_elasticsearch_topology.sh
+++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/main/scripts/start_elasticsearch_topology.sh
@@ -19,10 +19,11 @@
METRON_VERSION=${project.version}
METRON_HOME=/usr/metron/$METRON_VERSION
TOPOLOGY_JAR=${project.artifactId}-$METRON_VERSION-uber.jar
+STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar
INDEXING_CONTRIB=${INDEXING_CONTRIB:-$METRON_HOME/indexing_contrib}
if [ -d "$INDEXING_CONTRIB" ]; then
export EXTRA_JARS=$(ls -m $INDEXING_CONTRIB/*.jar | tr -d ' ' | tr -d '\n' | sed 's/\/\//\//g')
- storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/random_access/remote.yaml --filter $METRON_HOME/config/elasticsearch.properties --jars "$EXTRA_JARS"
+ storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/random_access/remote.yaml --filter $METRON_HOME/config/elasticsearch.properties --jars "$METRON_HOME/lib/$STELLAR_JAR,$EXTRA_JARS"
else
- storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/random_access/remote.yaml --filter $METRON_HOME/config/elasticsearch.properties
+ storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/random_access/remote.yaml --filter $METRON_HOME/config/elasticsearch.properties --jars "$METRON_HOME/lib/$STELLAR_JAR"
fi
diff --git a/metron-platform/metron-enrichment/metron-enrichment-common/pom.xml b/metron-platform/metron-enrichment/metron-enrichment-common/pom.xml
index 77ec1af..67938bc 100644
--- a/metron-platform/metron-enrichment/metron-enrichment-common/pom.xml
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/pom.xml
@@ -41,6 +41,7 @@
<groupId>org.apache.metron</groupId>
<artifactId>stellar-common</artifactId>
<version>${project.parent.version}</version>
+ <scope>provided</scope>
</dependency>
<!-- Other -->
diff --git a/metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml b/metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml
index fe21399..6fe47e4 100644
--- a/metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml
@@ -61,6 +61,12 @@
<artifactId>metron-statistics</artifactId>
<version>${project.parent.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
<!-- Hadoop -->
diff --git a/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/scripts/start_enrichment_topology.sh b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/scripts/start_enrichment_topology.sh
index d9c2dc4..34886c9 100755
--- a/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/scripts/start_enrichment_topology.sh
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/scripts/start_enrichment_topology.sh
@@ -19,9 +19,10 @@
METRON_VERSION=${project.version}
METRON_HOME=/usr/metron/$METRON_VERSION
TOPOLOGY_JAR=${project.artifactId}-$METRON_VERSION-uber.jar
+STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar
DEFAULT_ARGS="--remote $METRON_HOME/flux/enrichment/remote.yaml --filter $METRON_HOME/config/enrichment.properties"
# by passing in different args, the user can execute an alternative enrichment topology
ARGS=${@:-$DEFAULT_ARGS}
-storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux $ARGS
+storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux $ARGS --jars $METRON_HOME/lib/$STELLAR_JAR
diff --git a/metron-platform/metron-hbase-client/pom.xml b/metron-platform/metron-hbase-client/pom.xml
index 37a7bf5..fe497df 100644
--- a/metron-platform/metron-hbase-client/pom.xml
+++ b/metron-platform/metron-hbase-client/pom.xml
@@ -66,6 +66,8 @@
<artifactId>maven-shade-plugin</artifactId>
<version>${global_shade_version}</version>
<configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
<createDependencyReducedPom>true</createDependencyReducedPom>
<artifactSet>
<excludes>
diff --git a/metron-platform/metron-hbase-server/pom.xml b/metron-platform/metron-hbase-server/pom.xml
index 0cbdeeb..171f800 100644
--- a/metron-platform/metron-hbase-server/pom.xml
+++ b/metron-platform/metron-hbase-server/pom.xml
@@ -58,6 +58,12 @@
<artifactId>metron-enrichment-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <classifier>uber</classifier>
+ </dependency>
<!-- HBase -->
diff --git a/metron-platform/metron-indexing/metron-indexing-common/pom.xml b/metron-platform/metron-indexing/metron-indexing-common/pom.xml
index df81532..abbeb99 100644
--- a/metron-platform/metron-indexing/metron-indexing-common/pom.xml
+++ b/metron-platform/metron-indexing/metron-indexing-common/pom.xml
@@ -44,6 +44,12 @@
<artifactId>metron-writer-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
<!-- Other -->
@@ -91,11 +97,6 @@
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava_version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${global_hadoop_version}</version>
diff --git a/metron-platform/metron-indexing/metron-indexing-storm/pom.xml b/metron-platform/metron-indexing/metron-indexing-storm/pom.xml
index 581e819..36c0ef8 100644
--- a/metron-platform/metron-indexing/metron-indexing-storm/pom.xml
+++ b/metron-platform/metron-indexing/metron-indexing-storm/pom.xml
@@ -85,6 +85,12 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>${global_kafka_version}</version>
diff --git a/metron-platform/metron-indexing/metron-indexing-storm/src/main/scripts/start_hdfs_topology.sh b/metron-platform/metron-indexing/metron-indexing-storm/src/main/scripts/start_hdfs_topology.sh
index 77007be..c29e6c4 100755
--- a/metron-platform/metron-indexing/metron-indexing-storm/src/main/scripts/start_hdfs_topology.sh
+++ b/metron-platform/metron-indexing/metron-indexing-storm/src/main/scripts/start_hdfs_topology.sh
@@ -19,4 +19,5 @@
METRON_VERSION=${project.version}
METRON_HOME=/usr/metron/$METRON_VERSION
TOPOLOGY_JAR=metron-elasticsearch-storm-$METRON_VERSION-uber.jar
-storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/batch/remote.yaml --filter $METRON_HOME/config/hdfs.properties
+STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar
+storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/batch/remote.yaml --filter $METRON_HOME/config/hdfs.properties --jars "$METRON_HOME/lib/$STELLAR_JAR"
diff --git a/metron-platform/metron-integration-test/pom.xml b/metron-platform/metron-integration-test/pom.xml
index 427290f..c89bce0 100644
--- a/metron-platform/metron-integration-test/pom.xml
+++ b/metron-platform/metron-integration-test/pom.xml
@@ -228,6 +228,7 @@
<groupId>org.apache.metron</groupId>
<artifactId>stellar-common</artifactId>
<version>${project.parent.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git a/metron-platform/metron-management/pom.xml b/metron-platform/metron-management/pom.xml
index c5c319e..e308f8e 100644
--- a/metron-platform/metron-management/pom.xml
+++ b/metron-platform/metron-management/pom.xml
@@ -72,6 +72,7 @@
<groupId>org.apache.metron</groupId>
<artifactId>stellar-common</artifactId>
<version>${project.parent.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>com.jakewharton.fliptables</groupId>
@@ -211,6 +212,8 @@
<artifactId>maven-shade-plugin</artifactId>
<version>${global_shade_version}</version>
<configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
<createDependencyReducedPom>false</createDependencyReducedPom>
<artifactSet>
<excludes>
diff --git a/metron-platform/metron-management/src/main/assembly/assembly.xml b/metron-platform/metron-management/src/main/assembly/assembly.xml
index a7e20ed..4cc9735 100644
--- a/metron-platform/metron-management/src/main/assembly/assembly.xml
+++ b/metron-platform/metron-management/src/main/assembly/assembly.xml
@@ -21,7 +21,7 @@
<fileSet>
<directory>${project.basedir}/target</directory>
<includes>
- <include>${project.artifactId}-${project.version}.jar</include>
+ <include>${project.artifactId}-${project.version}-uber.jar</include>
</includes>
<outputDirectory>lib</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
diff --git a/metron-platform/metron-parsing/metron-parsers-common/pom.xml b/metron-platform/metron-parsing/metron-parsers-common/pom.xml
index 631bdd8..407d311 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/pom.xml
+++ b/metron-platform/metron-parsing/metron-parsers-common/pom.xml
@@ -55,6 +55,12 @@
<version>${project.parent.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${global_hadoop_version}</version>
diff --git a/metron-platform/metron-parsing/metron-parsers/pom.xml b/metron-platform/metron-parsing/metron-parsers/pom.xml
index d888e19..cbf9c2f 100644
--- a/metron-platform/metron-parsing/metron-parsers/pom.xml
+++ b/metron-platform/metron-parsing/metron-parsers/pom.xml
@@ -41,6 +41,12 @@
<artifactId>metron-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
<!-- 3rd party dependencies -->
<dependency>
diff --git a/metron-platform/metron-parsing/metron-parsing-storm/pom.xml b/metron-platform/metron-parsing/metron-parsing-storm/pom.xml
index 2c5fd37..98bfcff 100644
--- a/metron-platform/metron-parsing/metron-parsing-storm/pom.xml
+++ b/metron-platform/metron-parsing/metron-parsing-storm/pom.xml
@@ -56,6 +56,12 @@
<artifactId>metron-writer-storm</artifactId>
<version>${project.parent.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
<!-- Storm dependencies -->
<dependency>
diff --git a/metron-platform/metron-parsing/metron-parsing-storm/src/main/scripts/start_parser_topology.sh b/metron-platform/metron-parsing/metron-parsing-storm/src/main/scripts/start_parser_topology.sh
index db777dc..0b21f35 100755
--- a/metron-platform/metron-parsing/metron-parsing-storm/src/main/scripts/start_parser_topology.sh
+++ b/metron-platform/metron-parsing/metron-parsing-storm/src/main/scripts/start_parser_topology.sh
@@ -19,13 +19,15 @@
METRON_VERSION=${project.version}
METRON_HOME=/usr/metron/$METRON_VERSION
TOPOLOGY_JAR=metron-parsing-storm-$METRON_VERSION-uber.jar
+STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar
PARSER_CONTRIB=${PARSER_CONTRIB:-$METRON_HOME/parser_contrib}
+EXTRA_JARS="--jars $METRON_HOME/lib/$STELLAR_JAR"
EXTRA_ARGS="$@"
if [ -d "$PARSER_CONTRIB" ]; then
export STORM_EXT_CLASSPATH=$METRON_HOME/lib/$TOPOLOGY_JAR
- export EXTRA_JARS=$(ls -m $PARSER_CONTRIB/*.jar | tr -d ' ' | tr -d '\n' | sed 's/\/\//\//g')
+ export PARSER_CONTRIB_JARS=$(ls -m $PARSER_CONTRIB/*.jar | tr -d ' ' | tr -d '\n' | sed 's/\/\//\//g')
STORM_VERSION=`storm version | grep Storm | awk '{print $2}' | cut -d. -f1,2,3`
STORM_COMPAT_CUTOFF="1.1.0"
@@ -33,7 +35,7 @@ if [ -d "$PARSER_CONTRIB" ]; then
if [ "$STORM_COMPAT_VERSION" = "$STORM_COMPAT_CUTOFF" ]; then
echo "Third party parser support for Storm >= 1.1.0; found Storm $STORM_VERSION"
- EXTRA_ARGS+=" --jars $EXTRA_JARS"
+ EXTRA_JARS+=",$PARSER_CONTRIB_JARS"
else
echo "Legacy third party parser support for Storm < 1.1.0; found Storm $STORM_VERSION"
EXTRA_ARGS+=" -c client.jartransformer.class=org.apache.metron.parsers.topology.MergeAndShadeTransformer"
@@ -41,4 +43,4 @@ if [ -d "$PARSER_CONTRIB" ]; then
fi
echo "Submitting parser topology; args='$EXTRA_ARGS'"
-storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.metron.parsers.topology.ParserTopologyCLI $EXTRA_ARGS
+storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.metron.parsers.topology.ParserTopologyCLI $EXTRA_ARGS $EXTRA_JARS
diff --git a/metron-platform/metron-pcap-backend/pom.xml b/metron-platform/metron-pcap-backend/pom.xml
index ccc69b8..5042b70 100644
--- a/metron-platform/metron-pcap-backend/pom.xml
+++ b/metron-platform/metron-pcap-backend/pom.xml
@@ -58,6 +58,12 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${global_junit_version}</version>
@@ -248,16 +254,18 @@
<goal>shade</goal>
</goals>
<configuration>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
@@ -267,13 +275,13 @@
<transformers>
<transformer implementation="org.atteo.classindex.ClassIndexTransformer"/>
<transformer
- implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
- <resources>
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resources>
<resource>.yaml</resource>
<resource>LICENSE.txt</resource>
<resource>ASL2.0</resource>
<resource>NOTICE.txt</resource>
- </resources>
+ </resources>
</transformer>
<!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE -->
<!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
@@ -281,7 +289,7 @@
<projectName>${project.name}</projectName>
</transformer-->
<transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass></mainClass>
@@ -297,8 +305,8 @@
<version>${global_classindex_version}</version>
</dependency>
</dependencies>
- </plugin>
- <plugin>
+ </plugin>
+ <plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptor>src/main/assembly/assembly.xml</descriptor>
diff --git a/metron-platform/metron-pcap-backend/src/main/assembly/assembly.xml b/metron-platform/metron-pcap-backend/src/main/assembly/assembly.xml
index f38582d..1df0f14 100644
--- a/metron-platform/metron-pcap-backend/src/main/assembly/assembly.xml
+++ b/metron-platform/metron-pcap-backend/src/main/assembly/assembly.xml
@@ -56,7 +56,7 @@
<fileSet>
<directory>${project.basedir}/target</directory>
<includes>
- <include>${project.artifactId}-${project.version}.jar</include>
+ <include>${project.artifactId}-${project.version}-uber.jar</include>
</includes>
<outputDirectory>lib</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
diff --git a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_inspector.sh b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_inspector.sh
index 438152c..12189cc 100755
--- a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_inspector.sh
+++ b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_inspector.sh
@@ -29,6 +29,6 @@ fi
export METRON_VERSION=${project.version}
export METRON_HOME=/usr/metron/$METRON_VERSION
-export TOPOLOGIES_JAR=${project.artifactId}-$METRON_VERSION.jar
+export TOPOLOGIES_JAR=${project.artifactId}-$METRON_VERSION-uber.jar
export HADOOP_OPTS="$HADOOP_OPTS $METRON_JVMFLAGS"
yarn jar $METRON_HOME/lib/$TOPOLOGIES_JAR org.apache.metron.utils.PcapInspector "$@"
diff --git a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_query.sh b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_query.sh
index 206e2b8..a3e3bb6 100755
--- a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_query.sh
+++ b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_query.sh
@@ -29,7 +29,7 @@ fi
export METRON_VERSION=${project.version}
export METRON_HOME=/usr/metron/$METRON_VERSION
-export PCAP_BACKEND_JAR=${project.artifactId}-$METRON_VERSION.jar
+export PCAP_BACKEND_JAR=${project.artifactId}-$METRON_VERSION-uber.jar
export HADOOP_OPTS="$HADOOP_OPTS $METRON_JVMFLAGS"
yarn jar $METRON_HOME/lib/$PCAP_BACKEND_JAR org.apache.metron.pcap.query.PcapCli "$@"
diff --git a/metron-platform/metron-pcap-backend/src/main/scripts/start_pcap_topology.sh b/metron-platform/metron-pcap-backend/src/main/scripts/start_pcap_topology.sh
index a38f905..f42ec6a 100644
--- a/metron-platform/metron-pcap-backend/src/main/scripts/start_pcap_topology.sh
+++ b/metron-platform/metron-pcap-backend/src/main/scripts/start_pcap_topology.sh
@@ -18,6 +18,6 @@
#
export METRON_VERSION=${project.version}
export METRON_HOME=/usr/metron/$METRON_VERSION
-export TOPOLOGIES_JAR=${project.artifactId}-$METRON_VERSION.jar
-
-storm jar $METRON_HOME/lib/$TOPOLOGIES_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/pcap/remote.yaml --filter $METRON_HOME/config/pcap.properties
\ No newline at end of file
+export TOPOLOGIES_JAR=${project.artifactId}-$METRON_VERSION-uber.jar
+export STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar
+storm jar $METRON_HOME/lib/$TOPOLOGIES_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/pcap/remote.yaml --filter $METRON_HOME/config/pcap.properties --jars "$METRON_HOME/lib/$STELLAR_JAR"
\ No newline at end of file
diff --git a/metron-platform/metron-pcap/pom.xml b/metron-platform/metron-pcap/pom.xml
index c839285..4fda810 100644
--- a/metron-platform/metron-pcap/pom.xml
+++ b/metron-platform/metron-pcap/pom.xml
@@ -46,6 +46,12 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
<artifactId>metron-job</artifactId>
<version>${project.parent.version}</version>
</dependency>
@@ -92,6 +98,14 @@
<artifactId>commons-beanutils</artifactId>
<groupId>commons-beanutils</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git a/metron-platform/metron-solr/metron-solr-common/pom.xml b/metron-platform/metron-solr/metron-solr-common/pom.xml
index ffaf6d9..bcf9366 100644
--- a/metron-platform/metron-solr/metron-solr-common/pom.xml
+++ b/metron-platform/metron-solr/metron-solr-common/pom.xml
@@ -30,6 +30,12 @@
</properties>
<dependencies>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava_version}</version>
diff --git a/metron-platform/metron-solr/metron-solr-storm/pom.xml b/metron-platform/metron-solr/metron-solr-storm/pom.xml
index 22c8c50..b089986 100644
--- a/metron-platform/metron-solr/metron-solr-storm/pom.xml
+++ b/metron-platform/metron-solr/metron-solr-storm/pom.xml
@@ -55,6 +55,12 @@
<type>test-jar</type>
</dependency>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-test-framework</artifactId>
<version>${global_solr_version}</version>
diff --git a/metron-platform/metron-solr/metron-solr-storm/src/main/scripts/start_solr_topology.sh b/metron-platform/metron-solr/metron-solr-storm/src/main/scripts/start_solr_topology.sh
index 614423e..e386fcb 100755
--- a/metron-platform/metron-solr/metron-solr-storm/src/main/scripts/start_solr_topology.sh
+++ b/metron-platform/metron-solr/metron-solr-storm/src/main/scripts/start_solr_topology.sh
@@ -19,4 +19,5 @@
METRON_VERSION=${project.version}
METRON_HOME=/usr/metron/$METRON_VERSION
TOPOLOGY_JAR=${project.artifactId}-$METRON_VERSION-uber.jar
-storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/random_access/remote.yaml --filter $METRON_HOME/config/solr.properties
+STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar
+storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/random_access/remote.yaml --filter $METRON_HOME/config/solr.properties --jars "$METRON_HOME/lib/$STELLAR_JAR"
diff --git a/metron-platform/metron-storm-kafka-override/pom.xml b/metron-platform/metron-storm-kafka-override/pom.xml
index 3addee3..f557956 100644
--- a/metron-platform/metron-storm-kafka-override/pom.xml
+++ b/metron-platform/metron-storm-kafka-override/pom.xml
@@ -106,6 +106,8 @@
<artifactId>maven-shade-plugin</artifactId>
<version>${global_shade_version}</version>
<configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
<createDependencyReducedPom>true</createDependencyReducedPom>
</configuration>
<executions>
diff --git a/metron-platform/metron-storm-kafka/pom.xml b/metron-platform/metron-storm-kafka/pom.xml
index da762bf..0749a19 100644
--- a/metron-platform/metron-storm-kafka/pom.xml
+++ b/metron-platform/metron-storm-kafka/pom.xml
@@ -101,6 +101,7 @@
<groupId>org.apache.metron</groupId>
<artifactId>stellar-common</artifactId>
<version>${project.parent.version}</version>
+ <scope>provided</scope>
</dependency>
</dependencies>
diff --git a/metron-platform/metron-writer/metron-writer-common/pom.xml b/metron-platform/metron-writer/metron-writer-common/pom.xml
index 0d91b52..6b902ec 100644
--- a/metron-platform/metron-writer/metron-writer-common/pom.xml
+++ b/metron-platform/metron-writer/metron-writer-common/pom.xml
@@ -169,7 +169,7 @@
</exclusions>
<scope>provided</scope>
</dependency>
- <dependency>
+ <dependency>
<groupId>org.apache.metron</groupId>
<artifactId>metron-common</artifactId>
<version>${project.parent.version}</version>
@@ -189,6 +189,7 @@
<groupId>org.apache.metron</groupId>
<artifactId>stellar-common</artifactId>
<version>${project.parent.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
@@ -206,14 +207,14 @@
<artifactId>antlr4-maven-plugin</artifactId>
<version>${antlr.version}</version>
<configuration>
- <outputDirectory>${basedir}/src/main/java</outputDirectory>
+ <outputDirectory>${basedir}/src/main/java</outputDirectory>
</configuration>
<executions>
- <execution>
- <goals>
- <goal>antlr4</goal>
- </goals>
- </execution>
+ <execution>
+ <goals>
+ <goal>antlr4</goal>
+ </goals>
+ </execution>
</executions>
</plugin>
@@ -236,16 +237,18 @@
<goal>shade</goal>
</goals>
<configuration>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
@@ -254,13 +257,13 @@
</relocations>
<transformers>
<transformer
- implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
- <resources>
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resources>
<resource>.yaml</resource>
<resource>LICENSE.txt</resource>
<resource>ASL2.0</resource>
<resource>NOTICE.txt</resource>
- </resources>
+ </resources>
</transformer>
<!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE -->
<!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
@@ -268,7 +271,7 @@
<projectName>${project.name}</projectName>
</transformer-->
<transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass></mainClass>
diff --git a/metron-platform/metron-writer/metron-writer-storm/pom.xml b/metron-platform/metron-writer/metron-writer-storm/pom.xml
index 7d2c829..b1e33a9 100644
--- a/metron-platform/metron-writer/metron-writer-storm/pom.xml
+++ b/metron-platform/metron-writer/metron-writer-storm/pom.xml
@@ -41,6 +41,12 @@
<version>${project.parent.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${global_hadoop_version}</version>
diff --git a/metron-stellar/stellar-common/pom.xml b/metron-stellar/stellar-common/pom.xml
index 7b79b84..c98cb32 100644
--- a/metron-stellar/stellar-common/pom.xml
+++ b/metron-stellar/stellar-common/pom.xml
@@ -353,6 +353,10 @@
<shadedPattern>org.apache.metron.commons.validator</shadedPattern>
</relocation>
-->
+ <relocation>
+ <pattern>com.jakewharton.fliptables</pattern>
+ <shadedPattern>org.apache.metron.fliptables</shadedPattern>
+ </relocation>
</relocations>
<artifactSet>
<excludes>
@@ -398,8 +402,10 @@
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
- <descriptor>src/main/assembly/assembly.xml</descriptor>
- <descriptor>src/main/assembly/stand-alone-assembly.xml</descriptor>
+ <descriptors>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
+ <descriptor>src/main/assembly/stand-alone-assembly.xml</descriptor>
+ </descriptors>
</configuration>
<executions>
<execution>
diff --git a/metron-stellar/stellar-common/src/main/assembly/assembly.xml b/metron-stellar/stellar-common/src/main/assembly/assembly.xml
index b50be00..4cc9735 100644
--- a/metron-stellar/stellar-common/src/main/assembly/assembly.xml
+++ b/metron-stellar/stellar-common/src/main/assembly/assembly.xml
@@ -19,18 +19,6 @@
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
- <directory>${project.basedir}/src/main/scripts/deployed</directory>
- <outputDirectory>bin</outputDirectory>
- <useDefaultExcludes>true</useDefaultExcludes>
- <excludes>
- <exclude>**/*.formatted</exclude>
- <exclude>**/*.filtered</exclude>
- </excludes>
- <fileMode>0755</fileMode>
- <lineEnding>unix</lineEnding>
- <filtered>true</filtered>
- </fileSet>
- <fileSet>
<directory>${project.basedir}/target</directory>
<includes>
<include>${project.artifactId}-${project.version}-uber.jar</include>
diff --git a/metron-stellar/stellar-common/src/main/scripts/deployed/stellar b/metron-stellar/stellar-common/src/main/scripts/deployed/stellar
index 7b0f06d..b76f398 100644
--- a/metron-stellar/stellar-common/src/main/scripts/deployed/stellar
+++ b/metron-stellar/stellar-common/src/main/scripts/deployed/stellar
@@ -31,6 +31,6 @@ fi
export HBASE_CONFIGS=$(hbase classpath)
export METRON_VERSION=${project.version}
export METRON_HOME=/usr/metron/$METRON_VERSION
-export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar)
+export STELLAR_LIB=stellar-common-$METRON_VERSION-uber.jar
export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar)
-java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib}:$STELLAR_LIB:$MANAGEMENT_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@"
+java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib}:$METRON_HOME/lib/$STELLAR_LIB:$MANAGEMENT_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@"
diff --git a/metron-stellar/stellar-zeppelin/pom.xml b/metron-stellar/stellar-zeppelin/pom.xml
index e9731b5..3984afe 100644
--- a/metron-stellar/stellar-zeppelin/pom.xml
+++ b/metron-stellar/stellar-zeppelin/pom.xml
@@ -33,6 +33,7 @@
<groupId>org.apache.metron</groupId>
<artifactId>stellar-common</artifactId>
<version>${project.parent.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>