You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2019/03/20 14:00:10 UTC
[metron] branch master updated: METRON-2038 Enrichment Loader Fails
When Run as MR Job (nickwallen) closes apache/metron#1358
This is an automated email from the ASF dual-hosted git repository.
nickallen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new e63faf8 METRON-2038 Enrichment Loader Fails When Run as MR Job (nickwallen) closes apache/metron#1358
e63faf8 is described below
commit e63faf84f1b227c44b2d6b224b89c74449bcf0cc
Author: nickwallen <ni...@nickallen.org>
AuthorDate: Wed Mar 20 09:59:43 2019 -0400
METRON-2038 Enrichment Loader Fails When Run as MR Job (nickwallen) closes apache/metron#1358
---
dependencies_with_url.csv | 7 ++++++-
metron-platform/metron-data-management/pom.xml | 9 +++++----
.../src/main/scripts/flatfile_loader.sh | 23 ++++++++--------------
3 files changed, 19 insertions(+), 20 deletions(-)
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 0ae7ab4..04f7935 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -194,6 +194,7 @@ com.google.guava:guava:jar:18.0:compile,ASLv2,
com.google.inject.extensions:guice-servlet:jar:3.0:compile,ASLv2,
com.google.inject:guice:jar:3.0:compile,ASLv2,
com.googlecode.disruptor:disruptor:jar:2.10.4:compile,The Apache Software License, Version 2.0,http://code.google.com/p/disruptor/
+com.lmax:disruptor:jar:3.3.0:compile,The Apache Software License, Version 2.0,https://github.com/LMAX-Exchange/disruptor/
com.lmax:disruptor:jar:3.3.2:compile,The Apache Software License, Version 2.0,https://github.com/LMAX-Exchange/disruptor/
com.googlecode.json-simple:json-simple:jar:1.1:compile,The Apache Software License, Version 2.0,http://code.google.com/p/json-simple/
com.googlecode.json-simple:json-simple:jar:1.1.1:compile,The Apache Software License, Version 2.0,http://code.google.com/p/json-simple/
@@ -301,6 +302,10 @@ org.jboss.resteasy:resteasy-jaxrs:jar:3.0.1.Final:compile,ASLv2,
org.joda:joda-convert:jar:1.2:compile,Apache 2,http://joda-convert.sourceforge.net
org.mortbay.jetty:jetty-util:jar:6.1.26:compile,ASLv2,
org.mortbay.jetty:jetty:jar:6.1.26:compile,ASLv2,
+org.mortbay.jetty:jetty-sslengine:jar:6.1.26:compile,ASLv2,
+org.mortbay.jetty:jsp-2.1:jar:6.1.14:compile,ASLv2,
+org.mortbay.jetty:jsp-api-2.1:jar:6.1.14:compile,ASLv2,
+org.mortbay.jetty:servlet-api-2.5:jar:6.1.14:compile,ASLv2,
org.noggit:noggit:jar:0.6:compile,Apache License, Version 2.0,http://github.com/yonik/noggit
org.scannotation:scannotation:jar:1.0.3:compile,Apache License V2.0,http://scannotation.sf.net
org.slf4j:log4j-over-slf4j:jar:1.6.6:compile,Apache Software Licenses,http://www.slf4j.org
@@ -498,4 +503,4 @@ joda-time:joda-time:jar:2.10:compile
org.elasticsearch:securesm:jar:1.2:compile
com.github.stephenc.jcip:jcip-annotations:jar:1.0-1:compile,ASLv2,http://stephenc.github.io/jcip-annotations/
com.nimbusds:nimbus-jose-jwt:jar:4.41.2:compile,ASLv2,https://bitbucket.org/connect2id/nimbus-jose-jwt/wiki/Home
-
+tomcat:jasper-compiler:jar:5.5.23:compile,ASLv2,https://tomcat.apache.org/
diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml
index 72412ed..22e3947 100644
--- a/metron-platform/metron-data-management/pom.xml
+++ b/metron-platform/metron-data-management/pom.xml
@@ -186,7 +186,6 @@
<artifactId>hadoop-common</artifactId>
</exclusion>
</exclusions>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
@@ -210,7 +209,6 @@
<artifactId>hadoop-common</artifactId>
</exclusion>
</exclusions>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>com.opencsv</groupId>
@@ -344,7 +342,6 @@
<scope>test</scope>
</dependency>
</dependencies>
-
<build>
<plugins>
<plugin>
@@ -380,7 +377,6 @@
</relocation>
</relocations>
<transformers>
- <transformer implementation="org.atteo.classindex.ClassIndexTransformer"/>
<transformer
implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
<resources>
@@ -395,6 +391,11 @@
<addHeader>false</addHeader>
<projectName>${project.name}</projectName>
</transformer-->
+ <!--
+ ClassIndexTransformer needs to go LAST. For some reason it will clobber other
+ transformers from operating when it is put first.
+ -->
+ <transformer implementation="org.atteo.classindex.ClassIndexTransformer"/>
</transformers>
<artifactSet>
<excludes>
diff --git a/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh b/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh
index f92d418..feccc33 100755
--- a/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh
+++ b/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-#
+#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -7,15 +7,15 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
+#
BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
@@ -31,21 +31,14 @@ export METRON_VERSION=${project.version}
export METRON_HOME=/usr/metron/$METRON_VERSION
export CLASSNAME="org.apache.metron.dataloads.nonbulk.flatfile.SimpleEnrichmentFlatFileLoader"
export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
-export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
+export HBASE_CONF=${HBASE_CONF:-/etc/hbase/conf}
export HADOOP_OPTS="$HADOOP_OPTS $METRON_JVMFLAGS"
if [ $(which hadoop) ]
then
- HADOOP_CLASSPATH=${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath`
- for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
- if [ -f $jar ];then
- LIBJARS="$jar,$LIBJARS"
- fi
- done
- export HADOOP_CLASSPATH
- hadoop jar $METRON_HOME/lib/$DM_JAR $CLASSNAME -libjars ${LIBJARS} "$@"
+ export HADOOP_CLASSPATH="$HBASE_CONF"
+ hadoop jar $METRON_HOME/lib/$DM_JAR $CLASSNAME "$@"
else
echo "Warning: Metron cannot find the hadoop client on this node. This means that loading via Map Reduce will NOT function."
- CP=$METRON_HOME/lib/$DM_JAR:/usr/metron/${METRON_VERSION}/lib/taxii-1.1.0.1.jar:`${HBASE_HOME}/bin/hbase classpath`
+ CP=$METRON_HOME/lib/$DM_JAR:$HBASE_CONF
java $METRON_JVMFLAGS -cp $CP $CLASSNAME "$@"
fi
-