You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@accumulo.apache.org by GitBox <gi...@apache.org> on 2018/11/28 19:12:30 UTC
[GitHub] mikewalch closed pull request #44: Build shaded jar without Hadoop
for M/R
mikewalch closed pull request #44: Build shaded jar without Hadoop for M/R
URL: https://github.com/apache/accumulo-testing/pull/44
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/Dockerfile b/Dockerfile
index 85cce6e..4038764 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -18,24 +18,30 @@ FROM centos:7
RUN yum install -y java-1.8.0-openjdk-devel
ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk
-ARG TEST_JAR_VERSION
-ENV TEST_JAR_VERSION 2.0.0-SNAPSHOT
-ENV TEST_JAR_PATH /opt/accumulo-testing-${TEST_JAR_VERSION}-shaded.jar
-ENV ACCUMULO_CLIENT_PROPS /opt/conf/accumulo-client.properties
-ENV TEST_PROPS /opt/conf/accumulo-testing.properties
-ENV TEST_LOG4J /opt/conf/log4j.properties.example
-
-RUN mkdir /opt/bin
-RUN mkdir /opt/conf
-RUN touch /opt/conf/env.sh
-
-ADD ./conf/accumulo-client.properties /opt/conf/
-ADD ./conf/accumulo-testing.properties /opt/conf/
-ADD ./conf/log4j.properties.example /opt/conf/
-ADD ./bin/cingest /opt/bin
-ADD ./bin/rwalk /opt/bin
-ADD ./src/main/docker/docker-entry /opt/bin
-ADD ./target/accumulo-testing-${TEST_JAR_VERSION}-shaded.jar /opt/
-
-ENTRYPOINT ["/opt/bin/docker-entry"]
+ENV HADOOP_API_JAR /opt/at/hadoop-client-api.jar
+ENV HADOOP_RUNTIME_JAR /opt/at/hadoop-client-runtime.jar
+ENV TEST_JAR_PATH /opt/at/accumulo-testing-shaded.jar
+ENV ACCUMULO_CLIENT_PROPS /opt/at/conf/accumulo-client.properties
+ENV TEST_PROPS /opt/at/conf/accumulo-testing.properties
+ENV TEST_LOG4J /opt/at/conf/log4j.properties
+
+RUN mkdir /opt/at
+RUN mkdir /opt/at/bin
+RUN mkdir /opt/at/conf
+
+COPY ./conf/accumulo-client.properties /opt/at/conf/
+COPY ./conf/accumulo-testing.properties /opt/at/conf/
+COPY ./conf/log4j.properties* /opt/at/conf/
+RUN if [[ ! -f /opt/at/conf/log4j.properties ]]; then mv /opt/at/conf/log4j.properties.example /opt/at/conf/log4j.properties; fi
+RUN touch /opt/at/conf/env.sh
+
+COPY ./bin/cingest /opt/at/bin
+COPY ./bin/rwalk /opt/at/bin
+COPY ./src/main/docker/docker-entry /opt/at/bin
+
+COPY ./target/accumulo-testing-shaded.jar /opt/at/
+COPY ./target/dependency/hadoop-client-api.jar /opt/at/
+COPY ./target/dependency/hadoop-client-runtime.jar /opt/at/
+
+ENTRYPOINT ["/opt/at/bin/docker-entry"]
CMD ["help"]
diff --git a/bin/cingest b/bin/cingest
index 5801a12..6cbbb40 100755
--- a/bin/cingest
+++ b/bin/cingest
@@ -77,12 +77,11 @@ case "$1" in
exit 1
esac
-export CLASSPATH="$TEST_JAR_PATH:$CLASSPATH"
+export CLASSPATH="$TEST_JAR_PATH:$HADOOP_API_JAR:$HADOOP_RUNTIME_JAR:$CLASSPATH"
case "$1" in
verify|moru)
if [ ! -z $HADOOP_HOME ]; then
- export CLASSPATH=$HADOOP_HOME/etc/hadoop:$CLASSPATH
export HADOOP_USE_CLIENT_CLASSLOADER=true
"$HADOOP_HOME"/bin/yarn jar "$TEST_JAR_PATH" "$ci_main" "$TEST_PROPS" "$ACCUMULO_CLIENT_PROPS"
else
diff --git a/bin/rwalk b/bin/rwalk
index 8a17b4e..30cf0a7 100755
--- a/bin/rwalk
+++ b/bin/rwalk
@@ -34,7 +34,7 @@ else
. "$at_home"/conf/env.sh.example
fi
-export CLASSPATH="$TEST_JAR_PATH:$CLASSPATH"
+export CLASSPATH="$TEST_JAR_PATH:$HADOOP_API_JAR:$HADOOP_RUNTIME_JAR:$CLASSPATH"
randomwalk_main="org.apache.accumulo.testing.randomwalk.Framework"
diff --git a/conf/env.sh.example b/conf/env.sh.example
index a3368fc..0a2cdbe 100644
--- a/conf/env.sh.example
+++ b/conf/env.sh.example
@@ -42,21 +42,27 @@ fi
# Shaded test jar
# ===============
-# Version of shaded test jar
-export TEST_JAR_VERSION=2.0.0-SNAPSHOT
# Versions set below will be what is included in the shaded jar
export ACCUMULO_VERSION="`$ACCUMULO_HOME/bin/accumulo version`"
export HADOOP_VERSION="`hadoop version | head -n1 | awk '{print $2}'`"
export ZOOKEEPER_VERSION=3.4.9
# Path to shaded test jar
at_home=$( cd "$( dirname "$conf_dir" )" && pwd )
-export TEST_JAR_PATH="${at_home}/target/accumulo-testing-${TEST_JAR_VERSION}-shaded.jar"
-if [ ! -f "$TEST_JAR_PATH" ]; then
+export TEST_JAR_PATH="${at_home}/target/accumulo-testing-shaded.jar"
+if [[ ! -f "$TEST_JAR_PATH" ]]; then
echo "Building $TEST_JAR_PATH"
cd "${at_home}" || exit 1
mvn clean package -P create-shade-jar -D skipTests -D accumulo.version="$ACCUMULO_VERSION" -D hadoop.version="$HADOOP_VERSION" -D zookeeper.version="$ZOOKEEPER_VERSION"
fi
+# Hadoop jars
+# ===========
+export HADOOP_API_JAR="${at_home}"/target/dependency/hadoop-client-api.jar
+export HADOOP_RUNTIME_JAR="${at_home}"/target/dependency/hadoop-client-runtime.jar
+if [[ ! -f "$HADOOP_API_JAR" || ! -f "$HADOOP_RUNTIME_JAR" ]]; then
+ mvn dependency:copy-dependencies -Dmdep.stripVersion=true -DincludeArtifactIds=hadoop-client-api,hadoop-client-runtime -Dhadoop.version="$HADOOP_VERSION"
+fi
+
# Agitator
# ========
# Accumulo user
diff --git a/pom.xml b/pom.xml
index 8d4c940..a9c534f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -208,11 +208,13 @@
</goals>
<phase>package</phase>
<configuration>
+ <finalName>${project.artifactId}-shaded</finalName>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>shaded</shadedClassifierName>
<artifactSet>
<excludes>
<exclude>org.apache.accumulo:accumulo-native</exclude>
+ <exclude>org.apache.hadoop:*</exclude>
</excludes>
</artifactSet>
<filters>
@@ -222,8 +224,6 @@
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
- <!-- Twill 0.13 fails if shaded jar includes this directory !-->
- <exclude>META-INF/license/*</exclude>
</excludes>
</filter>
</filters>
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services