You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2021/09/20 18:12:19 UTC
[accumulo-testing] branch main updated: Migrate accumulo-testing to
log4j2. (#146)
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new 50e4c02 Migrate accumulo-testing to log4j2. (#146)
50e4c02 is described below
commit 50e4c021e7db4df546138d43d778f0adee4bf7a5
Author: AlbertWhitlock <al...@gmail.com>
AuthorDate: Mon Sep 20 14:12:12 2021 -0400
Migrate accumulo-testing to log4j2. (#146)
Migrate accumulo-testing to use log4j2 (via slf4j)
Co-authored-by: Jeffrey Manno <je...@gmail.com>
Co-authored-by: Christopher Tubbs <ct...@apache.org>
---
Dockerfile | 2 +-
bin/cingest | 2 +-
bin/gcs | 2 +-
bin/monitor | 2 +-
bin/performance | 18 +++++++--------
bin/rwalk | 2 +-
conf/env.sh | 4 ++--
conf/log4j.properties | 27 -----------------------
conf/log4j2.properties | 56 +++++++++++++++++++++++++++++++++++++++++++++++
pom.xml | 59 ++++++++++++++++++++++++++++++++++++++++++++------
10 files changed, 124 insertions(+), 50 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 36afd92..e432a95 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -28,7 +28,7 @@ ENV HADOOP_RUNTIME_JAR /opt/at/hadoop-client-runtime.jar
ENV TEST_JAR_PATH /opt/at/accumulo-testing-shaded.jar
ENV ACCUMULO_CLIENT_PROPS /opt/at/conf/accumulo-client.properties
ENV TEST_PROPS /opt/at/conf/accumulo-testing.properties
-ENV TEST_LOG4J /opt/at/conf/log4j.properties
+ENV TEST_LOG4J /opt/at/conf/log4j2.properties
RUN mkdir /opt/at
RUN mkdir /opt/at/bin
diff --git a/bin/cingest b/bin/cingest
index 0753608..84452b6 100755
--- a/bin/cingest
+++ b/bin/cingest
@@ -92,5 +92,5 @@ case "$1" in
fi
;;
*)
- java $JAVA_OPTS -Dlog4j.configuration="file:$TEST_LOG4J" "$ci_main" "${@:2}" "$TEST_PROPS" "$ACCUMULO_CLIENT_PROPS"
+ java $JAVA_OPTS -Dlog4j.configurationFile="file:$TEST_LOG4J" "$ci_main" "${@:2}" "$TEST_PROPS" "$ACCUMULO_CLIENT_PROPS"
esac
diff --git a/bin/gcs b/bin/gcs
index 9959354..31c30af 100755
--- a/bin/gcs
+++ b/bin/gcs
@@ -61,4 +61,4 @@ esac
export CLASSPATH="$TEST_JAR_PATH:$HADOOP_API_JAR:$HADOOP_RUNTIME_JAR:$CLASSPATH"
-java $JAVA_OPTS -Dlog4j.configuration="file:$TEST_LOG4J" "$ci_main" "${@:2}" "$TEST_PROPS" "$ACCUMULO_CLIENT_PROPS"
+java $JAVA_OPTS -Dlog4j.configurationFile="file:$TEST_LOG4J" "$ci_main" "${@:2}" "$TEST_PROPS" "$ACCUMULO_CLIENT_PROPS"
diff --git a/bin/monitor b/bin/monitor
index eb606a8..6c8498d 100755
--- a/bin/monitor
+++ b/bin/monitor
@@ -48,4 +48,4 @@ case "$1" in
esac
export CLASSPATH="$TEST_JAR_PATH:$HADOOP_API_JAR:$HADOOP_RUNTIME_JAR:$CLASSPATH"
-java $JAVA_OPTS -DINSTRUMENTATIONKEY="$INSTRUMENTATIONKEY" -Dlog4j.configuration="file:$TEST_LOG4J" "$ci_main" "${@:2}" "-c" "$ACCUMULO_CLIENT_PROPS"
\ No newline at end of file
+java $JAVA_OPTS -DINSTRUMENTATIONKEY="$INSTRUMENTATIONKEY" -Dlog4j.configurationFile="file:$TEST_LOG4J" "$ci_main" "${@:2}" "-c" "$ACCUMULO_CLIENT_PROPS"
\ No newline at end of file
diff --git a/bin/performance b/bin/performance
index 45e14d6..05d87d3 100755
--- a/bin/performance
+++ b/bin/performance
@@ -46,9 +46,9 @@ function build_shade_jar() {
fi
}
-log4j_config="$at_home/conf/log4j.properties"
+log4j_config="$at_home/conf/log4j2.properties"
if [ ! -f "$log4j_config" ]; then
- echo "Could not find logj4.properties in $at_home/conf"
+ echo "Could not find log4j2.properties in $at_home/conf"
exit 1
fi
@@ -71,7 +71,7 @@ case "$1" in
fi
# don't start unless we can find the class provided
found=false
- CLASSPATH="$CP" java $JAVA_OPTS -Dlog4j.configuration="file:$log4j_config" ${perf_pkg}.ListTests | while read -r test_class; do
+ CLASSPATH="$CP" java $JAVA_OPTS -Dlog4j.configurationFile="file:$log4j_config" ${perf_pkg}.ListTests | while read -r test_class; do
if [[ "$test_class" == "$3" ]]; then
found=true
fi
@@ -83,30 +83,30 @@ case "$1" in
fi
mkdir -p "$2"
start_cluster
- CLASSPATH="$CP" java $JAVA_OPTS -Dlog4j.configuration="file:$log4j_config" ${perf_pkg}.ListTests | while read -r test_class; do
+ CLASSPATH="$CP" java $JAVA_OPTS -Dlog4j.configurationFile="file:$log4j_config" ${perf_pkg}.ListTests | while read -r test_class; do
if [[ -z $3 || $test_class == *$3* ]]; then
echo "Running test $test_class"
pt_tmp=$(mktemp -d -t accumulo_pt_XXXXXXX)
setup_accumulo
get_config_file accumulo.properties "$pt_tmp"
- CLASSPATH="$CP" java $JAVA_OPTS -Dlog4j.configuration="file:$log4j_config" ${perf_pkg}.MergeSiteConfig "$test_class" "$pt_tmp"
+ CLASSPATH="$CP" java $JAVA_OPTS -Dlog4j.configurationFile="file:$log4j_config" ${perf_pkg}.MergeSiteConfig "$test_class" "$pt_tmp"
put_config_file "$pt_tmp/accumulo.properties"
put_server_code "$at_home/target/accumulo-testing-$at_version.jar"
start_accumulo
get_config_file accumulo-client.properties "$pt_tmp"
- CLASSPATH="$CP" java $JAVA_OPTS -Dlog4j.configuration="file:$log4j_config" ${perf_pkg}.PerfTestRunner "$pt_tmp/accumulo-client.properties" "$test_class" "$(get_version 'ACCUMULO')" "$2"
+ CLASSPATH="$CP" java $JAVA_OPTS -Dlog4j.configurationFile="file:$log4j_config" ${perf_pkg}.PerfTestRunner "$pt_tmp/accumulo-client.properties" "$test_class" "$(get_version 'ACCUMULO')" "$2"
fi
done
stop_cluster
;;
compare)
- CLASSPATH="$CP" java $JAVA_OPTS -Dlog4j.configuration="file:$log4j_config" ${perf_pkg}.Compare "$2" "$3"
+ CLASSPATH="$CP" java $JAVA_OPTS -Dlog4j.configurationFile="file:$log4j_config" ${perf_pkg}.Compare "$2" "$3"
;;
csv)
- CLASSPATH="$CP" java $JAVA_OPTS -Dlog4j.configuration="file:$log4j_config" ${perf_pkg}.Csv "${@:2}"
+ CLASSPATH="$CP" java $JAVA_OPTS -Dlog4j.configurationFile="file:$log4j_config" ${perf_pkg}.Csv "${@:2}"
;;
list)
- CLASSPATH="$CP" java $JAVA_OPTS -Dlog4j.configuration="file:$log4j_config" ${perf_pkg}.ListTests
+ CLASSPATH="$CP" java $JAVA_OPTS -Dlog4j.configurationFile="file:$log4j_config" ${perf_pkg}.ListTests
;;
*)
echo "Unknown command : $1"
diff --git a/bin/rwalk b/bin/rwalk
index ab9a36c..42009d3 100755
--- a/bin/rwalk
+++ b/bin/rwalk
@@ -41,4 +41,4 @@ if [ -z "$1" ]; then
exit 1
fi
-java $JAVA_OPTS -Dlog4j.configuration="file:$TEST_LOG4J" "$randomwalk_main" "$TEST_PROPS" "$ACCUMULO_CLIENT_PROPS" "$1"
+java $JAVA_OPTS -Dlog4j.configurationFile="file:$TEST_LOG4J" "$randomwalk_main" "$TEST_PROPS" "$ACCUMULO_CLIENT_PROPS" "$1"
diff --git a/conf/env.sh b/conf/env.sh
index f4b4a2a..aab56dc 100644
--- a/conf/env.sh
+++ b/conf/env.sh
@@ -50,9 +50,9 @@ if [ ! -f "$TEST_PROPS" ]; then
echo "Please create and edit accumulo-testing.properties in $conf_dir"
exit 1
fi
-export TEST_LOG4J="${conf_dir}/log4j.properties"
+export TEST_LOG4J="${conf_dir}/log4j2.properties"
if [ ! -f "$TEST_LOG4J" ]; then
- echo "Could not find logj4.properties in $conf_dir"
+ echo "Could not find log4j2.properties in $conf_dir"
exit 1
fi
diff --git a/conf/log4j.properties b/conf/log4j.properties
deleted file mode 100644
index 525a6e4..0000000
--- a/conf/log4j.properties
+++ /dev/null
@@ -1,27 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-log4j.rootLogger=INFO, CA
-log4j.appender.CA=org.apache.log4j.ConsoleAppender
-log4j.appender.CA.layout=org.apache.log4j.PatternLayout
-log4j.appender.CA.layout.ConversionPattern=%d{ISO8601} [%c{3}] %-5p: %m%n
-
-log4j.logger.org.apache.accumulo=WARN
-log4j.logger.org.apache.accumulo.testing=DEBUG
-log4j.logger.org.apache.curator=ERROR
-log4j.logger.org.apache.hadoop=WARN
-log4j.logger.org.apache.hadoop.mapreduce=ERROR
-log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-log4j.logger.org.apache.zookeeper=ERROR
diff --git a/conf/log4j2.properties b/conf/log4j2.properties
new file mode 100644
index 0000000..7ca2c33
--- /dev/null
+++ b/conf/log4j2.properties
@@ -0,0 +1,56 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name = AccumuloTestingDefaultLoggingProperties
+status = info
+dest = err
+
+appenders = console
+appender.console.type = Console
+appender.console.name = STDERR
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{ISO8601} [%c{3}] %-5p: %m%n
+
+loggers = accumulotesting, accumulo, hadooptest, hadoopmapred, hadooputil, zookeepertest, curatortest
+logger.accumulotesting.name = org.apache.accumulo.testing
+logger.accumulotesting.level = DEBUG
+
+logger.accumulo.name = org.apache.accumulo
+logger.accumulo.level = WARN
+
+logger.hadooptest.name = org.apache.hadoop
+logger.hadooptest.level = WARN
+logger.hadoopmapred.name = org.apache.hadoop.mapreduce
+logger.hadoopmapred.level = ERROR
+logger.hadooputil.name = org.apache.hadoop.util
+logger.hadooputil.level = ERROR
+
+logger.zookeepertest.name = org.apache.zookeeper
+logger.zookeepertest.level = ERROR
+
+logger.curatortest.name = org.apache.curator.test
+logger.curatortest.level = ERROR
+
+rootLogger.level = info
+rootLogger.appenderRefs = stderr
+rootLogger.appenderRef.stderr.ref = STDERR
+
+
+
diff --git a/pom.xml b/pom.xml
index 7f782fd..45176b3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,12 +29,14 @@
<description>Testing tools for Apache Accumulo</description>
<properties>
<accumulo.version>2.1.0-SNAPSHOT</accumulo.version>
+ <disruptor.version>3.4.4</disruptor.version>
<eclipseFormatterStyle>${project.basedir}/contrib/Eclipse-Accumulo-Codestyle.xml</eclipseFormatterStyle>
<hadoop.version>3.2.0</hadoop.version>
+ <log4j2.version>2.4</log4j2.version>
<maven.compiler.release>11</maven.compiler.release>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
- <slf4j.version>1.7.25</slf4j.version>
+ <slf4j.version>1.7.30</slf4j.version>
<spotbugs.version>3.1.7</spotbugs.version>
<zookeeper.version>3.4.14</zookeeper.version>
</properties>
@@ -44,6 +46,19 @@
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>${zookeeper.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-bom</artifactId>
+ <version>${log4j2.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
@@ -60,6 +75,11 @@
<version>30.1.1-jre</version>
</dependency>
<dependency>
+ <groupId>com.lmax</groupId>
+ <artifactId>disruptor</artifactId>
+ <version>${disruptor.version}</version>
+ </dependency>
+ <dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.11</version>
@@ -95,13 +115,12 @@
<version>${hadoop.version}</version>
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>${slf4j.version}</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
+ <artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
@@ -121,6 +140,12 @@
<artifactId>accumulo-test</artifactId>
<version>${accumulo.version}</version>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
<build>
@@ -140,6 +165,17 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
+ <configuration>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>**/Log4j2Plugins.dat</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
</plugin>
<plugin>
<groupId>com.github.ekryd.sortpom</groupId>
@@ -342,6 +378,13 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>com.github.edwgiz</groupId>
+ <artifactId>maven-shade-plugin.log4j2-cachefile-transformer</artifactId>
+ <version>2.1</version>
+ </dependency>
+ </dependencies>
<executions>
<execution>
<id>testing-shade-jar</id>
@@ -357,8 +400,7 @@
<excludes>
<exclude>org.apache.accumulo:accumulo-native</exclude>
<exclude>org.apache.hadoop:*</exclude>
- <exclude>org.apache.logging.log4j:log4j-api:jar:</exclude>
- <exclude>org.apache.logging.log4j:log4j-1.2-api:jar:</exclude>
+ <exclude>log4j:*</exclude>
</excludes>
</artifactSet>
<filters>
@@ -374,6 +416,9 @@
<transformers>
<!-- Hadoop uses service loader to find filesystem impls, without this may not find them -->
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <!-- New transformer. Provides workaround for 2 bugs. Read README.md for
+ more information at: https://github.com/edwgiz/maven-shaded-log4j-transformer -->
+ <transformer implementation="com.github.edwgiz.mavenShadePlugin.log4j2CacheTransformer.PluginsCacheFileTransformer" />
</transformers>
</configuration>
</execution>