You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2013/01/11 01:39:49 UTC
git commit: CRUNCH-113: Move the log4j shenanigans out of MRPipeline
and into crunch-test's DebugLogging and make the debug logging code more
robust and configurable.
Updated Branches:
refs/heads/master 522a691c9 -> 018d16984
CRUNCH-113: Move the log4j shenanigans out of MRPipeline and into crunch-test's DebugLogging
and make the debug logging code more robust and configurable.
Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/018d1698
Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/018d1698
Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/018d1698
Branch: refs/heads/master
Commit: 018d16984f127a4a14ae3ab2d0af0ac839dfb439
Parents: 522a691
Author: Josh Wills <jw...@apache.org>
Authored: Tue Nov 13 13:23:41 2012 -0800
Committer: Josh Wills <jw...@apache.org>
Committed: Thu Jan 10 16:30:50 2013 -0800
----------------------------------------------------------------------
crunch-test/pom.xml | 6 ++
.../java/org/apache/crunch/test/DebugLogging.java | 56 +++++++++++++++
crunch/pom.xml | 6 --
.../java/org/apache/crunch/impl/mr/MRPipeline.java | 15 ----
4 files changed, 62 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/018d1698/crunch-test/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-test/pom.xml b/crunch-test/pom.xml
index 9185ef0..4953783 100644
--- a/crunch-test/pom.xml
+++ b/crunch-test/pom.xml
@@ -49,6 +49,12 @@ under the License.
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
<!-- Required by LocalJobRunner -->
<dependency>
<groupId>commons-io</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/018d1698/crunch-test/src/main/java/org/apache/crunch/test/DebugLogging.java
----------------------------------------------------------------------
diff --git a/crunch-test/src/main/java/org/apache/crunch/test/DebugLogging.java b/crunch-test/src/main/java/org/apache/crunch/test/DebugLogging.java
new file mode 100644
index 0000000..3bf2303
--- /dev/null
+++ b/crunch-test/src/main/java/org/apache/crunch/test/DebugLogging.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch.test;
+
+import org.apache.log4j.Appender;
+import org.apache.log4j.ConsoleAppender;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
+
+/**
+ * Allows direct manipulation of the Hadoop log4j settings to aid in
+ * unit testing. Not recommended for production use.
+ */
+public final class DebugLogging {
+
+ /**
+ * Enables logging Hadoop output to the console using the pattern
+ * '%-4r [%t] %-5p %c %x - %m%n' at the specified {@code Level}.
+ *
+ * @param level The log4j level
+ */
+ public static void enable(Level level) {
+ enable(level, new ConsoleAppender(new PatternLayout("%-4r [%t] %-5p %c %x - %m%n")));
+ }
+
+ /**
+ * Enables logging to the given {@code Appender} at the specified {@code Level}.
+ *
+ * @param level The log4j level
+ * @param appender The log4j appender
+ */
+ public static void enable(Level level, Appender appender) {
+ Logger hadoopLogger = LogManager.getLogger("org.apache.hadoop");
+ hadoopLogger.setLevel(level);
+ hadoopLogger.addAppender(appender);
+ }
+
+ private DebugLogging() { }
+}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/018d1698/crunch/pom.xml
----------------------------------------------------------------------
diff --git a/crunch/pom.xml b/crunch/pom.xml
index f8993c1..3fcdacf 100644
--- a/crunch/pom.xml
+++ b/crunch/pom.xml
@@ -111,12 +111,6 @@ under the License.
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<scope>provided</scope>
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/018d1698/crunch/src/main/java/org/apache/crunch/impl/mr/MRPipeline.java
----------------------------------------------------------------------
diff --git a/crunch/src/main/java/org/apache/crunch/impl/mr/MRPipeline.java b/crunch/src/main/java/org/apache/crunch/impl/mr/MRPipeline.java
index 40c5863..4d8fa84 100644
--- a/crunch/src/main/java/org/apache/crunch/impl/mr/MRPipeline.java
+++ b/crunch/src/main/java/org/apache/crunch/impl/mr/MRPipeline.java
@@ -52,10 +52,6 @@ import org.apache.crunch.types.writable.WritableTypeFamily;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.log4j.Appender;
-import org.apache.log4j.Level;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -344,17 +340,6 @@ public class MRPipeline implements Pipeline {
public void enableDebug() {
// Turn on Crunch runtime error catching.
getConfiguration().setBoolean(RuntimeParameters.DEBUG, true);
-
- // Write Hadoop's WARN logs to the console.
- Logger crunchInfoLogger = LogManager.getLogger("org.apache.crunch");
- Appender console = crunchInfoLogger.getAppender("A");
- if (console != null) {
- Logger hadoopLogger = LogManager.getLogger("org.apache.hadoop");
- hadoopLogger.setLevel(Level.WARN);
- hadoopLogger.addAppender(console);
- } else {
- LOG.warn("Could not find console appender named 'A' for writing Hadoop warning logs");
- }
}
@Override