You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2012/06/21 09:32:48 UTC
git commit: MRPipeline's enableDebug method will now write Hadoop's
WARN log messages to the console
Updated Branches:
refs/heads/master e578d29d2 -> e74fc23e1
MRPipeline's enableDebug method will now write Hadoop's WARN log messages to the console
Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/e74fc23e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/e74fc23e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/e74fc23e
Branch: refs/heads/master
Commit: e74fc23e182418572fcf44c66bfd5065c7a56dc1
Parents: e578d29
Author: Josh Wills <jw...@cloudera.com>
Authored: Thu Jun 21 00:32:33 2012 -0700
Committer: Josh Wills <jw...@cloudera.com>
Committed: Thu Jun 21 00:32:33 2012 -0700
----------------------------------------------------------------------
.../com/cloudera/crunch/impl/mr/MRPipeline.java | 18 ++++++++++++++-
1 files changed, 17 insertions(+), 1 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/e74fc23e/src/main/java/com/cloudera/crunch/impl/mr/MRPipeline.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/cloudera/crunch/impl/mr/MRPipeline.java b/src/main/java/com/cloudera/crunch/impl/mr/MRPipeline.java
index e491b01..420e8dc 100644
--- a/src/main/java/com/cloudera/crunch/impl/mr/MRPipeline.java
+++ b/src/main/java/com/cloudera/crunch/impl/mr/MRPipeline.java
@@ -25,6 +25,10 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Appender;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
import com.cloudera.crunch.MapFn;
import com.cloudera.crunch.PCollection;
@@ -259,7 +263,19 @@ public class MRPipeline implements Pipeline {
@Override
public void enableDebug() {
- getConfiguration().setBoolean(RuntimeParameters.DEBUG, true);
+ // Turn on Crunch runtime error catching.
+ getConfiguration().setBoolean(RuntimeParameters.DEBUG, true);
+
+ // Write Hadoop's WARN logs to the console.
+ Logger crunchInfoLogger = LogManager.getLogger("com.cloudera.crunch");
+ Appender console = crunchInfoLogger.getAppender("A");
+ if (console != null) {
+ Logger hadoopLogger = LogManager.getLogger("org.apache.hadoop");
+ hadoopLogger.setLevel(Level.WARN);
+ hadoopLogger.addAppender(console);
+ } else {
+ LOG.warn("Could not find console appender named 'A' for writing Hadoop warning logs");
+ }
}
@Override