You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2014/02/28 03:49:50 UTC
git commit: TEZ-535. Redirect AM logs to different files when running
multiple DAGs in the same AM. Contributed by Mohammad Kamrul Islam.
Repository: incubator-tez
Updated Branches:
refs/heads/master 74078d309 -> 6d29107ce
TEZ-535. Redirect AM logs to different files when running multiple DAGs
in the same AM. Contributed by Mohammad Kamrul Islam.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/6d29107c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/6d29107c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/6d29107c
Branch: refs/heads/master
Commit: 6d29107ceef4f7972a4f134caaf8e93307b99fc7
Parents: 74078d3
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu Feb 27 18:49:07 2014 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Feb 27 18:49:07 2014 -0800
----------------------------------------------------------------------
.../tez/common/TezContainerLogAppender.java | 79 ++++++++++++++++++++
.../java/org/apache/tez/common/TezUtils.java | 45 +++++++++++
.../hadoop/mapred/TezContainerLogAppender.java | 79 --------------------
.../apache/hadoop/mapred/YarnTezDagChild.java | 45 +----------
.../org/apache/tez/dag/app/DAGAppMaster.java | 20 ++++-
.../resources/tez-container-log4j.properties | 2 +-
6 files changed, 145 insertions(+), 125 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d29107c/tez-common/src/main/java/org/apache/tez/common/TezContainerLogAppender.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezContainerLogAppender.java b/tez-common/src/main/java/org/apache/tez/common/TezContainerLogAppender.java
new file mode 100644
index 0000000..a6455c3
--- /dev/null
+++ b/tez-common/src/main/java/org/apache/tez/common/TezContainerLogAppender.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common;
+
+import java.io.File;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.log4j.FileAppender;
+import org.apache.tez.dag.api.TezConfiguration;
+
+/**
+ * A simple log4j-appender for a tez container's logs.
+ *
+ */
+@Public
+@Unstable
+public class TezContainerLogAppender extends FileAppender {
+ private String containerLogDir;
+ private String logFileName = TezConfiguration.TEZ_CONTAINER_LOG_FILE_NAME;
+ //so that log4j can configure it from the configuration(log4j.properties).
+
+ @Override
+ public void activateOptions() {
+ synchronized (this) {
+ setFile(new File(this.containerLogDir, logFileName).toString());
+ setAppend(true);
+ super.activateOptions();
+ }
+ }
+
+ /**
+ * Set the name of the file for logging. This should NOT be an absolute path.
+ * The file will be created within the container's log directory.
+ *
+ * @param fileName
+ */
+ public void setLogFileName(String fileName) {
+ if (fileName == null || fileName.contains(File.pathSeparator)) {
+ throw new RuntimeException(
+ "Invalid filename specified: "
+ + fileName
+ + " . FileName should not have a path component and should not be empty.");
+ }
+ this.logFileName = fileName;
+ }
+
+ public String getLogFileName() {
+ return logFileName;
+ }
+
+ /**
+ * Getter/Setter methods for log4j.
+ */
+
+ public String getContainerLogDir() {
+ return this.containerLogDir;
+ }
+
+ public void setContainerLogDir(String containerLogDir) {
+ this.containerLogDir = containerLogDir;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d29107c/tez-common/src/main/java/org/apache/tez/common/TezUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezUtils.java b/tez-common/src/main/java/org/apache/tez/common/TezUtils.java
index 3df9adb..c0f5528 100644
--- a/tez-common/src/main/java/org/apache/tez/common/TezUtils.java
+++ b/tez-common/src/main/java/org/apache/tez/common/TezUtils.java
@@ -18,9 +18,12 @@
package org.apache.tez.common;
import java.io.ByteArrayOutputStream;
+import java.io.File;
import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
+import java.io.PrintStream;
import java.util.Iterator;
import java.util.List;
import java.util.Map.Entry;
@@ -33,6 +36,8 @@ import java.util.zip.InflaterInputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Appender;
+import org.apache.log4j.Logger;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto;
import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair;
@@ -216,4 +221,44 @@ public class TezUtils {
byte[] output = bos.toByteArray();
return output;
}
+
+ public static void updateLoggers(String addend) throws FileNotFoundException {
+ String containerLogDir = null;
+
+ LOG.info("Redirecting log files based on addend: " + addend);
+
+ Appender appender = Logger.getRootLogger().getAppender(
+ TezConfiguration.TEZ_CONTAINER_LOGGER_NAME);
+ if (appender != null) {
+ if (appender instanceof TezContainerLogAppender) {
+ TezContainerLogAppender claAppender = (TezContainerLogAppender) appender;
+ containerLogDir = claAppender.getContainerLogDir();
+ claAppender.setLogFileName(constructLogFileName(
+ TezConfiguration.TEZ_CONTAINER_LOG_FILE_NAME, addend));
+ claAppender.activateOptions();
+ } else {
+ LOG.warn("Appender is a " + appender.getClass() + "; require an instance of "
+ + TezContainerLogAppender.class.getName() + " to reconfigure the logger output");
+ }
+ } else {
+ LOG.warn("Not configured with appender named: " + TezConfiguration.TEZ_CONTAINER_LOGGER_NAME
+ + ". Cannot reconfigure logger output");
+ }
+
+ if (containerLogDir != null) {
+ System.setOut(new PrintStream(new File(containerLogDir, constructLogFileName(
+ TezConfiguration.TEZ_CONTAINER_OUT_FILE_NAME, addend))));
+ System.setErr(new PrintStream(new File(containerLogDir, constructLogFileName(
+ TezConfiguration.TEZ_CONTAINER_ERR_FILE_NAME, addend))));
+ }
+ }
+
+ private static String constructLogFileName(String base, String addend) {
+ if (addend == null || addend.isEmpty()) {
+ return base;
+ } else {
+ return base + "_" + addend;
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d29107c/tez-dag/src/main/java/org/apache/hadoop/mapred/TezContainerLogAppender.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/TezContainerLogAppender.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/TezContainerLogAppender.java
deleted file mode 100644
index cd5f4c9..0000000
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/TezContainerLogAppender.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.File;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.log4j.FileAppender;
-import org.apache.tez.dag.api.TezConfiguration;
-
-/**
- * A simple log4j-appender for a tez container's logs.
- *
- */
-@Public
-@Unstable
-public class TezContainerLogAppender extends FileAppender {
- private String containerLogDir;
- private String logFileName = TezConfiguration.TEZ_CONTAINER_LOG_FILE_NAME;
- //so that log4j can configure it from the configuration(log4j.properties).
-
- @Override
- public void activateOptions() {
- synchronized (this) {
- setFile(new File(this.containerLogDir, logFileName).toString());
- setAppend(true);
- super.activateOptions();
- }
- }
-
- /**
- * Set the name of the file for logging. This should NOT be an absolute path.
- * The file will be created within the container's log directory.
- *
- * @param fileName
- */
- public void setLogFileName(String fileName) {
- if (fileName == null || fileName.contains(File.pathSeparator)) {
- throw new RuntimeException(
- "Invalid filename specified: "
- + fileName
- + " . FileName should not have a path component and should not be empty.");
- }
- this.logFileName = fileName;
- }
-
- public String getLogFileName() {
- return logFileName;
- }
-
- /**
- * Getter/Setter methods for log4j.
- */
-
- public String getContainerLogDir() {
- return this.containerLogDir;
- }
-
- public void setContainerLogDir(String containerLogDir) {
- this.containerLogDir = containerLogDir;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d29107c/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
index 1288369..81d211a 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
@@ -679,50 +679,13 @@ public class YarnTezDagChild {
private static void updateLoggers(TezTaskAttemptID tezTaskAttemptID)
throws FileNotFoundException {
- String containerLogDir = null;
-
- LOG.info("Redirecting log files based on TaskAttemptId: " + tezTaskAttemptID);
-
- Appender appender = Logger.getRootLogger().getAppender(
- TezConfiguration.TEZ_CONTAINER_LOGGER_NAME);
- if (appender != null) {
- if (appender instanceof TezContainerLogAppender) {
- TezContainerLogAppender claAppender = (TezContainerLogAppender) appender;
- containerLogDir = claAppender.getContainerLogDir();
- claAppender.setLogFileName(constructLogFileName(
- TezConfiguration.TEZ_CONTAINER_LOG_FILE_NAME, tezTaskAttemptID));
- claAppender.activateOptions();
- } else {
- LOG.warn("Appender is a " + appender.getClass()
- + "; require an instance of "
- + TezContainerLogAppender.class.getName()
- + " to reconfigure the logger output");
- }
- } else {
- LOG.warn("Not configured with appender named: "
- + TezConfiguration.TEZ_CONTAINER_LOGGER_NAME
- + ". Cannot reconfigure logger output");
- }
-
- if (containerLogDir != null) {
- System.setOut(new PrintStream(new File(containerLogDir,
- constructLogFileName(TezConfiguration.TEZ_CONTAINER_OUT_FILE_NAME,
- tezTaskAttemptID))));
- System.setErr(new PrintStream(new File(containerLogDir,
- constructLogFileName(TezConfiguration.TEZ_CONTAINER_ERR_FILE_NAME,
- tezTaskAttemptID))));
+ String addend = "";
+ if (tezTaskAttemptID != null) {
+ addend = tezTaskAttemptID.toString();
}
+ TezUtils.updateLoggers(addend);
}
- private static String constructLogFileName(String base,
- TezTaskAttemptID tezTaskAttemptID) {
- if (tezTaskAttemptID == null) {
- return base;
- } else {
- return base + "_" + tezTaskAttemptID.toString();
- }
- }
-
private static void processAdditionalResources(Map<String, TezLocalResource> additionalResources,
Configuration conf) throws IOException, TezException {
if (additionalResources == null || additionalResources.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d29107c/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 6098289..a481aaa 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -20,6 +20,7 @@ package org.apache.tez.dag.app;
import java.io.File;
import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
import java.security.PrivilegedExceptionAction;
@@ -400,6 +401,7 @@ public class DAGAppMaster extends AbstractService {
case INTERNAL_ERROR:
state = DAGAppMasterState.ERROR;
if(currentDAG != null) {
+ _updateLoggers(currentDAG, "_post");
// notify dag to finish which will send the DAG_FINISHED event
LOG.info("Internal Error. Notifying dags to finish.");
sendEvent(new DAGEvent(currentDAG.getID(), DAGEventType.INTERNAL_ERROR));
@@ -412,6 +414,7 @@ public class DAGAppMaster extends AbstractService {
DAGAppMasterEventDAGFinished finishEvt =
(DAGAppMasterEventDAGFinished) event;
if (!isSession) {
+ _updateLoggers(currentDAG, "_post");
setStateOnDAGCompletion();
LOG.info("Shutting down on completion of dag:" +
finishEvt.getDAGId().toString());
@@ -421,6 +424,7 @@ public class DAGAppMaster extends AbstractService {
+ finishEvt.getDAGId().toString()
+ ", dagState=" + finishEvt.getDAGState());
lastDAGCompletionTime = clock.getTime();
+ _updateLoggers(currentDAG, "_post");
switch(finishEvt.getDAGState()) {
case SUCCEEDED:
if (!currentDAG.getName().startsWith(
@@ -468,6 +472,14 @@ public class DAGAppMaster extends AbstractService {
}
}
+ private void _updateLoggers(DAG dag, String appender) {
+ try {
+ TezUtils.updateLoggers(dag.getID().toString() + appender);
+ } catch (FileNotFoundException e) {
+ LOG.warn("Unable to update the logger. Continue with the old logger", e );
+ }
+ }
+
private class DAGAppMasterEventHandler implements
EventHandler<DAGAppMasterEvent> {
@Override
@@ -1581,6 +1593,10 @@ public class DAGAppMaster extends AbstractService {
private void startDAG(DAGPlan dagPlan) {
long submitTime = this.clock.getTime();
this.state = DAGAppMasterState.RUNNING;
+ this.appName = dagPlan.getName();
+ // /////////////////// Create the job itself.
+ DAG newDAG = createDAG(dagPlan);
+ _updateLoggers(newDAG, "");
if (LOG.isDebugEnabled()) {
LOG.debug("Running a DAG with " + dagPlan.getVertexCount()
+ " vertices ");
@@ -1592,10 +1608,6 @@ public class DAGAppMaster extends AbstractService {
LOG.info("Running DAG: " + dagPlan.getName());
// Job name is the same as the app name until we support multiple dags
// for an app later
- appName = dagPlan.getName();
-
- // /////////////////// Create the job itself.
- DAG newDAG = createDAG(dagPlan);
DAGSubmittedEvent submittedEvent = new DAGSubmittedEvent(newDAG.getID(),
submitTime, dagPlan, this.appAttemptID);
historyEventHandler.handle(new DAGHistoryEvent(newDAG.getID(), submittedEvent));
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d29107c/tez-dag/src/main/resources/tez-container-log4j.properties
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/resources/tez-container-log4j.properties b/tez-dag/src/main/resources/tez-container-log4j.properties
index 2bbfad8..628d3b5 100644
--- a/tez-dag/src/main/resources/tez-container-log4j.properties
+++ b/tez-dag/src/main/resources/tez-container-log4j.properties
@@ -24,7 +24,7 @@ log4j.threshold=ALL
# ContainerLog Appender
#
-log4j.appender.CLA=org.apache.hadoop.mapred.TezContainerLogAppender
+log4j.appender.CLA=org.apache.tez.common.TezContainerLogAppender
log4j.appender.CLA.containerLogDir=${yarn.app.container.log.dir}
log4j.appender.CLA.layout=org.apache.log4j.PatternLayout