You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by to...@apache.org on 2011/05/19 22:16:34 UTC
svn commit: r1125082 - in /hadoop/mapreduce/trunk: CHANGES.txt
src/java/org/apache/hadoop/mapred/TaskLogAppender.java
src/java/org/apache/hadoop/mapred/TaskRunner.java
Author: todd
Date: Thu May 19 20:16:33 2011
New Revision: 1125082
URL: http://svn.apache.org/viewvc?rev=1125082&view=rev
Log:
MAPREDUCE-2372. TaskLogAppender mechanism shouldn't be set up in log4j.properties. Contributed by Todd Lipcon.
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskLogAppender.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1125082&r1=1125081&r2=1125082&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Thu May 19 20:16:33 2011
@@ -116,6 +116,9 @@ Trunk (unreleased changes)
MAPREDUCE-2483. Remove duplication of jars between Hadoop subprojects
from build artifacts. (Eric Yang via omalley)
+ MAPREDUCE-2372. TaskLogAppender mechanism shouldn't be set up in
+ log4j.properties (todd)
+
OPTIMIZATIONS
MAPREDUCE-2026. Make JobTracker.getJobCounters() and
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskLogAppender.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskLogAppender.java?rev=1125082&r1=1125081&r2=1125082&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskLogAppender.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskLogAppender.java Thu May 19 20:16:33 2011
@@ -34,13 +34,20 @@ import org.apache.log4j.spi.LoggingEvent
public class TaskLogAppender extends FileAppender {
private String taskId; //taskId should be managed as String rather than TaskID object
//so that log4j can configure it from the configuration(log4j.properties).
- private int maxEvents;
+ private Integer maxEvents;
private Queue<LoggingEvent> tail = null;
- private boolean isCleanup;
+ private Boolean isCleanup;
+
+ // System properties passed in from JVM runner
+ static final String ISCLEANUP_PROPERTY = "hadoop.tasklog.iscleanup";
+ static final String LOGSIZE_PROPERTY = "hadoop.tasklog.totalLogFileSize";
+ static final String TASKID_PROPERTY = "hadoop.tasklog.taskid";
@Override
public void activateOptions() {
synchronized (this) {
+ setOptionsFromSystemProperties();
+
if (maxEvents > 0) {
tail = new LinkedList<LoggingEvent>();
}
@@ -50,6 +57,26 @@ public class TaskLogAppender extends Fil
super.activateOptions();
}
}
+
+ /**
+ * The Task Runner passes in the options as system properties. Set
+ * the options if the setters haven't already been called.
+ */
+ private synchronized void setOptionsFromSystemProperties() {
+ if (isCleanup == null) {
+ String propValue = System.getProperty(ISCLEANUP_PROPERTY, "false");
+ isCleanup = Boolean.valueOf(propValue);
+ }
+
+ if (taskId == null) {
+ taskId = System.getProperty(TASKID_PROPERTY);
+ }
+
+ if (maxEvents == null) {
+ String propValue = System.getProperty(LOGSIZE_PROPERTY, "0");
+ setTotalLogFileSize(Long.valueOf(propValue));
+ }
+ }
@Override
public void append(LoggingEvent event) {
@@ -83,21 +110,21 @@ public class TaskLogAppender extends Fil
* Getter/Setter methods for log4j.
*/
- public String getTaskId() {
+ public synchronized String getTaskId() {
return taskId;
}
- public void setTaskId(String taskId) {
+ public synchronized void setTaskId(String taskId) {
this.taskId = taskId;
}
private static final int EVENT_SIZE = 100;
- public long getTotalLogFileSize() {
+ public synchronized long getTotalLogFileSize() {
return maxEvents * EVENT_SIZE;
}
- public void setTotalLogFileSize(long logSize) {
+ public synchronized void setTotalLogFileSize(long logSize) {
maxEvents = (int) logSize / EVENT_SIZE;
}
@@ -107,7 +134,7 @@ public class TaskLogAppender extends Fil
* @param isCleanup
* true if the task is cleanup attempt, false otherwise.
*/
- public void setIsCleanup(boolean isCleanup) {
+ public synchronized void setIsCleanup(boolean isCleanup) {
this.isCleanup = isCleanup;
}
@@ -116,7 +143,7 @@ public class TaskLogAppender extends Fil
*
* @return true if the task is cleanup attempt, false otherwise.
*/
- public boolean getIsCleanup() {
+ public synchronized boolean getIsCleanup() {
return isCleanup;
}
}
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java?rev=1125082&r1=1125081&r2=1125082&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java Thu May 19 20:16:33 2011
@@ -460,9 +460,10 @@ abstract class TaskRunner extends Thread
vargs.add("-Dhadoop.log.dir=" +
new File(System.getProperty("hadoop.log.dir")).getAbsolutePath());
vargs.add("-Dhadoop.root.logger=" + getLogLevel(conf).toString() + ",TLA");
- vargs.add("-Dhadoop.tasklog.taskid=" + taskid);
- vargs.add("-Dhadoop.tasklog.iscleanup=" + t.isTaskCleanupTask());
- vargs.add("-Dhadoop.tasklog.totalLogFileSize=" + logSize);
+ vargs.add("-D" + TaskLogAppender.TASKID_PROPERTY + "=" + taskid);
+ vargs.add("-D" + TaskLogAppender.ISCLEANUP_PROPERTY +
+ "=" + t.isTaskCleanupTask());
+ vargs.add("-D" + TaskLogAppender.LOGSIZE_PROPERTY + "=" + logSize);
}
/**