You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 05:44:47 UTC
svn commit: r1077689 - in
/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred:
JvmManager.java TaskTracker.java
Author: omalley
Date: Fri Mar 4 04:44:46 2011
New Revision: 1077689
URL: http://svn.apache.org/viewvc?rev=1077689&view=rev
Log:
commit da204a8d0d5d96ed14f0390cf7a0d7db198201ae
Author: Devaraj Das <dd...@yahoo-inc.com>
Date: Fri Sep 17 14:59:30 2010 -0700
Makes the TT check the task logs size after the jvm exits, and if the task logs size is more than the configured limit, the TT wipes it out
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JvmManager.java
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JvmManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JvmManager.java?rev=1077689&r1=1077688&r2=1077689&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JvmManager.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JvmManager.java Fri Mar 4 04:44:46 2011
@@ -30,6 +30,9 @@ import java.util.Vector;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.CleanupQueue.PathDeletionContext;
import org.apache.hadoop.mapred.TaskTracker.TaskInProgress;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.server.tasktracker.JVMInfo;
@@ -177,6 +180,24 @@ class JvmManager {
workDir.substring(userDir.length())));
}
+
+ static void checkAndDeleteTaskLogs(TaskTracker tracker, Task firstTask) {
+ File logdir = TaskLog.getAttemptDir(firstTask.getTaskID(),
+ firstTask.isTaskCleanupTask());
+ //allow for 10% over the limit
+ if (FileUtil.getDU(logdir) >
+ (1.1 * tracker.getRetainSize(firstTask.getTaskID()))){
+ LOG.info("Deleting user log path since the amount of data in the logs" +
+ " exceeded the allowed " +
+ "log limits " + logdir);
+ String user = firstTask.getUser();
+ String jobLogDir = firstTask.getJobID().toString() + logdir.getName();
+ PathDeletionContext item =
+ new TaskController.DeletionContext(tracker.getTaskController(),
+ true, user, jobLogDir);
+ tracker.getCleanupThread().addToQueue(item);
+ }
+ }
static class JvmManagerForType {
//Mapping from the JVM IDs to running Tasks
@@ -479,6 +500,7 @@ class JvmManager {
LOG.info("JVM : " + jvmId + " exited with exit code " + exitCode
+ ". Number of tasks it ran: " + numTasksRan);
deleteWorkDir(tracker, firstTask);
+ checkAndDeleteTaskLogs(tracker, firstTask);
}
}
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=1077689&r1=1077688&r2=1077689&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java Fri Mar 4 04:44:46 2011
@@ -259,6 +259,8 @@ public class TaskTracker implements MRCo
private int maxMapSlots;
private int maxReduceSlots;
private int failures;
+ final long mapRetainSize;
+ final long reduceRetainSize;
private ACLsManager aclsManager;
@@ -1204,6 +1206,8 @@ public class TaskTracker implements MRCo
TaskTracker() {
server = null;
workerThreads = 0;
+ mapRetainSize = TaskLogsTruncater.DEFAULT_RETAIN_SIZE;
+ reduceRetainSize = TaskLogsTruncater.DEFAULT_RETAIN_SIZE;
}
void setConf(JobConf conf) {
@@ -1262,6 +1266,10 @@ public class TaskTracker implements MRCo
server.start();
this.httpPort = server.getPort();
checkJettyPort(httpPort);
+ mapRetainSize = conf.getLong(TaskLogsTruncater.MAP_USERLOG_RETAIN_SIZE,
+ TaskLogsTruncater.DEFAULT_RETAIN_SIZE);
+ reduceRetainSize = conf.getLong(TaskLogsTruncater.REDUCE_USERLOG_RETAIN_SIZE,
+ TaskLogsTruncater.DEFAULT_RETAIN_SIZE);
}
private void checkJettyPort(int port) throws IOException {
@@ -1633,6 +1641,10 @@ public class TaskTracker implements MRCo
return totalMemoryAllottedForTasks;
}
+ long getRetainSize(org.apache.hadoop.mapreduce.TaskAttemptID tid) {
+ return tid.isMap() ? mapRetainSize : reduceRetainSize;
+ }
+
/**
* Check if the jobtracker directed a 'reset' of the tasktracker.
*