svn commit: r1127760 - in /hadoop/mapreduce/branches/MR-279: CHANGES.txt yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java

[cd...@apache.org]

Author: cdouglas
Date: Thu May 26 02:30:14 2011
New Revision: 1127760

URL: http://svn.apache.org/viewvc?rev=1127760&view=rev
Log:
Add debug config for delaying delete of local files

Modified:
    hadoop/mapreduce/branches/MR-279/CHANGES.txt
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java

Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1127760&r1=1127759&r2=1127760&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Thu May 26 02:30:14 2011
@@ -4,6 +4,8 @@ Trunk (unreleased changes)
 
   MAPREDUCE-279
 
+    Add debug config for delaying delete of local files. (cdouglas)
+
     Fixing race condition leader to hung jobs in scheduler negotiator (mahadev)
 
     Add debug statements for AM not launching (mahadev)

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java?rev=1127760&r1=1127759&r2=1127760&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java Thu May 26 02:30:14 2011
@@ -20,7 +20,8 @@ package org.apache.hadoop.yarn.server.no
 
 import java.io.IOException;
 import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import static java.util.concurrent.TimeUnit.*;
 
 import org.apache.hadoop.fs.Path;
@@ -38,11 +39,13 @@ import org.apache.commons.logging.LogFac
 public class DeletionService extends AbstractService {
 
   static final Log LOG = LogFactory.getLog(DeletionService.class);
+  /** Delay before deleting resource to ease debugging of NM issues */
+  static final String DEBUG_DELAY_SEC =
+    NMConfig.NM_PREFIX + "debug.delete.delay";
 
-  private final ThreadPoolExecutor sched =
-    new ThreadPoolExecutor(1, DEFAULT_MAX_DELETE_THREADS, 60L, SECONDS,
-        new LinkedBlockingQueue<Runnable>());
+  private int debugDelay;
   private final ContainerExecutor exec;
+  private final ScheduledThreadPoolExecutor sched;
   private final FileContext lfs = getLfs();
   static final FileContext getLfs() {
     try {
@@ -53,8 +56,17 @@ public class DeletionService extends Abs
   }
 
   public DeletionService(ContainerExecutor exec) {
+    this(exec, new ScheduledThreadPoolExecutor(1));
+    sched.setMaximumPoolSize(DEFAULT_MAX_DELETE_THREADS);
+    sched.setKeepAliveTime(60L, SECONDS);
+  }
+
+  public DeletionService(ContainerExecutor exec,
+      ScheduledThreadPoolExecutor sched) {
     super(DeletionService.class.getName());
     this.exec = exec;
+    this.sched = sched;
+    this.debugDelay = 0;
   }
 
   /**
@@ -64,7 +76,8 @@ public class DeletionService extends Abs
    */
   public void delete(String user, Path subDir, Path... baseDirs) {
     // TODO if parent owned by NM, rename within parent inline
-    sched.submit(new FileDeletion(user, subDir, baseDirs));
+    sched.schedule(new FileDeletion(user, subDir, baseDirs),
+        debugDelay, TimeUnit.SECONDS);
   }
 
   @Override
@@ -72,6 +85,7 @@ public class DeletionService extends Abs
     if (conf != null) {
       sched.setMaximumPoolSize(
           conf.getInt(NM_MAX_DELETE_THREADS, DEFAULT_MAX_DELETE_THREADS));
+      debugDelay = conf.getInt(DEBUG_DELAY_SEC, 0);
     }
     super.init(conf);
   }