svn commit: r1103657 - in /hadoop/mapreduce/branches/MR-279: CHANGES.txt mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java

[vi...@apache.org]

Author: vinodkv
Date: Mon May 16 09:24:04 2011
New Revision: 1103657

URL: http://svn.apache.org/viewvc?rev=1103657&view=rev
Log:
Launching bin/yarn and bin/mapred only *once* in AM for constructing classpaths to avoid multiple forks and huge vmem usage by AM. Contributed by Vinod Kumar Vavilapalli.

Modified:
    hadoop/mapreduce/branches/MR-279/CHANGES.txt
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java

Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1103657&r1=1103656&r2=1103657&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Mon May 16 09:24:04 2011
@@ -4,6 +4,9 @@ Trunk (unreleased changes)
 
   MAPREDUCE-279
 
+    Launching bin/yarn and bin/mapred only *once* in AM for constructing
+    classpaths to avoid multiple forks and huge vmem usage by AM. (vinodkv)
+
     Fix calculation of maximum capacity in ParentQueue to use its parent's 
     absolute-capacity rather than its own absolute-capacity. (acmurthy)
 

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1103657&r1=1103656&r2=1103657&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Mon May 16 09:24:04 2011
@@ -29,6 +29,7 @@ import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -143,6 +144,8 @@ public abstract class TaskAttemptImpl im
   private final Lock writeLock;
   private Collection<Token<? extends TokenIdentifier>> fsTokens;
   private Token<JobTokenIdentifier> jobToken;
+  private AtomicBoolean initialEnvFlag = new AtomicBoolean();
+  private Map<String, String> initialEnv = null;
 
   private long launchTime;
   private long finishTime;
@@ -451,6 +454,23 @@ public abstract class TaskAttemptImpl im
   }
 
   /**
+   * Lock this on initialEnv so that there is only one fork in the AM for
+   * getting the initial class-path. TODO: This should go away once we construct
+   * a parent CLC and use it for all the containers.
+   */
+  private Map<String, String> getInitialEnv() throws IOException {
+    synchronized (initialEnvFlag) {
+      if (initialEnvFlag.get()) {
+        return initialEnv;
+      }
+      initialEnv = new HashMap<String, String>();
+      MRApps.setInitialClasspath(initialEnv);
+      initialEnvFlag.set(true);
+      return initialEnv;
+    }
+  }
+
+  /**
    * Create the {@link ContainerLaunchContext} for this attempt.
    */
   private ContainerLaunchContext createContainerLaunchContext() {
@@ -524,9 +544,7 @@ public abstract class TaskAttemptImpl im
               ByteBuffer.wrap(jobToken_dob.getData(), 0,
                   jobToken_dob.getLength()));
 
-      Map<String, String> env = new HashMap<String, String>();
-      MRApps.setInitialClasspath(env);
-      container.addAllEnv(env);
+      container.addAllEnv(getInitialEnv());
     } catch (IOException e) {
       throw new YarnException(e);
     }