You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by dd...@apache.org on 2011/05/17 18:48:12 UTC
svn commit: r1104398 - in /hadoop/mapreduce/branches/MR-279: ./ mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ yarn/ya...

Author: ddas
Date: Tue May 17 16:48:12 2011
New Revision: 1104398

URL: http://svn.apache.org/viewvc?rev=1104398&view=rev
Log:
Fixing a bug to do with setting the staging dir. Contributed by Devaraj Das.

Modified:
    hadoop/mapreduce/branches/MR-279/CHANGES.txt
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/util/SecurityUtil.java

Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1104398&r1=1104397&r2=1104398&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Tue May 17 16:48:12 2011
@@ -4,6 +4,8 @@ Trunk (unreleased changes)
 
   MAPREDUCE-279
 
+    Fixing a bug to do with setting the staging dir. (ddas)
+
     Fixing a bug in previous patch (r1103657). Now bin/yarn truly shouldn't be
     launched multiple times in a single AM. (vinodkv)
 

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1104398&r1=1104397&r2=1104398&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Tue May 17 16:48:12 2011
@@ -106,6 +106,7 @@ import org.apache.hadoop.yarn.state.Mult
 import org.apache.hadoop.yarn.state.SingleArcTransition;
 import org.apache.hadoop.yarn.state.StateMachine;
 import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.SecurityUtil;
 
 /** Implementation of Job interface. Maintains the state machines of Job.
  * The read and write calls use ReadWriteLock for concurrency.
@@ -863,17 +864,18 @@ public class JobImpl implements org.apac
     protected void setup(JobImpl job) throws IOException {
 
       String oldJobIDString = job.oldJobId.toString();
+      String user = 
+        UserGroupInformation.getCurrentUser().getShortUserName();
+      Path path = SecurityUtil.getStagingAreaDir(job.conf, user);
       LOG.info("DEBUG --- startJobs:"
           + " parent="
-          + job.conf.get(YARNApplicationConstants.APPS_STAGING_DIR_KEY,
-              "/tmp/hadoop-yarn/${user.name}/staging") + " child="
+          + path + " child="
           + oldJobIDString);
 
       job.remoteJobSubmitDir =
           FileSystem.get(job.conf).makeQualified(
               new Path(
-                  job.conf.get(YARNApplicationConstants.APPS_STAGING_DIR_KEY),
-                  oldJobIDString));
+                  path, oldJobIDString));
       job.remoteJobConfFile =
           new Path(job.remoteJobSubmitDir,
               YARNApplicationConstants.JOB_CONF_FILE);

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1104398&r1=1104397&r2=1104398&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Tue May 17 16:48:12 2011
@@ -109,6 +109,7 @@ import org.apache.hadoop.yarn.state.Stat
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.hadoop.yarn.util.ContainerBuilderHelper;
+import org.apache.hadoop.yarn.util.SecurityUtil;
 import org.apache.hadoop.mapreduce.v2.api.records.Counters;
 import org.apache.hadoop.mapreduce.v2.api.records.Phase;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
@@ -494,9 +495,10 @@ public abstract class TaskAttemptImpl im
       // //////////// End of JobJar setup
 
       // //////////// Set up JobConf to be localized properly on the remote NM.
+      Path path = SecurityUtil.getStagingAreaDir(conf, 
+          UserGroupInformation.getCurrentUser().getShortUserName());
       Path remoteJobSubmitDir =
-          new Path(conf.get(YARNApplicationConstants.APPS_STAGING_DIR_KEY),
-              oldJobId.toString());
+          new Path(path, oldJobId.toString());
       Path remoteJobConfPath =
           new Path(remoteJobSubmitDir, YARNApplicationConstants.JOB_CONF_FILE);
       container.setLocalResource(

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java?rev=1104398&r1=1104397&r2=1104398&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java Tue May 17 16:48:12 2011
@@ -42,6 +42,7 @@ import org.apache.hadoop.mapreduce.TypeC
 import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.SecurityInfo;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.yarn.api.ClientRMProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest;
@@ -69,6 +70,7 @@ import org.apache.hadoop.yarn.factories.
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo;
+import org.apache.hadoop.yarn.util.SecurityUtil;
 
 
 // TODO: This should be part of something like yarn-client.
@@ -242,7 +244,9 @@ public class ResourceMgrDelegate {
 
   public String getStagingAreaDir() throws IOException, InterruptedException {
 //    Path path = new Path(MRJobConstants.JOB_SUBMIT_DIR);
-    Path path = new Path(conf.get(YARNApplicationConstants.APPS_STAGING_DIR_KEY));
+    String user = 
+      UserGroupInformation.getCurrentUser().getShortUserName();
+    Path path = SecurityUtil.getStagingAreaDir(conf, user);
     LOG.info("DEBUG --- getStagingAreaDir: dir=" + path);
     return path.toString();
   }
@@ -251,7 +255,7 @@ public class ResourceMgrDelegate {
   public String getSystemDir() throws IOException, InterruptedException {
     Path sysDir = new Path(
         YARNApplicationConstants.JOB_SUBMIT_DIR);
-    FileContext.getFileContext(conf).delete(sysDir, true);
+    //FileContext.getFileContext(conf).delete(sysDir, true);
     return sysDir.toString();
   }
   

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/util/SecurityUtil.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/util/SecurityUtil.java?rev=1104398&r1=1104397&r2=1104398&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/util/SecurityUtil.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/util/SecurityUtil.java Tue May 17 16:48:12 2011
@@ -34,6 +34,7 @@ import org.apache.hadoop.security.UserGr
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YARNApplicationConstants;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier;
 import org.apache.hadoop.yarn.security.ApplicationTokenSecretManager;
@@ -116,4 +117,11 @@ public class SecurityUtil {
           + " with identifer " + t.getIdentifier());
     }
   }
+  
+  private static final String STAGING_CONSTANT = ".staging";
+  public static Path getStagingAreaDir(Configuration conf, String user) {
+    return new Path(
+        conf.get(YARNApplicationConstants.APPS_STAGING_DIR_KEY) + 
+        Path.SEPARATOR + user + Path.SEPARATOR + STAGING_CONSTANT);
+  }
 }