You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2013/07/25 00:17:39 UTC

git commit: TEZ-317. Refactor AM-Task identifier transfer code path (bikas)

Updated Branches:
  refs/heads/master e563d0e54 -> adf5972be


TEZ-317. Refactor AM-Task identifier transfer code path (bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/adf5972b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/adf5972b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/adf5972b

Branch: refs/heads/master
Commit: adf5972be7494bfeb7d0b75db73c01e9ff6f8d00
Parents: e563d0e
Author: Bikas Saha <bi...@apache.org>
Authored: Wed Jul 24 15:16:53 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Wed Jul 24 15:16:53 2013 -0700

----------------------------------------------------------------------
 .../org/apache/tez/common/ContainerContext.java | 22 ++++++-----------
 .../apache/hadoop/mapred/YarnTezDagChild.java   | 26 ++++++++++----------
 .../dag/app/TaskAttemptListenerImpTezDag.java   |  6 +++--
 .../apache/tez/dag/app/dag/impl/DAGImpl.java    | 10 --------
 .../tez/dag/app/dag/impl/TaskAttemptImpl.java   |  1 -
 .../app/rm/container/AMContainerHelpers.java    |  6 +++--
 .../apache/tez/dag/utils/TezEngineChildJVM.java | 14 +++++------
 .../dag/app/rm/container/TestAMContainer.java   |  3 +++
 8 files changed, 39 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/adf5972b/tez-common/src/main/java/org/apache/tez/common/ContainerContext.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/ContainerContext.java b/tez-common/src/main/java/org/apache/tez/common/ContainerContext.java
index 91fdf88..df92bdc 100644
--- a/tez-common/src/main/java/org/apache/tez/common/ContainerContext.java
+++ b/tez-common/src/main/java/org/apache/tez/common/ContainerContext.java
@@ -24,29 +24,26 @@ import java.io.IOException;
 
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.tez.records.TezContainerId;
 
 // TODO EVENTUALLY move this over to PB. Fix package/module.
 // TODO EVENTUALLY unit tests for functionality.
 public class ContainerContext implements Writable {
 
-  ContainerId containerId;
+  String containerIdentifier;
   String pid;
 
   public ContainerContext() {
-    containerId = Records.newRecord(ContainerId.class);
+    containerIdentifier = "";
     pid = "";
   }
 
-  public ContainerContext(ContainerId containerId, String pid) {
-    this.containerId = containerId;
+  public ContainerContext(String containerIdStr, String pid) {
+    this.containerIdentifier = containerIdStr;
     this.pid = pid;
   }
 
-  public ContainerId getContainerId() {
-    return containerId;
+  public String getContainerIdentifier() {
+    return containerIdentifier;
   }
 
   public String getPid() {
@@ -55,16 +52,13 @@ public class ContainerContext implements Writable {
 
   @Override
   public void readFields(DataInput in) throws IOException {
-    TezContainerId tezContainerId = new TezContainerId();
-    tezContainerId.readFields(in);
-    this.containerId = tezContainerId.getContainerId();
+    this.containerIdentifier = Text.readString(in);
     this.pid = Text.readString(in);
   }
 
   @Override
   public void write(DataOutput out) throws IOException {
-    TezContainerId tezContainerId = new TezContainerId(containerId);
-    tezContainerId.write(out);
+    Text.writeString(out, containerIdentifier);
     Text.writeString(out, pid);
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/adf5972b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
index 00cfb4d..f1a2c18 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
@@ -43,9 +43,6 @@ import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.log4j.LogManager;
 import org.apache.tez.common.ContainerContext;
 import org.apache.tez.common.ContainerTask;
@@ -85,15 +82,19 @@ public class YarnTezDagChild {
     UserGroupInformation.setConfiguration(defaultConf);
     Limits.setConfiguration(defaultConf);
 
+    assert args.length == 5;
     String host = args[0];
     int port = Integer.parseInt(args[1]);
     final InetSocketAddress address =
         NetUtils.createSocketAddrForHost(host, port);
-
-    final ContainerId containerId = ConverterUtils.toContainerId(args[2]);
-    final ApplicationId appID =
-        containerId.getApplicationAttemptId().getApplicationId();
-
+    final String containerIdentifier = args[2];
+    final String tokenIdentifier = args[3];
+    final int attemptNumber = Integer.parseInt(args[4]);
+    if (LOG.isDebugEnabled()) {
+      LOG.info("Info from cmd line: AM-host: " + host + " AM-port: " + port
+          + " containerIdentifier: " + containerIdentifier + " attemptNumber: "
+          + attemptNumber + " tokenIdentifier: " + tokenIdentifier);
+    }
     // FIXME fix initialize metrics in child runner
     DefaultMetricsSystem.initialize("VertexTask");
 
@@ -110,7 +111,7 @@ public class YarnTezDagChild {
 
     // Create TaskUmbilicalProtocol as actual task owner.
     UserGroupInformation taskOwner =
-      UserGroupInformation.createRemoteUser(appID.toString());
+      UserGroupInformation.createRemoteUser(tokenIdentifier);
 
     Token<JobTokenIdentifier> jobToken = TokenCache.getJobToken(credentials);
     SecurityUtil.setTokenService(jobToken, address);
@@ -127,13 +128,13 @@ public class YarnTezDagChild {
     // report non-pid to application master
     String pid = System.getenv().get("JVM_PID");
     if (LOG.isDebugEnabled()) {
-      LOG.debug("PID, containerId: " + pid + ", " + containerId);
+      LOG.debug("PID, containerId: " + pid + ", " + containerIdentifier);
     }
     TezEngineTaskContext taskContext = null;
     ContainerTask containerTask = null;
     UserGroupInformation childUGI = null;
     TezTaskAttemptID taskAttemptId = null;
-    ContainerContext containerContext = new ContainerContext(containerId, pid);
+    ContainerContext containerContext = new ContainerContext(containerIdentifier, pid);
     int getTaskMaxSleepTime = defaultConf.getInt(
         TezConfiguration.TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX,
         TezConfiguration.TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX_DEFAULT);
@@ -165,8 +166,7 @@ public class YarnTezDagChild {
         taskAttemptId = taskContext.getTaskAttemptId();
 
         final Task t = createAndConfigureTezTask(taskContext, umbilical,
-            credentials, jobToken,
-            containerId.getApplicationAttemptId().getAttemptId());
+            credentials, jobToken, attemptNumber);
 
         final Configuration conf = ((RuntimeTask)t).getConfiguration();
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/adf5972b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index 9a33ef6..244d7d5 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.security.authorize.PolicyProvider;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.common.ContainerContext;
 import org.apache.tez.common.ContainerTask;
 import org.apache.tez.common.TezTaskStatus;
@@ -194,11 +195,12 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
 
     ContainerTask task = null;
 
-    if (containerContext == null || containerContext.getContainerId() == null) {
+    if (containerContext == null || containerContext.getContainerIdentifier() == null) {
       LOG.info("Invalid task request with an empty containerContext or containerId");
       task = TASK_FOR_INVALID_JVM;
     } else {
-      ContainerId containerId = containerContext.getContainerId();
+      ContainerId containerId = ConverterUtils.toContainerId(containerContext
+          .getContainerIdentifier());
       if (LOG.isDebugEnabled()) {
         LOG.debug("Container with id: " + containerId + " asked for a task");
       }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/adf5972b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index b02bc37..f261d9c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -33,7 +33,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -87,7 +86,6 @@ import org.apache.tez.dag.history.events.DAGFinishedEvent;
 import org.apache.tez.dag.history.events.DAGStartedEvent;
 import org.apache.tez.dag.records.TezDAGID;
 import org.apache.tez.dag.records.TezVertexID;
-import org.apache.tez.dag.utils.DAGApps;
 import org.apache.tez.dag.utils.TezBuilderUtils;
 import org.apache.tez.engine.common.security.JobTokenIdentifier;
 import org.apache.tez.engine.common.security.JobTokenSecretManager;
@@ -954,14 +952,6 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
       job.initTime = job.clock.getTime();
       String dagIdString = job.dagId.toString().replace("application", "job");
 
-      // TODO remove - TEZ-71
-      String user =
-        UserGroupInformation.getCurrentUser().getShortUserName();
-      Path path = DAGApps.getStagingAreaDir(job.conf, user);
-      if(LOG.isDebugEnabled()) {
-        LOG.debug("startJobs: parent=" + path + " child=" + dagIdString);
-      }
-
       // Prepare the TaskAttemptListener server for authentication of Containers
       // TaskAttemptListener gets the information via jobTokenSecretManager.
       JobTokenIdentifier identifier =

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/adf5972b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index e85be2a..ce388db 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -20,7 +20,6 @@ package org.apache.tez.dag.app.dag.impl;
 
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/adf5972b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index ac06ab3..470fee2 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -176,8 +176,10 @@ public class AMContainerHelpers {
 
     // Set up the launch command
     List<String> commands = TezEngineChildJVM.getVMCommand(
-        taskAttemptListener.getAddress(), conf, vertexId, containerId,
-        vertexId.getDAGId().getApplicationId(), shouldProfile, javaOpts);
+        taskAttemptListener.getAddress(), conf, containerId.toString(),
+        appContext.getApplicationID().toString(),
+        appContext.getApplicationAttemptId().getAttemptId(), 
+        shouldProfile, javaOpts);
 
     // Duplicate the ByteBuffers for access by multiple containers.
     Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/adf5972b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
index bc92ed8..a065218 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
@@ -26,11 +26,8 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.YarnTezDagChild;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.records.TezVertexID;
 
 public class TezEngineChildJVM {
 
@@ -70,8 +67,10 @@ public class TezEngineChildJVM {
   
   public static List<String> getVMCommand(
       InetSocketAddress taskAttemptListenerAddr, TezConfiguration conf, 
-      TezVertexID vertexId, 
-      ContainerId containerId, ApplicationId jobID, boolean shouldProfile,
+      String containerIdentifier,
+      String tokenIdentifier,
+      int applicationAttemptNumber,
+      boolean shouldProfile,
       String javaOpts) {
 
     Vector<String> vargs = new Vector<String>(9);
@@ -104,9 +103,10 @@ public class TezEngineChildJVM {
     // pass TaskAttemptListener's address
     vargs.add(taskAttemptListenerAddr.getAddress().getHostAddress()); 
     vargs.add(Integer.toString(taskAttemptListenerAddr.getPort()));
+    vargs.add(containerIdentifier);
+    vargs.add(tokenIdentifier);
+    vargs.add(Integer.toString(applicationAttemptNumber));
 
-    // Finally add the containerId.
-    vargs.add(String.valueOf(containerId.toString()));
     vargs.add("1>" + getTaskLogFile(LogName.STDOUT));
     vargs.add("2>" + getTaskLogFile(LogName.STDERR));
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/adf5972b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
index 332f737..616562c 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
@@ -29,6 +29,7 @@ import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.reset;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 import java.net.InetSocketAddress;
 import java.util.HashMap;
@@ -852,6 +853,8 @@ public class TestAMContainer {
       doReturn(new HashMap<ApplicationAccessType, String>()).when(appContext)
       .getApplicationACLs();
       doReturn(eventHandler).when(appContext).getEventHandler();
+      when(appContext.getApplicationAttemptId()).thenReturn(appAttemptID);
+      when(appContext.getApplicationID()).thenReturn(applicationID);
 
       dagID = new TezDAGID(applicationID, 1);
       vertexID = new TezVertexID(dagID, 1);