You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2013/07/25 00:17:39 UTC
git commit: TEZ-317. Refactor AM-Task identifier transfer code path
(bikas)
Updated Branches:
refs/heads/master e563d0e54 -> adf5972be
TEZ-317. Refactor AM-Task identifier transfer code path (bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/adf5972b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/adf5972b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/adf5972b
Branch: refs/heads/master
Commit: adf5972be7494bfeb7d0b75db73c01e9ff6f8d00
Parents: e563d0e
Author: Bikas Saha <bi...@apache.org>
Authored: Wed Jul 24 15:16:53 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Wed Jul 24 15:16:53 2013 -0700
----------------------------------------------------------------------
.../org/apache/tez/common/ContainerContext.java | 22 ++++++-----------
.../apache/hadoop/mapred/YarnTezDagChild.java | 26 ++++++++++----------
.../dag/app/TaskAttemptListenerImpTezDag.java | 6 +++--
.../apache/tez/dag/app/dag/impl/DAGImpl.java | 10 --------
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 1 -
.../app/rm/container/AMContainerHelpers.java | 6 +++--
.../apache/tez/dag/utils/TezEngineChildJVM.java | 14 +++++------
.../dag/app/rm/container/TestAMContainer.java | 3 +++
8 files changed, 39 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/adf5972b/tez-common/src/main/java/org/apache/tez/common/ContainerContext.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/ContainerContext.java b/tez-common/src/main/java/org/apache/tez/common/ContainerContext.java
index 91fdf88..df92bdc 100644
--- a/tez-common/src/main/java/org/apache/tez/common/ContainerContext.java
+++ b/tez-common/src/main/java/org/apache/tez/common/ContainerContext.java
@@ -24,29 +24,26 @@ import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.tez.records.TezContainerId;
// TODO EVENTUALLY move this over to PB. Fix package/module.
// TODO EVENTUALLY unit tests for functionality.
public class ContainerContext implements Writable {
- ContainerId containerId;
+ String containerIdentifier;
String pid;
public ContainerContext() {
- containerId = Records.newRecord(ContainerId.class);
+ containerIdentifier = "";
pid = "";
}
- public ContainerContext(ContainerId containerId, String pid) {
- this.containerId = containerId;
+ public ContainerContext(String containerIdStr, String pid) {
+ this.containerIdentifier = containerIdStr;
this.pid = pid;
}
- public ContainerId getContainerId() {
- return containerId;
+ public String getContainerIdentifier() {
+ return containerIdentifier;
}
public String getPid() {
@@ -55,16 +52,13 @@ public class ContainerContext implements Writable {
@Override
public void readFields(DataInput in) throws IOException {
- TezContainerId tezContainerId = new TezContainerId();
- tezContainerId.readFields(in);
- this.containerId = tezContainerId.getContainerId();
+ this.containerIdentifier = Text.readString(in);
this.pid = Text.readString(in);
}
@Override
public void write(DataOutput out) throws IOException {
- TezContainerId tezContainerId = new TezContainerId(containerId);
- tezContainerId.write(out);
+ Text.writeString(out, containerIdentifier);
Text.writeString(out, pid);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/adf5972b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
index 00cfb4d..f1a2c18 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
@@ -43,9 +43,6 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.log4j.LogManager;
import org.apache.tez.common.ContainerContext;
import org.apache.tez.common.ContainerTask;
@@ -85,15 +82,19 @@ public class YarnTezDagChild {
UserGroupInformation.setConfiguration(defaultConf);
Limits.setConfiguration(defaultConf);
+ assert args.length == 5;
String host = args[0];
int port = Integer.parseInt(args[1]);
final InetSocketAddress address =
NetUtils.createSocketAddrForHost(host, port);
-
- final ContainerId containerId = ConverterUtils.toContainerId(args[2]);
- final ApplicationId appID =
- containerId.getApplicationAttemptId().getApplicationId();
-
+ final String containerIdentifier = args[2];
+ final String tokenIdentifier = args[3];
+ final int attemptNumber = Integer.parseInt(args[4]);
+ if (LOG.isDebugEnabled()) {
+ LOG.info("Info from cmd line: AM-host: " + host + " AM-port: " + port
+ + " containerIdentifier: " + containerIdentifier + " attemptNumber: "
+ + attemptNumber + " tokenIdentifier: " + tokenIdentifier);
+ }
// FIXME fix initialize metrics in child runner
DefaultMetricsSystem.initialize("VertexTask");
@@ -110,7 +111,7 @@ public class YarnTezDagChild {
// Create TaskUmbilicalProtocol as actual task owner.
UserGroupInformation taskOwner =
- UserGroupInformation.createRemoteUser(appID.toString());
+ UserGroupInformation.createRemoteUser(tokenIdentifier);
Token<JobTokenIdentifier> jobToken = TokenCache.getJobToken(credentials);
SecurityUtil.setTokenService(jobToken, address);
@@ -127,13 +128,13 @@ public class YarnTezDagChild {
// report non-pid to application master
String pid = System.getenv().get("JVM_PID");
if (LOG.isDebugEnabled()) {
- LOG.debug("PID, containerId: " + pid + ", " + containerId);
+ LOG.debug("PID, containerId: " + pid + ", " + containerIdentifier);
}
TezEngineTaskContext taskContext = null;
ContainerTask containerTask = null;
UserGroupInformation childUGI = null;
TezTaskAttemptID taskAttemptId = null;
- ContainerContext containerContext = new ContainerContext(containerId, pid);
+ ContainerContext containerContext = new ContainerContext(containerIdentifier, pid);
int getTaskMaxSleepTime = defaultConf.getInt(
TezConfiguration.TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX,
TezConfiguration.TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX_DEFAULT);
@@ -165,8 +166,7 @@ public class YarnTezDagChild {
taskAttemptId = taskContext.getTaskAttemptId();
final Task t = createAndConfigureTezTask(taskContext, umbilical,
- credentials, jobToken,
- containerId.getApplicationAttemptId().getAttemptId());
+ credentials, jobToken, attemptNumber);
final Configuration conf = ((RuntimeTask)t).getConfiguration();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/adf5972b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index 9a33ef6..244d7d5 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.service.AbstractService;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.tez.common.ContainerContext;
import org.apache.tez.common.ContainerTask;
import org.apache.tez.common.TezTaskStatus;
@@ -194,11 +195,12 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
ContainerTask task = null;
- if (containerContext == null || containerContext.getContainerId() == null) {
+ if (containerContext == null || containerContext.getContainerIdentifier() == null) {
LOG.info("Invalid task request with an empty containerContext or containerId");
task = TASK_FOR_INVALID_JVM;
} else {
- ContainerId containerId = containerContext.getContainerId();
+ ContainerId containerId = ConverterUtils.toContainerId(containerContext
+ .getContainerIdentifier());
if (LOG.isDebugEnabled()) {
LOG.debug("Container with id: " + containerId + " asked for a task");
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/adf5972b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index b02bc37..f261d9c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -33,7 +33,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
@@ -87,7 +86,6 @@ import org.apache.tez.dag.history.events.DAGFinishedEvent;
import org.apache.tez.dag.history.events.DAGStartedEvent;
import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.dag.records.TezVertexID;
-import org.apache.tez.dag.utils.DAGApps;
import org.apache.tez.dag.utils.TezBuilderUtils;
import org.apache.tez.engine.common.security.JobTokenIdentifier;
import org.apache.tez.engine.common.security.JobTokenSecretManager;
@@ -954,14 +952,6 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
job.initTime = job.clock.getTime();
String dagIdString = job.dagId.toString().replace("application", "job");
- // TODO remove - TEZ-71
- String user =
- UserGroupInformation.getCurrentUser().getShortUserName();
- Path path = DAGApps.getStagingAreaDir(job.conf, user);
- if(LOG.isDebugEnabled()) {
- LOG.debug("startJobs: parent=" + path + " child=" + dagIdString);
- }
-
// Prepare the TaskAttemptListener server for authentication of Containers
// TaskAttemptListener gets the information via jobTokenSecretManager.
JobTokenIdentifier identifier =
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/adf5972b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index e85be2a..ce388db 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -20,7 +20,6 @@ package org.apache.tez.dag.app.dag.impl;
import java.net.InetSocketAddress;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/adf5972b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index ac06ab3..470fee2 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -176,8 +176,10 @@ public class AMContainerHelpers {
// Set up the launch command
List<String> commands = TezEngineChildJVM.getVMCommand(
- taskAttemptListener.getAddress(), conf, vertexId, containerId,
- vertexId.getDAGId().getApplicationId(), shouldProfile, javaOpts);
+ taskAttemptListener.getAddress(), conf, containerId.toString(),
+ appContext.getApplicationID().toString(),
+ appContext.getApplicationAttemptId().getAttemptId(),
+ shouldProfile, javaOpts);
// Duplicate the ByteBuffers for access by multiple containers.
Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/adf5972b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
index bc92ed8..a065218 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
@@ -26,11 +26,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.YarnTezDagChild;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.records.TezVertexID;
public class TezEngineChildJVM {
@@ -70,8 +67,10 @@ public class TezEngineChildJVM {
public static List<String> getVMCommand(
InetSocketAddress taskAttemptListenerAddr, TezConfiguration conf,
- TezVertexID vertexId,
- ContainerId containerId, ApplicationId jobID, boolean shouldProfile,
+ String containerIdentifier,
+ String tokenIdentifier,
+ int applicationAttemptNumber,
+ boolean shouldProfile,
String javaOpts) {
Vector<String> vargs = new Vector<String>(9);
@@ -104,9 +103,10 @@ public class TezEngineChildJVM {
// pass TaskAttemptListener's address
vargs.add(taskAttemptListenerAddr.getAddress().getHostAddress());
vargs.add(Integer.toString(taskAttemptListenerAddr.getPort()));
+ vargs.add(containerIdentifier);
+ vargs.add(tokenIdentifier);
+ vargs.add(Integer.toString(applicationAttemptNumber));
- // Finally add the containerId.
- vargs.add(String.valueOf(containerId.toString()));
vargs.add("1>" + getTaskLogFile(LogName.STDOUT));
vargs.add("2>" + getTaskLogFile(LogName.STDERR));
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/adf5972b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
index 332f737..616562c 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
@@ -29,6 +29,7 @@ import static org.mockito.Mockito.never;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
import java.net.InetSocketAddress;
import java.util.HashMap;
@@ -852,6 +853,8 @@ public class TestAMContainer {
doReturn(new HashMap<ApplicationAccessType, String>()).when(appContext)
.getApplicationACLs();
doReturn(eventHandler).when(appContext).getEventHandler();
+ when(appContext.getApplicationAttemptId()).thenReturn(appAttemptID);
+ when(appContext.getApplicationID()).thenReturn(applicationID);
dagID = new TezDAGID(applicationID, 1);
vertexID = new TezVertexID(dagID, 1);