You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2013/04/24 22:51:54 UTC
svn commit: r1471669 - in /incubator/tez/branches/TEZ-1:
tez-dag-api/src/main/java/org/apache/tez/dag/api/
tez-dag/src/main/java/org/apache/hadoop/mapred/
tez-dag/src/main/java/org/apache/tez/dag/app/
tez-dag/src/main/java/org/apache/tez/dag/app/rm/con...
Author: hitesh
Date: Wed Apr 24 20:51:53 2013
New Revision: 1471669
URL: http://svn.apache.org/r1471669
Log:
TEZ-77. Fix build break due to changes in YARN related to ApplicationConstants.Environment. (hitesh)
Modified:
incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
Modified: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java?rev=1471669&r1=1471668&r2=1471669&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java (original)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java Wed Apr 24 20:51:53 2013
@@ -91,9 +91,6 @@ public class TezConfiguration extends Co
TEZ_HOME_ENV + "/lib/*"
};
- public static final String APPLICATION_ATTEMPT_ID_ENV = "APPLICATION_ATTEMPT_ID_ENV";
-
-
public static final String DAG_AM_PLAN_CONFIG_XML = "tez-dag.xml";
}
Modified: incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java?rev=1471669&r1=1471668&r2=1471669&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java (original)
+++ incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java Wed Apr 24 20:51:53 2013
@@ -60,6 +60,7 @@ import org.apache.hadoop.util.StringUtil
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -105,9 +106,9 @@ public class YarnTezDagChild {
final InetSocketAddress address =
NetUtils.createSocketAddrForHost(host, port);
- final ApplicationId appID = ConverterUtils.toApplicationId(args[2]);
-
- final ContainerId containerId = ConverterUtils.toContainerId(args[3]);
+ final ContainerId containerId = ConverterUtils.toContainerId(args[2]);
+ final ApplicationId appID =
+ containerId.getApplicationAttemptId().getApplicationId();
// FIXME fix initialize metrics in child runner
DefaultMetricsSystem.initialize("VertexTask");
@@ -172,7 +173,8 @@ public class YarnTezDagChild {
taskAttemptId = taskContext.getTaskAttemptId();
final Task t = createAndConfigureTezTask(taskContext, umbilical,
- credentials, jt);
+ credentials, jt,
+ containerId.getApplicationAttemptId().getAttemptId());
task = (MRTask) t.getProcessor();
final JobConf job = task.getConf();
@@ -251,7 +253,7 @@ public class YarnTezDagChild {
*/
private static void configureLocalDirs(MRTask task, JobConf job) throws IOException {
String[] localSysDirs = StringUtils.getTrimmedStrings(
- System.getenv(ApplicationConstants.LOCAL_DIR_ENV));
+ System.getenv(Environment.LOCAL_DIRS.name()));
job.setStrings(TezJobConfig.LOCAL_DIR, localSysDirs);
LOG.info(TezJobConfig.LOCAL_DIR + " for child: " +
job.get(TezJobConfig.LOCAL_DIR));
@@ -289,15 +291,12 @@ public class YarnTezDagChild {
}
private static JobConf configureTask(MRTask task, Credentials credentials,
- Token<JobTokenIdentifier> jt) throws IOException, InterruptedException {
+ Token<JobTokenIdentifier> jt, int appAttemptId)
+ throws IOException, InterruptedException {
JobConf job = task.getConf();
- String appAttemptIdEnv = System
- .getenv(TezConfiguration.APPLICATION_ATTEMPT_ID_ENV);
- LOG.debug("APPLICATION_ATTEMPT_ID: " + appAttemptIdEnv);
// Set it in conf, so as to be able to be used the the OutputCommitter.
- job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, Integer
- .parseInt(appAttemptIdEnv));
+ job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, appAttemptId);
// set tcp nodelay
job.setBoolean("ipc.client.tcpnodelay", true);
@@ -406,8 +405,9 @@ public class YarnTezDagChild {
private static Task createAndConfigureTezTask(
TezEngineTaskContext taskContext,
- TezTaskUmbilicalProtocol master,
- Credentials credentials, Token<JobTokenIdentifier> jt)
+ TezTaskUmbilicalProtocol master,
+ Credentials credentials, Token<JobTokenIdentifier> jt,
+ int appAttemptId)
throws IOException, InterruptedException {
Configuration jConf = new JobConf(MRJobConfig.JOB_CONF_FILE);
Configuration conf;
@@ -466,7 +466,7 @@ public class YarnTezDagChild {
t.initialize(job, master);
MRTask task = (MRTask)t.getProcessor();
- configureTask(task, credentials, jt);
+ configureTask(task, credentials, jt, appAttemptId);
return t;
}
Modified: incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java?rev=1471669&r1=1471668&r2=1471669&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java (original)
+++ incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java Wed Apr 24 20:51:53 2013
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.SystemCloc
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -1239,20 +1240,14 @@ public class DAGAppMaster extends Compos
// TODO: Deprecated keys?
//DeprecatedKeys.init();
String containerIdStr =
- System.getenv(ApplicationConstants.AM_CONTAINER_ID_ENV);
- String nodeHostString = System.getenv(ApplicationConstants.NM_HOST_ENV);
- String nodePortString = System.getenv(ApplicationConstants.NM_PORT_ENV);
+ System.getenv(Environment.CONTAINER_ID.name());
+ String nodeHostString = System.getenv(Environment.NM_HOST.name());
+ String nodePortString = System.getenv(Environment.NM_PORT.name());
String nodeHttpPortString =
- System.getenv(ApplicationConstants.NM_HTTP_PORT_ENV);
+ System.getenv(Environment.NM_HTTP_PORT.name());
String appSubmitTimeStr =
System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV);
- validateInputParam(containerIdStr,
- ApplicationConstants.AM_CONTAINER_ID_ENV);
- validateInputParam(nodeHostString, ApplicationConstants.NM_HOST_ENV);
- validateInputParam(nodePortString, ApplicationConstants.NM_PORT_ENV);
- validateInputParam(nodeHttpPortString,
- ApplicationConstants.NM_HTTP_PORT_ENV);
validateInputParam(appSubmitTimeStr,
ApplicationConstants.APP_SUBMIT_TIME_ENV);
Modified: incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java?rev=1471669&r1=1471668&r2=1471669&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java (original)
+++ incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java Wed Apr 24 20:51:53 2013
@@ -175,7 +175,6 @@ public class AMContainerHelpers {
myEnv.putAll(env);
myEnv.putAll(vertexEnv);
// TODO TEZ-38 MRChildJVM2.setEnv should become a no-op
- TezEngineChildJVM.setVMEnv(myEnv, conf, vertexId, appContext);
// Set up the launch command
List<String> commands = TezEngineChildJVM.getVMCommand(
Modified: incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java?rev=1471669&r1=1471668&r2=1471669&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java (original)
+++ incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java Wed Apr 24 20:51:53 2013
@@ -20,10 +20,8 @@ package org.apache.tez.dag.utils;
import java.net.InetSocketAddress;
import java.util.List;
-import java.util.Map;
import java.util.Vector;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.YarnTezDagChild;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -32,7 +30,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.app.AppContext;
import org.apache.tez.engine.records.TezVertexID;
public class TezEngineChildJVM {
@@ -71,15 +68,6 @@ public class TezEngineChildJVM {
filter.toString();
}
- public static void setVMEnv(Map<String, String> environment, TezConfiguration conf,
- TezVertexID vertexId, AppContext appContext) {
-
- // FIXME this should be derivable from the container id set by the NM
- // and not require the AM to set
- environment.put(TezConfiguration.APPLICATION_ATTEMPT_ID_ENV,
- String.valueOf(appContext.getApplicationAttemptId().getAttemptId()));
- }
-
public static List<String> getVMCommand(
InetSocketAddress taskAttemptListenerAddr, TezConfiguration conf,
TezVertexID vertexId,
@@ -106,11 +94,10 @@ public class TezEngineChildJVM {
// Add main class and its arguments
vargs.add(YarnTezDagChild.class.getName()); // main of Child
+
// pass TaskAttemptListener's address
vargs.add(taskAttemptListenerAddr.getAddress().getHostAddress());
vargs.add(Integer.toString(taskAttemptListenerAddr.getPort()));
- // Set the job id
- vargs.add(jobID.toString());
// Finally add the containerId.
vargs.add(String.valueOf(containerId.toString()));
Modified: incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java?rev=1471669&r1=1471668&r2=1471669&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java (original)
+++ incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java Wed Apr 24 20:51:53 2013
@@ -549,8 +549,6 @@ public interface MRJobConfig {
public static final String STDOUT_LOGFILE_ENV = "STDOUT_LOGFILE_ENV";
public static final String STDERR_LOGFILE_ENV = "STDERR_LOGFILE_ENV";
- public static final String APPLICATION_ATTEMPT_ID_ENV = "APPLICATION_ATTEMPT_ID_ENV";
-
// This should be the directory where splits file gets localized on the node
// running ApplicationMaster.
public static final String JOB_SUBMIT_DIR = "jobSubmitDir";