You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sz...@apache.org on 2013/01/14 04:44:52 UTC
svn commit: r1432796 [4/5] - in
/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project: ./ conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/mai...
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/service/MRClientProtocolPBServiceImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/service/MRClientProtocolPBServiceImpl.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/service/MRClientProtocolPBServiceImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/service/MRClientProtocolPBServiceImpl.java Mon Jan 14 03:44:35 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapreduce.v2.a
import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
import org.apache.hadoop.mapreduce.v2.api.MRClientProtocolPB;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest;
@@ -43,6 +44,9 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.CancelDelegationTokenResponsePBImpl;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.FailTaskAttemptRequestPBImpl;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.FailTaskAttemptResponsePBImpl;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.GetCountersRequestPBImpl;
@@ -67,6 +71,10 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.KillTaskAttemptResponsePBImpl;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.KillTaskRequestPBImpl;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.KillTaskResponsePBImpl;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.RenewDelegationTokenRequestPBImpl;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.RenewDelegationTokenResponsePBImpl;
+import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenRequestProto;
+import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenResponseProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.FailTaskAttemptRequestProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.FailTaskAttemptResponseProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetCountersRequestProto;
@@ -91,6 +99,8 @@ import org.apache.hadoop.mapreduce.v2.pr
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskAttemptResponseProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskRequestProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskResponseProto;
+import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenRequestProto;
+import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenResponseProto;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import com.google.protobuf.RpcController;
@@ -252,4 +262,32 @@ public class MRClientProtocolPBServiceIm
}
}
+ @Override
+ public RenewDelegationTokenResponseProto renewDelegationToken(
+ RpcController controller, RenewDelegationTokenRequestProto proto)
+ throws ServiceException {
+ RenewDelegationTokenRequestPBImpl request =
+ new RenewDelegationTokenRequestPBImpl(proto);
+ try {
+ RenewDelegationTokenResponse response = real.renewDelegationToken(request);
+ return ((RenewDelegationTokenResponsePBImpl)response).getProto();
+ } catch (YarnRemoteException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public CancelDelegationTokenResponseProto cancelDelegationToken(
+ RpcController controller, CancelDelegationTokenRequestProto proto)
+ throws ServiceException {
+ CancelDelegationTokenRequestPBImpl request =
+ new CancelDelegationTokenRequestPBImpl(proto);
+ try {
+ CancelDelegationTokenResponse response = real.cancelDelegationToken(request);
+ return ((CancelDelegationTokenResponsePBImpl)response).getProto();
+ } catch (YarnRemoteException e) {
+ throw new ServiceException(e);
+ }
+ }
+
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java Mon Jan 14 03:44:35 2013
@@ -20,10 +20,10 @@ package org.apache.hadoop.mapreduce.v2.a
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenResponse;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDelegationTokenResponseProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDelegationTokenResponseProtoOrBuilder;
+import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.yarn.api.records.DelegationToken;
import org.apache.hadoop.yarn.api.records.ProtoBase;
import org.apache.hadoop.yarn.api.records.impl.pb.DelegationTokenPBImpl;
-import org.apache.hadoop.yarn.proto.YarnProtos.DelegationTokenProto;
public class GetDelegationTokenResponsePBImpl extends
@@ -99,11 +99,11 @@ public class GetDelegationTokenResponseP
}
- private DelegationTokenPBImpl convertFromProtoFormat(DelegationTokenProto p) {
+ private DelegationTokenPBImpl convertFromProtoFormat(TokenProto p) {
return new DelegationTokenPBImpl(p);
}
- private DelegationTokenProto convertToProtoFormat(DelegationToken t) {
+ private TokenProto convertToProtoFormat(DelegationToken t) {
return ((DelegationTokenPBImpl)t).getProto();
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java Mon Jan 14 03:44:35 2013
@@ -182,14 +182,16 @@ public class JobHistoryUtils {
/**
* Gets the configured directory prefix for In Progress history files.
- * @param conf
+ * @param conf the configuration for hte job
+ * @param jobId the id of the job the history file is for.
* @return A string representation of the prefix.
*/
public static String
- getConfiguredHistoryStagingDirPrefix(Configuration conf)
+ getConfiguredHistoryStagingDirPrefix(Configuration conf, String jobId)
throws IOException {
String user = UserGroupInformation.getCurrentUser().getShortUserName();
- Path path = MRApps.getStagingAreaDir(conf, user);
+ Path stagingPath = MRApps.getStagingAreaDir(conf, user);
+ Path path = new Path(stagingPath, jobId);
String logDir = path.toString();
return logDir;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java Mon Jan 14 03:44:35 2013
@@ -19,12 +19,18 @@
package org.apache.hadoop.mapreduce.v2.util;
import java.io.IOException;
+import java.net.MalformedURLException;
import java.net.URI;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -50,6 +56,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.ApplicationClassLoader;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -62,6 +69,8 @@ import com.google.common.base.Charsets;
@Private
@Unstable
public class MRApps extends Apps {
+ public static final Log LOG = LogFactory.getLog(MRApps.class);
+
public static String toString(JobId jid) {
return jid.toString();
}
@@ -157,38 +166,42 @@ public class MRApps extends Apps {
boolean userClassesTakesPrecedence =
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
+ String classpathEnvVar =
+ conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false)
+ ? Environment.APP_CLASSPATH.name() : Environment.CLASSPATH.name();
+
Apps.addToEnvironment(environment,
- Environment.CLASSPATH.name(),
+ classpathEnvVar,
Environment.PWD.$());
if (!userClassesTakesPrecedence) {
MRApps.setMRFrameworkClasspath(environment, conf);
}
Apps.addToEnvironment(
environment,
- Environment.CLASSPATH.name(),
+ classpathEnvVar,
MRJobConfig.JOB_JAR + Path.SEPARATOR + MRJobConfig.JOB_JAR);
Apps.addToEnvironment(
environment,
- Environment.CLASSPATH.name(),
+ classpathEnvVar,
MRJobConfig.JOB_JAR + Path.SEPARATOR + "classes" + Path.SEPARATOR);
Apps.addToEnvironment(
environment,
- Environment.CLASSPATH.name(),
+ classpathEnvVar,
MRJobConfig.JOB_JAR + Path.SEPARATOR + "lib" + Path.SEPARATOR + "*");
Apps.addToEnvironment(
environment,
- Environment.CLASSPATH.name(),
+ classpathEnvVar,
Environment.PWD.$() + Path.SEPARATOR + "*");
// a * in the classpath will only find a .jar, so we need to filter out
// all .jars and add everything else
addToClasspathIfNotJar(DistributedCache.getFileClassPaths(conf),
DistributedCache.getCacheFiles(conf),
conf,
- environment);
+ environment, classpathEnvVar);
addToClasspathIfNotJar(DistributedCache.getArchiveClassPaths(conf),
DistributedCache.getCacheArchives(conf),
conf,
- environment);
+ environment, classpathEnvVar);
if (userClassesTakesPrecedence) {
MRApps.setMRFrameworkClasspath(environment, conf);
}
@@ -204,7 +217,8 @@ public class MRApps extends Apps {
*/
private static void addToClasspathIfNotJar(Path[] paths,
URI[] withLinks, Configuration conf,
- Map<String, String> environment) throws IOException {
+ Map<String, String> environment,
+ String classpathEnvVar) throws IOException {
if (paths != null) {
HashMap<Path, String> linkLookup = new HashMap<Path, String>();
if (withLinks != null) {
@@ -232,13 +246,64 @@ public class MRApps extends Apps {
if(!name.toLowerCase().endsWith(".jar")) {
Apps.addToEnvironment(
environment,
- Environment.CLASSPATH.name(),
+ classpathEnvVar,
Environment.PWD.$() + Path.SEPARATOR + name);
}
}
}
}
+ /**
+ * Sets a {@link ApplicationClassLoader} on the given configuration and as
+ * the context classloader, if
+ * {@link MRJobConfig#MAPREDUCE_JOB_CLASSLOADER} is set to true, and
+ * the APP_CLASSPATH environment variable is set.
+ * @param conf
+ * @throws IOException
+ */
+ public static void setJobClassLoader(Configuration conf)
+ throws IOException {
+ if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false)) {
+ String appClasspath = System.getenv(Environment.APP_CLASSPATH.key());
+ if (appClasspath == null) {
+ LOG.warn("Not using job classloader since APP_CLASSPATH is not set.");
+ } else {
+ LOG.info("Using job classloader");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("APP_CLASSPATH=" + appClasspath);
+ }
+ String[] systemClasses = conf.getStrings(
+ MRJobConfig.MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES);
+ ClassLoader jobClassLoader = createJobClassLoader(appClasspath,
+ systemClasses);
+ if (jobClassLoader != null) {
+ conf.setClassLoader(jobClassLoader);
+ Thread.currentThread().setContextClassLoader(jobClassLoader);
+ }
+ }
+ }
+ }
+
+ private static ClassLoader createJobClassLoader(final String appClasspath,
+ final String[] systemClasses) throws IOException {
+ try {
+ return AccessController.doPrivileged(
+ new PrivilegedExceptionAction<ClassLoader>() {
+ @Override
+ public ClassLoader run() throws MalformedURLException {
+ return new ApplicationClassLoader(appClasspath,
+ MRApps.class.getClassLoader(), Arrays.asList(systemClasses));
+ }
+ });
+ } catch (PrivilegedActionException e) {
+ Throwable t = e.getCause();
+ if (t instanceof MalformedURLException) {
+ throw (MalformedURLException) t;
+ }
+ throw new IOException(e);
+ }
+ }
+
private static final String STAGING_CONSTANT = ".staging";
public static Path getStagingAreaDir(Configuration conf, String user) {
return new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR,
@@ -253,7 +318,26 @@ public class MRApps extends Apps {
return jobFile.toString();
}
-
+ public static Path getEndJobCommitSuccessFile(Configuration conf, String user,
+ JobId jobId) {
+ Path endCommitFile = new Path(MRApps.getStagingAreaDir(conf, user),
+ jobId.toString() + Path.SEPARATOR + "COMMIT_SUCCESS");
+ return endCommitFile;
+ }
+
+ public static Path getEndJobCommitFailureFile(Configuration conf, String user,
+ JobId jobId) {
+ Path endCommitFile = new Path(MRApps.getStagingAreaDir(conf, user),
+ jobId.toString() + Path.SEPARATOR + "COMMIT_FAIL");
+ return endCommitFile;
+ }
+
+ public static Path getStartJobCommitFile(Configuration conf, String user,
+ JobId jobId) {
+ Path startCommitFile = new Path(MRApps.getStagingAreaDir(conf, user),
+ jobId.toString() + Path.SEPARATOR + "COMMIT_STARTED");
+ return startCommitFile;
+ }
private static long[] parseTimeStamps(String[] strs) {
if (null == strs) {
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto Mon Jan 14 03:44:35 2013
@@ -36,4 +36,6 @@ service MRClientProtocolService {
rpc killTask (KillTaskRequestProto) returns (KillTaskResponseProto);
rpc killTaskAttempt (KillTaskAttemptRequestProto) returns (KillTaskAttemptResponseProto);
rpc failTaskAttempt (FailTaskAttemptRequestProto) returns (FailTaskAttemptResponseProto);
+ rpc renewDelegationToken(RenewDelegationTokenRequestProto) returns (RenewDelegationTokenResponseProto);
+ rpc cancelDelegationToken(CancelDelegationTokenRequestProto) returns (CancelDelegationTokenResponseProto);
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto Mon Jan 14 03:44:35 2013
@@ -21,6 +21,7 @@ option java_outer_classname = "MRService
option java_generic_services = true;
option java_generate_equals_and_hash = true;
+import "Security.proto";
import "mr_protos.proto";
import "yarn_protos.proto";
@@ -81,7 +82,7 @@ message GetDelegationTokenRequestProto {
}
message GetDelegationTokenResponseProto {
- optional DelegationTokenProto m_r_delegation_token = 1;
+ optional hadoop.common.TokenProto m_r_delegation_token = 1;
}
message KillJobRequestProto {
@@ -107,3 +108,18 @@ message FailTaskAttemptRequestProto {
}
message FailTaskAttemptResponseProto {
}
+
+message RenewDelegationTokenRequestProto {
+ required hadoop.common.TokenProto delegation_token = 1;
+}
+
+message RenewDelegationTokenResponseProto {
+ required int64 next_expiry_ts = 1;
+}
+
+message CancelDelegationTokenRequestProto {
+ required hadoop.common.TokenProto delegation_token = 1;
+}
+
+message CancelDelegationTokenResponseProto {
+}
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/TestRPCFactories.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/TestRPCFactories.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/TestRPCFactories.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/TestRPCFactories.java Mon Jan 14 03:44:35 2013
@@ -26,6 +26,8 @@ import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest;
@@ -50,6 +52,8 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
@@ -198,6 +202,18 @@ public class TestRPCFactories {
public GetDelegationTokenResponse getDelegationToken(
GetDelegationTokenRequest request) throws YarnRemoteException {
return null;
+ }
+
+ @Override
+ public RenewDelegationTokenResponse renewDelegationToken(
+ RenewDelegationTokenRequest request) throws YarnRemoteException {
+ return null;
+ }
+
+ @Override
+ public CancelDelegationTokenResponse cancelDelegationToken(
+ CancelDelegationTokenRequest request) throws YarnRemoteException {
+ return null;
}
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java Mon Jan 14 03:44:35 2013
@@ -235,6 +235,22 @@ public class TestMRApps {
index, 0);
}
+ @Test public void testSetClasspathWithJobClassloader() throws IOException {
+ Configuration conf = new Configuration();
+ conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true);
+ Map<String, String> env = new HashMap<String, String>();
+ MRApps.setClasspath(env, conf);
+ String cp = env.get("CLASSPATH");
+ String appCp = env.get("APP_CLASSPATH");
+ assertSame("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is"
+ + " in the classpath!", cp.indexOf("jar:job"), -1);
+ assertSame("MAPREDUCE_JOB_CLASSLOADER true, but PWD is"
+ + " in the classpath!", cp.indexOf("PWD"), -1);
+ assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not"
+ + " in the app classpath!",
+ "$PWD:job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*", appCp);
+ }
+
@Test
public void testSetupDistributedCacheEmpty() throws IOException {
Configuration conf = new Configuration();
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java Mon Jan 14 03:44:35 2013
@@ -144,13 +144,9 @@ public class JobClient extends CLI {
* we have to add this hack.
*/
private boolean getDelegationTokenCalled = false;
- /* notes the renewer that will renew the delegation token */
- private String dtRenewer = null;
/* do we need a HS delegation token for this client */
static final String HS_DELEGATION_TOKEN_REQUIRED
= "mapreduce.history.server.delegationtoken.required";
- static final String HS_DELEGATION_TOKEN_RENEWER
- = "mapreduce.history.server.delegationtoken.renewer";
static{
ConfigUtil.loadResources();
@@ -496,36 +492,6 @@ public class JobClient extends CLI {
clientUgi = UserGroupInformation.getCurrentUser();
}
- @InterfaceAudience.Private
- public static class Renewer extends TokenRenewer {
-
- @Override
- public boolean handleKind(Text kind) {
- return DelegationTokenIdentifier.MAPREDUCE_DELEGATION_KIND.equals(kind);
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public long renew(Token<?> token, Configuration conf
- ) throws IOException, InterruptedException {
- return new Cluster(conf).
- renewDelegationToken((Token<DelegationTokenIdentifier>) token);
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void cancel(Token<?> token, Configuration conf
- ) throws IOException, InterruptedException {
- new Cluster(conf).
- cancelDelegationToken((Token<DelegationTokenIdentifier>) token);
- }
-
- @Override
- public boolean isManaged(Token<?> token) throws IOException {
- return true;
- }
- }
-
/**
* Build a job client, connect to the indicated job tracker.
*
@@ -606,8 +572,6 @@ public class JobClient extends CLI {
if (getDelegationTokenCalled) {
conf.setBoolean(HS_DELEGATION_TOKEN_REQUIRED, getDelegationTokenCalled);
getDelegationTokenCalled = false;
- conf.set(HS_DELEGATION_TOKEN_RENEWER, dtRenewer);
- dtRenewer = null;
}
Job job = clientUgi.doAs(new PrivilegedExceptionAction<Job> () {
@Override
@@ -1210,7 +1174,6 @@ public class JobClient extends CLI {
public Token<DelegationTokenIdentifier>
getDelegationToken(final Text renewer) throws IOException, InterruptedException {
getDelegationTokenCalled = true;
- dtRenewer = renewer.toString();
return clientUgi.doAs(new
PrivilegedExceptionAction<Token<DelegationTokenIdentifier>>() {
public Token<DelegationTokenIdentifier> run() throws IOException,
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputCommitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputCommitter.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputCommitter.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputCommitter.java Mon Jan 14 03:44:35 2013
@@ -52,6 +52,14 @@ import org.apache.hadoop.classification.
* Discard the task commit.
* </li>
* </ol>
+ * The methods in this class can be called from several different processes and
+ * from several different contexts. It is important to know which process and
+ * which context each is called from. Each method should be marked accordingly
+ * in its documentation. It is also important to note that not all methods are
+ * guaranteed to be called once and only once. If a method is not guaranteed to
+ * have this property the output committer needs to handle this appropriately.
+ * Also note it will only be in rare situations where they may be called
+ * multiple times for the same task.
*
* @see FileOutputCommitter
* @see JobContext
@@ -62,7 +70,9 @@ import org.apache.hadoop.classification.
public abstract class OutputCommitter
extends org.apache.hadoop.mapreduce.OutputCommitter {
/**
- * For the framework to setup the job output during initialization
+ * For the framework to setup the job output during initialization. This is
+ * called from the application master process for the entire job. This will be
+ * called multiple times, once per job attempt.
*
* @param jobContext Context of the job whose output is being written.
* @throws IOException if temporary output could not be created
@@ -70,7 +80,9 @@ public abstract class OutputCommitter
public abstract void setupJob(JobContext jobContext) throws IOException;
/**
- * For cleaning up the job's output after job completion
+ * For cleaning up the job's output after job completion. This is called
+ * from the application master process for the entire job. This may be called
+ * multiple times.
*
* @param jobContext Context of the job whose output is being written.
* @throws IOException
@@ -82,7 +94,10 @@ public abstract class OutputCommitter
/**
* For committing job's output after successful job completion. Note that this
- * is invoked for jobs with final runstate as SUCCESSFUL.
+ * is invoked for jobs with final runstate as SUCCESSFUL. This is called
+ * from the application master process for the entire job. This is guaranteed
+ * to only be called once. If it throws an exception the entire job will
+ * fail.
*
* @param jobContext Context of the job whose output is being written.
* @throws IOException
@@ -94,7 +109,8 @@ public abstract class OutputCommitter
/**
* For aborting an unsuccessful job's output. Note that this is invoked for
* jobs with final runstate as {@link JobStatus#FAILED} or
- * {@link JobStatus#KILLED}
+ * {@link JobStatus#KILLED}. This is called from the application
+ * master process for the entire job. This may be called multiple times.
*
* @param jobContext Context of the job whose output is being written.
* @param status final runstate of the job
@@ -106,7 +122,10 @@ public abstract class OutputCommitter
}
/**
- * Sets up output for the task.
+ * Sets up output for the task. This is called from each individual task's
+ * process that will output to HDFS, and it is called just for that task. This
+ * may be called multiple times for the same task, but for different task
+ * attempts.
*
* @param taskContext Context of the task whose output is being written.
* @throws IOException
@@ -115,7 +134,9 @@ public abstract class OutputCommitter
throws IOException;
/**
- * Check whether task needs a commit
+ * Check whether task needs a commit. This is called from each individual
+ * task's process that will output to HDFS, and it is called just for that
+ * task.
*
* @param taskContext
* @return true/false
@@ -125,9 +146,16 @@ public abstract class OutputCommitter
throws IOException;
/**
- * To promote the task's temporary output to final output location
- *
- * The task's output is moved to the job's output directory.
+ * To promote the task's temporary output to final output location.
+ * If {@link #needsTaskCommit(TaskAttemptContext)} returns true and this
+ * task is the task that the AM determines finished first, this method
+ * is called to commit an individual task's output. This is to mark
+ * that tasks output as complete, as {@link #commitJob(JobContext)} will
+ * also be called later on if the entire job finished successfully. This
+ * is called from a task's process. This may be called multiple times for the
+ * same task, but different task attempts. It should be very rare for this to
+ * be called multiple times and requires odd networking failures to make this
+ * happen. In the future the Hadoop framework may eliminate this race.
*
* @param taskContext Context of the task whose output is being written.
* @throws IOException if commit is not
@@ -136,7 +164,9 @@ public abstract class OutputCommitter
throws IOException;
/**
- * Discard the task output
+ * Discard the task output. This is called from a task's process to clean
+ * up a single task's output that can not yet been committed. This may be
+ * called multiple times for the same task, but for different task attempts.
*
* @param taskContext
* @throws IOException
@@ -160,7 +190,8 @@ public abstract class OutputCommitter
* The retry-count for the job will be passed via the
* {@link MRConstants#APPLICATION_ATTEMPT_ID} key in
* {@link TaskAttemptContext#getConfiguration()} for the
- * <code>OutputCommitter</code>.
+ * <code>OutputCommitter</code>. This is called from the application master
+ * process, but it is called individually for each task.
*
* If an exception is thrown the task will be attempted again.
*
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java Mon Jan 14 03:44:35 2013
@@ -33,14 +33,12 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
import org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider;
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.mapreduce.util.ConfigUtil;
import org.apache.hadoop.mapreduce.v2.LogParams;
-import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.Token;
@@ -401,12 +399,7 @@ public class Cluster {
public long renewDelegationToken(Token<DelegationTokenIdentifier> token
) throws InvalidToken, IOException,
InterruptedException {
- try {
- return client.renewDelegationToken(token);
- } catch (RemoteException re) {
- throw re.unwrapRemoteException(InvalidToken.class,
- AccessControlException.class);
- }
+ return token.renew(getConf());
}
/**
@@ -418,12 +411,7 @@ public class Cluster {
public void cancelDelegationToken(Token<DelegationTokenIdentifier> token
) throws IOException,
InterruptedException {
- try {
- client.cancelDelegationToken(token);
- } catch (RemoteException re) {
- throw re.unwrapRemoteException(InvalidToken.class,
- AccessControlException.class);
- }
+ token.cancel(getConf());
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java Mon Jan 14 03:44:35 2013
@@ -32,7 +32,7 @@ import org.apache.hadoop.io.Text;
* the job. JobID consists of two parts. First part
* represents the jobtracker identifier, so that jobID to jobtracker map
* is defined. For cluster setup this string is the jobtracker
- * start time, for local setting, it is "local".
+ * start time, for local setting, it is "local" and a random number.
* Second part of the JobID is the job number. <br>
* An example JobID is :
* <code>job_200707121733_0003</code> , which represents the third job
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java Mon Jan 14 03:44:35 2013
@@ -251,9 +251,8 @@ class JobSubmitter {
}
// set the timestamps of the archives and files
- ClientDistributedCacheManager.determineTimestamps(conf);
// set the public/private visibility of the archives and files
- ClientDistributedCacheManager.determineCacheVisibilities(conf);
+ ClientDistributedCacheManager.determineTimestampsAndCacheVisibilities(conf);
// get DelegationToken for each cached file
ClientDistributedCacheManager.getDelegationTokens(conf, job
.getCredentials());
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Mon Jan 14 03:44:35 2013
@@ -127,6 +127,10 @@ public interface MRJobConfig {
public static final String MAPREDUCE_JOB_USER_CLASSPATH_FIRST = "mapreduce.job.user.classpath.first";
+ public static final String MAPREDUCE_JOB_CLASSLOADER = "mapreduce.job.classloader";
+
+ public static final String MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES = "mapreduce.job.classloader.system.classes";
+
public static final String IO_SORT_FACTOR = "mapreduce.task.io.sort.factor";
public static final String IO_SORT_MB = "mapreduce.task.io.sort.mb";
@@ -184,7 +188,8 @@ public interface MRJobConfig {
public static final String MAP_MEMORY_MB = "mapreduce.map.memory.mb";
public static final int DEFAULT_MAP_MEMORY_MB = 1024;
- public static final String MAP_MEMORY_PHYSICAL_MB = "mapreduce.map.memory.physical.mb";
+ public static final String MAP_CPU_VCORES = "mapreduce.map.cpu.vcores";
+ public static final int DEFAULT_MAP_CPU_VCORES = 1;
public static final String MAP_ENV = "mapreduce.map.env";
@@ -228,11 +233,12 @@ public interface MRJobConfig {
public static final String REDUCE_MARKRESET_BUFFER_SIZE = "mapreduce.reduce.markreset.buffer.size";
- public static final String REDUCE_MEMORY_PHYSICAL_MB = "mapreduce.reduce.memory.physical.mb";
-
public static final String REDUCE_MEMORY_MB = "mapreduce.reduce.memory.mb";
public static final int DEFAULT_REDUCE_MEMORY_MB = 1024;
+ public static final String REDUCE_CPU_VCORES = "mapreduce.reduce.cpu.vcores";
+ public static final int DEFAULT_REDUCE_CPU_VCORES = 1;
+
public static final String REDUCE_MEMORY_TOTAL_BYTES = "mapreduce.reduce.memory.totalbytes";
public static final String SHUFFLE_INPUT_BUFFER_PERCENT = "mapreduce.reduce.shuffle.input.buffer.percent";
@@ -353,11 +359,21 @@ public interface MRJobConfig {
MR_AM_PREFIX+"resource.mb";
public static final int DEFAULT_MR_AM_VMEM_MB = 1536;
+ /** The number of virtual cores the MR app master needs.*/
+ public static final String MR_AM_CPU_VCORES =
+ MR_AM_PREFIX+"resource.cpu-vcores";
+ public static final int DEFAULT_MR_AM_CPU_VCORES = 1;
+
/** Command line arguments passed to the MR app master.*/
public static final String MR_AM_COMMAND_OPTS =
MR_AM_PREFIX+"command-opts";
public static final String DEFAULT_MR_AM_COMMAND_OPTS = "-Xmx1024m";
+ /** Admin command opts passed to the MR app master.*/
+ public static final String MR_AM_ADMIN_COMMAND_OPTS =
+ MR_AM_PREFIX+"admin-command-opts";
+ public static final String DEFAULT_MR_AM_ADMIN_COMMAND_OPTS = "";
+
/** Root Logging level passed to the MR app master.*/
public static final String MR_AM_LOG_LEVEL =
MR_AM_PREFIX+"log.level";
@@ -465,6 +481,25 @@ public interface MRJobConfig {
public static final int DEFAULT_MR_AM_TO_RM_WAIT_INTERVAL_MS = 360000;
/**
+ * How long to wait in milliseconds for the output committer to cancel
+ * an operation when the job is being killed
+ */
+ public static final String MR_AM_COMMITTER_CANCEL_TIMEOUT_MS =
+ MR_AM_PREFIX + "job.committer.cancel-timeout";
+ public static final int DEFAULT_MR_AM_COMMITTER_CANCEL_TIMEOUT_MS =
+ 60 * 1000;
+
+ /**
+ * Defines a time window in milliseconds for output committer operations.
+ * If contact with the RM has occurred within this window then commit
+ * operations are allowed, otherwise the AM will not allow output committer
+ * operations until contact with the RM has been re-established.
+ */
+ public static final String MR_AM_COMMIT_WINDOW_MS =
+ MR_AM_PREFIX + "job.committer.commit-window";
+ public static final int DEFAULT_MR_AM_COMMIT_WINDOW_MS = 10 * 1000;
+
+ /**
* Boolean. Create the base dirs in the JobHistoryEventHandler
* Set to false for multi-user clusters. This is an internal config that
* is set by the MR framework and read by it too.
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputCommitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputCommitter.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputCommitter.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputCommitter.java Mon Jan 14 03:44:35 2013
@@ -54,7 +54,11 @@ import org.apache.hadoop.classification.
* The methods in this class can be called from several different processes and
* from several different contexts. It is important to know which process and
* which context each is called from. Each method should be marked accordingly
- * in its documentation.
+ * in its documentation. It is also important to note that not all methods are
+ * guaranteed to be called once and only once. If a method is not guaranteed to
+ * have this property the output committer needs to handle this appropriately.
+ * Also note it will only be in rare situations where they may be called
+ * multiple times for the same task.
*
* @see org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
* @see JobContext
@@ -65,7 +69,8 @@ import org.apache.hadoop.classification.
public abstract class OutputCommitter {
/**
* For the framework to setup the job output during initialization. This is
- * called from the application master process for the entire job.
+ * called from the application master process for the entire job. This will be
+ * called multiple times, once per job attempt.
*
* @param jobContext Context of the job whose output is being written.
* @throws IOException if temporary output could not be created
@@ -74,7 +79,8 @@ public abstract class OutputCommitter {
/**
* For cleaning up the job's output after job completion. This is called
- * from the application master process for the entire job.
+ * from the application master process for the entire job. This may be called
+ * multiple times.
*
* @param jobContext Context of the job whose output is being written.
* @throws IOException
@@ -87,7 +93,9 @@ public abstract class OutputCommitter {
/**
* For committing job's output after successful job completion. Note that this
* is invoked for jobs with final runstate as SUCCESSFUL. This is called
- * from the application master process for the entire job.
+ * from the application master process for the entire job. This is guaranteed
+ * to only be called once. If it throws an exception the entire job will
+ * fail.
*
* @param jobContext Context of the job whose output is being written.
* @throws IOException
@@ -101,7 +109,7 @@ public abstract class OutputCommitter {
* For aborting an unsuccessful job's output. Note that this is invoked for
* jobs with final runstate as {@link JobStatus.State#FAILED} or
* {@link JobStatus.State#KILLED}. This is called from the application
- * master process for the entire job.
+ * master process for the entire job. This may be called multiple times.
*
* @param jobContext Context of the job whose output is being written.
* @param state final runstate of the job
@@ -114,7 +122,9 @@ public abstract class OutputCommitter {
/**
* Sets up output for the task. This is called from each individual task's
- * process that will output to HDFS, and it is called just for that task.
+ * process that will output to HDFS, and it is called just for that task. This
+ * may be called multiple times for the same task, but for different task
+ * attempts.
*
* @param taskContext Context of the task whose output is being written.
* @throws IOException
@@ -141,7 +151,10 @@ public abstract class OutputCommitter {
* is called to commit an individual task's output. This is to mark
* that tasks output as complete, as {@link #commitJob(JobContext)} will
* also be called later on if the entire job finished successfully. This
- * is called from a task's process.
+ * is called from a task's process. This may be called multiple times for the
+ * same task, but different task attempts. It should be very rare for this to
+ * be called multiple times and requires odd networking failures to make this
+ * happen. In the future the Hadoop framework may eliminate this race.
*
* @param taskContext Context of the task whose output is being written.
* @throws IOException if commit is not successful.
@@ -151,7 +164,8 @@ public abstract class OutputCommitter {
/**
* Discard the task output. This is called from a task's process to clean
- * up a single task's output that can not yet been committed.
+ * up a single task's output that can not yet been committed. This may be
+ * called multiple times for the same task, but for different task attempts.
*
* @param taskContext
* @throws IOException
@@ -184,6 +198,9 @@ public abstract class OutputCommitter {
*
* If an exception is thrown the task will be attempted again.
*
+ * This may be called multiple times for the same task. But from different
+ * application attempts.
+ *
* @param taskContext Context of the task whose output is being recovered
* @throws IOException
*/
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java Mon Jan 14 03:44:35 2013
@@ -19,6 +19,8 @@ package org.apache.hadoop.mapreduce.file
import java.io.IOException;
import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
@@ -39,6 +41,25 @@ public class ClientDistributedCacheManag
/**
* Determines timestamps of files to be cached, and stores those
+ * in the configuration. Determines the visibilities of the distributed cache
+ * files and archives. The visibility of a cache path is "public" if the leaf
+ * component has READ permissions for others, and the parent subdirs have
+ * EXECUTE permissions for others.
+ *
+ * This is an internal method!
+ *
+ * @param job
+ * @throws IOException
+ */
+ public static void determineTimestampsAndCacheVisibilities(Configuration job)
+ throws IOException {
+ Map<URI, FileStatus> statCache = new HashMap<URI, FileStatus>();
+ determineTimestamps(job, statCache);
+ determineCacheVisibilities(job, statCache);
+ }
+
+ /**
+ * Determines timestamps of files to be cached, and stores those
* in the configuration. This is intended to be used internally by JobClient
* after all cache files have been added.
*
@@ -47,16 +68,17 @@ public class ClientDistributedCacheManag
* @param job Configuration of a job.
* @throws IOException
*/
- public static void determineTimestamps(Configuration job) throws IOException {
+ public static void determineTimestamps(Configuration job,
+ Map<URI, FileStatus> statCache) throws IOException {
URI[] tarchives = DistributedCache.getCacheArchives(job);
if (tarchives != null) {
- FileStatus status = getFileStatus(job, tarchives[0]);
+ FileStatus status = getFileStatus(job, tarchives[0], statCache);
StringBuilder archiveFileSizes =
new StringBuilder(String.valueOf(status.getLen()));
StringBuilder archiveTimestamps =
new StringBuilder(String.valueOf(status.getModificationTime()));
for (int i = 1; i < tarchives.length; i++) {
- status = getFileStatus(job, tarchives[i]);
+ status = getFileStatus(job, tarchives[i], statCache);
archiveFileSizes.append(",");
archiveFileSizes.append(String.valueOf(status.getLen()));
archiveTimestamps.append(",");
@@ -68,13 +90,13 @@ public class ClientDistributedCacheManag
URI[] tfiles = DistributedCache.getCacheFiles(job);
if (tfiles != null) {
- FileStatus status = getFileStatus(job, tfiles[0]);
+ FileStatus status = getFileStatus(job, tfiles[0], statCache);
StringBuilder fileSizes =
new StringBuilder(String.valueOf(status.getLen()));
StringBuilder fileTimestamps = new StringBuilder(String.valueOf(
status.getModificationTime()));
for (int i = 1; i < tfiles.length; i++) {
- status = getFileStatus(job, tfiles[i]);
+ status = getFileStatus(job, tfiles[i], statCache);
fileSizes.append(",");
fileSizes.append(String.valueOf(status.getLen()));
fileTimestamps.append(",");
@@ -123,25 +145,25 @@ public class ClientDistributedCacheManag
* @param job
* @throws IOException
*/
- public static void determineCacheVisibilities(Configuration job)
- throws IOException {
+ public static void determineCacheVisibilities(Configuration job,
+ Map<URI, FileStatus> statCache) throws IOException {
URI[] tarchives = DistributedCache.getCacheArchives(job);
if (tarchives != null) {
StringBuilder archiveVisibilities =
- new StringBuilder(String.valueOf(isPublic(job, tarchives[0])));
+ new StringBuilder(String.valueOf(isPublic(job, tarchives[0], statCache)));
for (int i = 1; i < tarchives.length; i++) {
archiveVisibilities.append(",");
- archiveVisibilities.append(String.valueOf(isPublic(job, tarchives[i])));
+ archiveVisibilities.append(String.valueOf(isPublic(job, tarchives[i], statCache)));
}
setArchiveVisibilities(job, archiveVisibilities.toString());
}
URI[] tfiles = DistributedCache.getCacheFiles(job);
if (tfiles != null) {
StringBuilder fileVisibilities =
- new StringBuilder(String.valueOf(isPublic(job, tfiles[0])));
+ new StringBuilder(String.valueOf(isPublic(job, tfiles[0], statCache)));
for (int i = 1; i < tfiles.length; i++) {
fileVisibilities.append(",");
- fileVisibilities.append(String.valueOf(isPublic(job, tfiles[i])));
+ fileVisibilities.append(String.valueOf(isPublic(job, tfiles[i], statCache)));
}
setFileVisibilities(job, fileVisibilities.toString());
}
@@ -193,19 +215,13 @@ public class ClientDistributedCacheManag
}
/**
- * Returns {@link FileStatus} of a given cache file on hdfs.
- *
- * @param conf configuration
- * @param cache cache file
- * @return {@link FileStatus} of a given cache file on hdfs
- * @throws IOException
+ * Gets the file status for the given URI. If the URI is in the cache,
+ * returns it. Otherwise, fetches it and adds it to the cache.
*/
- static FileStatus getFileStatus(Configuration conf, URI cache)
- throws IOException {
- FileSystem fileSystem = FileSystem.get(cache, conf);
- Path filePath = new Path(cache.getPath());
-
- return fileSystem.getFileStatus(filePath);
+ private static FileStatus getFileStatus(Configuration job, URI uri,
+ Map<URI, FileStatus> statCache) throws IOException {
+ FileSystem fileSystem = FileSystem.get(uri, job);
+ return getFileStatus(fileSystem, uri, statCache);
}
/**
@@ -216,14 +232,15 @@ public class ClientDistributedCacheManag
* @return true if the path in the uri is visible to all, false otherwise
* @throws IOException
*/
- static boolean isPublic(Configuration conf, URI uri) throws IOException {
+ static boolean isPublic(Configuration conf, URI uri,
+ Map<URI, FileStatus> statCache) throws IOException {
FileSystem fs = FileSystem.get(uri, conf);
Path current = new Path(uri.getPath());
//the leaf level file should be readable by others
- if (!checkPermissionOfOther(fs, current, FsAction.READ)) {
+ if (!checkPermissionOfOther(fs, current, FsAction.READ, statCache)) {
return false;
}
- return ancestorsHaveExecutePermissions(fs, current.getParent());
+ return ancestorsHaveExecutePermissions(fs, current.getParent(), statCache);
}
/**
@@ -231,12 +248,12 @@ public class ClientDistributedCacheManag
* permission set for all users (i.e. that other users can traverse
* the directory heirarchy to the given path)
*/
- static boolean ancestorsHaveExecutePermissions(FileSystem fs, Path path)
- throws IOException {
+ static boolean ancestorsHaveExecutePermissions(FileSystem fs, Path path,
+ Map<URI, FileStatus> statCache) throws IOException {
Path current = path;
while (current != null) {
//the subdirs in the path should have execute permissions for others
- if (!checkPermissionOfOther(fs, current, FsAction.EXECUTE)) {
+ if (!checkPermissionOfOther(fs, current, FsAction.EXECUTE, statCache)) {
return false;
}
current = current.getParent();
@@ -254,8 +271,8 @@ public class ClientDistributedCacheManag
* @throws IOException
*/
private static boolean checkPermissionOfOther(FileSystem fs, Path path,
- FsAction action) throws IOException {
- FileStatus status = fs.getFileStatus(path);
+ FsAction action, Map<URI, FileStatus> statCache) throws IOException {
+ FileStatus status = getFileStatus(fs, path.toUri(), statCache);
FsPermission perms = status.getPermission();
FsAction otherAction = perms.getOtherAction();
if (otherAction.implies(action)) {
@@ -263,4 +280,14 @@ public class ClientDistributedCacheManag
}
return false;
}
+
+ private static FileStatus getFileStatus(FileSystem fs, URI uri,
+ Map<URI, FileStatus> statCache) throws IOException {
+ FileStatus stat = statCache.get(uri);
+ if (stat == null) {
+ stat = fs.getFileStatus(new Path(uri));
+ statCache.put(uri, stat);
+ }
+ return stat;
+ }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java Mon Jan 14 03:44:35 2013
@@ -54,7 +54,7 @@ import org.apache.hadoop.yarn.api.record
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-public class JobHistoryParser {
+public class JobHistoryParser implements HistoryEventHandler {
private static final Log LOG = LogFactory.getLog(JobHistoryParser.class);
@@ -94,6 +94,34 @@ public class JobHistoryParser {
this.in = in;
}
+ public synchronized void parse(HistoryEventHandler handler)
+ throws IOException {
+ parse(new EventReader(in), handler);
+ }
+
+ /**
+ * Only used for unit tests.
+ */
+ @Private
+ public synchronized void parse(EventReader reader, HistoryEventHandler handler)
+ throws IOException {
+ int eventCtr = 0;
+ HistoryEvent event;
+ try {
+ while ((event = reader.getNextEvent()) != null) {
+ handler.handleEvent(event);
+ ++eventCtr;
+ }
+ } catch (IOException ioe) {
+ LOG.info("Caught exception parsing history file after " + eventCtr +
+ " events", ioe);
+ parseException = ioe;
+ } finally {
+ in.close();
+ }
+ }
+
+
/**
* Parse the entire history file and populate the JobInfo object
* The first invocation will populate the object, subsequent calls
@@ -122,21 +150,7 @@ public class JobHistoryParser {
}
info = new JobInfo();
-
- int eventCtr = 0;
- HistoryEvent event;
- try {
- while ((event = reader.getNextEvent()) != null) {
- handleEvent(event);
- ++eventCtr;
- }
- } catch (IOException ioe) {
- LOG.info("Caught exception parsing history file after " + eventCtr +
- " events", ioe);
- parseException = ioe;
- } finally {
- in.close();
- }
+ parse(reader, this);
return info;
}
@@ -150,7 +164,8 @@ public class JobHistoryParser {
return parseException;
}
- private void handleEvent(HistoryEvent event) {
+ @Override
+ public void handleEvent(HistoryEvent event) {
EventType type = event.getEventType();
switch (type) {
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer Mon Jan 14 03:44:35 2013
@@ -1,2 +1 @@
-org.apache.hadoop.mapred.JobClient$Renewer
org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier$Renewer
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml Mon Jan 14 03:44:35 2013
@@ -219,6 +219,22 @@
</property>
<property>
+ <name>mapreduce.map.cpu.vcores</name>
+ <value>1</value>
+ <description>
+ The number of virtual cores required for each map task.
+ </description>
+</property>
+
+<property>
+ <name>mapreduce.reduce.cpu.vcores</name>
+ <value>1</value>
+ <description>
+ The number of virtual cores required for each reduce task.
+ </description>
+</property>
+
+<property>
<name>mapreduce.reduce.merge.inmem.threshold</name>
<value>1000</value>
<description>The threshold, in terms of the number of files
@@ -859,6 +875,20 @@
</property>
<property>
+ <name>yarn.app.mapreduce.am.admin-command-opts</name>
+ <value></value>
+ <description>Java opts for the MR App Master processes for admin purposes.
+ It will appears before the opts set by yarn.app.mapreduce.am.command-opts and
+ thus its options can be overridden user.
+
+ Usage of -Djava.library.path can cause programs to no longer function if
+ hadoop native libraries are used. These values should instead be set as part
+ of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and
+ mapreduce.reduce.env config settings.
+ </description>
+</property>
+
+<property>
<name>yarn.app.mapreduce.am.job.task.listener.thread-count</name>
<value>30</value>
<description>The number of threads used to handle RPC calls in the
@@ -874,6 +904,22 @@
</property>
<property>
+ <name>yarn.app.mapreduce.am.job.committer.cancel-timeout</name>
+ <value>60000</value>
+ <description>The amount of time in milliseconds to wait for the output
+ committer to cancel an operation if the job is killed</description>
+</property>
+
+<property>
+ <name>yarn.app.mapreduce.am.job.committer.commit-window</name>
+ <value>10000</value>
+ <description>Defines a time window in milliseconds for output commit
+ operations. If contact with the RM has occurred within this window then
+ commits are allowed, otherwise the AM will not allow output commits until
+ contact with the RM has been re-established.</description>
+</property>
+
+<property>
<name>yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms</name>
<value>1000</value>
<description>The interval in ms at which the MR AppMaster should send
@@ -901,12 +947,37 @@
</property>
<property>
+ <name>yarn.app.mapreduce.am.resource.cpu-vcores</name>
+ <value>1</value>
+ <description>
+ The number of virtual CPU cores the MR AppMaster needs.
+ </description>
+</property>
+
+<property>
<description>CLASSPATH for MR applications. A comma-separated list
of CLASSPATH entries</description>
<name>mapreduce.application.classpath</name>
<value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
</property>
+<property>
+ <name>mapreduce.job.classloader</name>
+ <value>false</value>
+ <description>Whether to use a separate (isolated) classloader for
+ user classes in the task JVM.</description>
+</property>
+
+<property>
+ <name>mapreduce.job.classloader.system.classes</name>
+ <value>java.,javax.,org.apache.commons.logging.,org.apache.log4j.,org.apache.hadoop.</value>
+ <description>A comma-separated list of classes that should be loaded from the
+ system classpath, not the user-supplied JARs, when mapreduce.job.classloader
+ is enabled. Names ending in '.' (period) are treated as package names,
+ and names starting with a '-' are treated as negative matches.
+ </description>
+</property>
+
<!-- jobhistory properties -->
<property>
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1426019-1432788
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Mon Jan 14 03:44:35 2013
@@ -0,0 +1,4 @@
+target
+.classpath
+.project
+.settings
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java Mon Jan 14 03:44:35 2013
@@ -24,6 +24,7 @@ import java.security.AccessControlExcept
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.Collection;
+import java.util.EnumSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -37,6 +38,8 @@ import org.apache.hadoop.mapreduce.TypeC
import org.apache.hadoop.mapreduce.v2.api.HSClientProtocol;
import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
import org.apache.hadoop.mapreduce.v2.api.MRDelegationTokenIdentifier;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest;
@@ -61,6 +64,8 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
@@ -75,7 +80,6 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.records.DelegationToken;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -83,6 +87,7 @@ import org.apache.hadoop.yarn.ipc.RPCUti
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
@@ -314,13 +319,10 @@ public class HistoryClientService extend
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
// Verify that the connection is kerberos authenticated
- AuthenticationMethod authMethod = UserGroupInformation
- .getRealAuthenticationMethod(ugi);
- if (UserGroupInformation.isSecurityEnabled()
- && (authMethod != AuthenticationMethod.KERBEROS)) {
- throw new IOException(
- "Delegation Token can be issued only with kerberos authentication");
- }
+ if (!isAllowedDelegationTokenOp()) {
+ throw new IOException(
+ "Delegation Token can be issued only with kerberos authentication");
+ }
GetDelegationTokenResponse response = recordFactory.newRecordInstance(
GetDelegationTokenResponse.class);
@@ -347,6 +349,55 @@ public class HistoryClientService extend
}
}
+ @Override
+ public RenewDelegationTokenResponse renewDelegationToken(
+ RenewDelegationTokenRequest request) throws YarnRemoteException {
+ try {
+ if (!isAllowedDelegationTokenOp()) {
+ throw new IOException(
+ "Delegation Token can be renewed only with kerberos authentication");
+ }
+
+ DelegationToken protoToken = request.getDelegationToken();
+ Token<MRDelegationTokenIdentifier> token = new Token<MRDelegationTokenIdentifier>(
+ protoToken.getIdentifier().array(), protoToken.getPassword()
+ .array(), new Text(protoToken.getKind()), new Text(
+ protoToken.getService()));
+
+ String user = UserGroupInformation.getCurrentUser().getShortUserName();
+ long nextExpTime = jhsDTSecretManager.renewToken(token, user);
+ RenewDelegationTokenResponse renewResponse = Records
+ .newRecord(RenewDelegationTokenResponse.class);
+ renewResponse.setNextExpirationTime(nextExpTime);
+ return renewResponse;
+ } catch (IOException e) {
+ throw RPCUtil.getRemoteException(e);
+ }
+ }
+
+ @Override
+ public CancelDelegationTokenResponse cancelDelegationToken(
+ CancelDelegationTokenRequest request) throws YarnRemoteException {
+ try {
+ if (!isAllowedDelegationTokenOp()) {
+ throw new IOException(
+ "Delegation Token can be cancelled only with kerberos authentication");
+ }
+
+ DelegationToken protoToken = request.getDelegationToken();
+ Token<MRDelegationTokenIdentifier> token = new Token<MRDelegationTokenIdentifier>(
+ protoToken.getIdentifier().array(), protoToken.getPassword()
+ .array(), new Text(protoToken.getKind()), new Text(
+ protoToken.getService()));
+
+ String user = UserGroupInformation.getCurrentUser().getShortUserName();
+ jhsDTSecretManager.cancelToken(token, user);
+ return Records.newRecord(CancelDelegationTokenResponse.class);
+ } catch (IOException e) {
+ throw RPCUtil.getRemoteException(e);
+ }
+ }
+
private void checkAccess(Job job, JobACL jobOperation)
throws YarnRemoteException {
@@ -362,5 +413,18 @@ public class HistoryClientService extend
+ jobOperation.name() + " on " + job.getID()));
}
}
+
+ private boolean isAllowedDelegationTokenOp() throws IOException {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ return EnumSet.of(AuthenticationMethod.KERBEROS,
+ AuthenticationMethod.KERBEROS_SSL,
+ AuthenticationMethod.CERTIFICATE)
+ .contains(UserGroupInformation.getCurrentUser()
+ .getRealAuthenticationMethod());
+ } else {
+ return true;
+ }
+ }
+
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java Mon Jan 14 03:44:35 2013
@@ -667,6 +667,9 @@ public class HistoryFileManager extends
}
});
}
+ } else if (old != null && !old.isMovePending()) {
+ //This is a duplicate so just delete it
+ fileInfo.delete();
}
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java Mon Jan 14 03:44:35 2013
@@ -24,6 +24,8 @@ import java.util.HashMap;
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest;
@@ -48,6 +50,8 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse;
import org.apache.hadoop.mapreduce.v2.api.records.CounterGroup;
import org.apache.hadoop.mapreduce.v2.api.records.Counters;
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
@@ -215,7 +219,21 @@ public class NotRunningJob implements MR
/* Should not be invoked by anyone. */
throw new NotImplementedException();
}
-
+
+ @Override
+ public RenewDelegationTokenResponse renewDelegationToken(
+ RenewDelegationTokenRequest request) throws YarnRemoteException {
+ /* Should not be invoked by anyone. */
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public CancelDelegationTokenResponse cancelDelegationToken(
+ CancelDelegationTokenRequest request) throws YarnRemoteException {
+ /* Should not be invoked by anyone. */
+ throw new NotImplementedException();
+ }
+
@Override
public InetSocketAddress getConnectAddress() {
/* Should not be invoked by anyone. Normally used to set token service */
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java Mon Jan 14 03:44:35 2013
@@ -33,7 +33,6 @@ import org.apache.hadoop.mapreduce.Queue
import org.apache.hadoop.mapreduce.QueueInfo;
import org.apache.hadoop.mapreduce.TaskTrackerInfo;
import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -41,9 +40,9 @@ import org.apache.hadoop.yarn.api.Client
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
+import org.apache.hadoop.yarn.client.YarnClientImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.ProtoUtils;
-import org.apache.hadoop.yarn.client.YarnClientImpl;
public class ResourceMgrDelegate extends YarnClientImpl {
private static final Log LOG = LogFactory.getLog(ResourceMgrDelegate.class);
@@ -62,11 +61,6 @@ public class ResourceMgrDelegate extends
init(conf);
start();
}
-
- public void cancelDelegationToken(Token<DelegationTokenIdentifier> arg0)
- throws IOException, InterruptedException {
- return;
- }
public TaskTrackerInfo[] getActiveTrackers() throws IOException,
InterruptedException {
@@ -168,13 +162,6 @@ public class ResourceMgrDelegate extends
return 0;
}
- public long renewDelegationToken(Token<DelegationTokenIdentifier> arg0)
- throws IOException, InterruptedException {
- // TODO: Implement renewDelegationToken
- LOG.warn("renewDelegationToken - Not implemented");
- return 0;
- }
-
public ApplicationId getApplicationId() {
return applicationId;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1432796&r1=1432795&r2=1432796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Mon Jan 14 03:44:35 2013
@@ -85,6 +85,7 @@ import org.apache.hadoop.yarn.util.Build
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.ProtoUtils;
+import com.google.common.annotations.VisibleForTesting;
/**
* This class enables the current JobClient (0.22 hadoop) to run on YARN.
@@ -158,7 +159,7 @@ public class YARNRunner implements Clien
@Override
public void cancelDelegationToken(Token<DelegationTokenIdentifier> arg0)
throws IOException, InterruptedException {
- resMgrDelegate.cancelDelegationToken(arg0);
+ throw new UnsupportedOperationException("Use Token.renew instead");
}
@Override
@@ -184,18 +185,18 @@ public class YARNRunner implements Clien
return resMgrDelegate.getClusterMetrics();
}
- private Token<?> getDelegationTokenFromHS(
- MRClientProtocol hsProxy, Text renewer) throws IOException,
- InterruptedException {
+ @VisibleForTesting
+ Token<?> getDelegationTokenFromHS(MRClientProtocol hsProxy)
+ throws IOException, InterruptedException {
GetDelegationTokenRequest request = recordFactory
.newRecordInstance(GetDelegationTokenRequest.class);
- request.setRenewer(renewer.toString());
+ request.setRenewer(Master.getMasterPrincipal(conf));
DelegationToken mrDelegationToken = hsProxy.getDelegationToken(request)
.getDelegationToken();
return ProtoUtils.convertFromProtoFormat(mrDelegationToken,
hsProxy.getConnectAddress());
}
-
+
@Override
public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
throws IOException, InterruptedException {
@@ -269,8 +270,7 @@ public class YARNRunner implements Clien
// the delegation tokens for the HistoryServer also.
if (conf.getBoolean(JobClient.HS_DELEGATION_TOKEN_REQUIRED,
DEFAULT_HS_DELEGATION_TOKEN_REQUIRED)) {
- Token hsDT = getDelegationTokenFromHS(hsProxy, new Text(
- conf.get(JobClient.HS_DELEGATION_TOKEN_RENEWER)));
+ Token hsDT = getDelegationTokenFromHS(hsProxy);
ts.addToken(hsDT.getService(), hsDT);
}
}
@@ -324,8 +324,16 @@ public class YARNRunner implements Clien
// Setup resource requirements
Resource capability = recordFactory.newRecordInstance(Resource.class);
- capability.setMemory(conf.getInt(MRJobConfig.MR_AM_VMEM_MB,
- MRJobConfig.DEFAULT_MR_AM_VMEM_MB));
+ capability.setMemory(
+ conf.getInt(
+ MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB
+ )
+ );
+ capability.setVirtualCores(
+ conf.getInt(
+ MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES
+ )
+ );
LOG.debug("AppMaster capability = " + capability);
// Setup LocalResources
@@ -386,6 +394,11 @@ public class YARNRunner implements Clien
MRJobConfig.MR_AM_LOG_LEVEL, MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL);
MRApps.addLog4jSystemProperties(logLevel, logSize, vargs);
+ // Add AM admin command opts before user command opts
+ // so that it can be overridden by user
+ vargs.add(conf.get(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
+ MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS));
+
vargs.add(conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS));
@@ -466,7 +479,7 @@ public class YARNRunner implements Clien
@Override
public long renewDelegationToken(Token<DelegationTokenIdentifier> arg0)
throws IOException, InterruptedException {
- return resMgrDelegate.renewDelegationToken(arg0);
+ throw new UnsupportedOperationException("Use Token.renew instead");
}