You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ss...@apache.org on 2013/10/13 22:19:07 UTC
svn commit: r1531742 - in
/hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/s...
Author: sseth
Date: Sun Oct 13 20:19:06 2013
New Revision: 1531742
URL: http://svn.apache.org/r1531742
Log:
merge MAPREDUCE-5329. Allow MR applications to use additional AuxServices, which are compatible with the default MapReduce shuffle. Contributed by Avner BenHanoch.
Added:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestShuffleProvider.java
- copied unchanged from r1531741, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestShuffleProvider.java
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1531742&r1=1531741&r2=1531742&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Sun Oct 13 20:19:06 2013
@@ -61,6 +61,10 @@ Release 2.2.1 - UNRELEASED
NEW FEATURES
IMPROVEMENTS
+
+ MAPREDUCE-5329. Allow MR applications to use additional AuxServices,
+ which are compatible with the default MapReduce shuffle.
+ (Avner BenHanoch via sseth)
OPTIMIZATIONS
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1531742&r1=1531741&r2=1531742&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Sun Oct 13 20:19:06 2013
@@ -31,6 +31,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.Collection;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
@@ -722,6 +723,32 @@ public abstract class TaskAttemptImpl im
serviceData.put(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
ShuffleHandler.serializeServiceData(shuffleToken));
+ // add external shuffle-providers - if any
+ Collection<String> shuffleProviders = conf.getStringCollection(
+ MRJobConfig.MAPREDUCE_JOB_SHUFFLE_PROVIDER_SERVICES);
+ if (! shuffleProviders.isEmpty()) {
+ Collection<String> auxNames = conf.getStringCollection(
+ YarnConfiguration.NM_AUX_SERVICES);
+
+ for (final String shuffleProvider : shuffleProviders) {
+ if (shuffleProvider.equals(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID)) {
+ continue; // skip built-in shuffle-provider that was already inserted with shuffle secret key
+ }
+ if (auxNames.contains(shuffleProvider)) {
+ LOG.info("Adding ShuffleProvider Service: " + shuffleProvider + " to serviceData");
+ // This only serves for INIT_APP notifications
+ // The shuffle service needs to be able to work with the host:port information provided by the AM
+ // (i.e. shuffle services which require custom location / other configuration are not supported)
+ serviceData.put(shuffleProvider, ByteBuffer.allocate(0));
+ }
+ else {
+ throw new YarnRuntimeException("ShuffleProvider Service: " + shuffleProvider +
+ " was NOT found in the list of aux-services that are available in this NM." +
+ " You may need to specify this ShuffleProvider as an aux-service in your yarn-site.xml");
+ }
+ }
+ }
+
Apps.addToEnvironment(
environment,
Environment.CLASSPATH.name(),
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1531742&r1=1531741&r2=1531742&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Sun Oct 13 20:19:06 2013
@@ -133,6 +133,13 @@ public interface MRJobConfig {
public static final String MAPREDUCE_JOB_CLASSLOADER = "mapreduce.job.classloader";
+ /**
+ * A comma-separated list of services that function as ShuffleProvider aux-services
+ * (in addition to the built-in ShuffleHandler).
+ * These services can serve shuffle requests from reducetasks.
+ */
+ public static final String MAPREDUCE_JOB_SHUFFLE_PROVIDER_SERVICES = "mapreduce.job.shuffle.provider.services";
+
public static final String MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES = "mapreduce.job.classloader.system.classes";
public static final String IO_SORT_FACTOR = "mapreduce.task.io.sort.factor";