You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by to...@apache.org on 2012/07/31 01:31:50 UTC
svn commit: r1367365 - in
/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project: ./ conf/
dev-support/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/
hadoop-mapreduce-client/hadoop-ma...
Author: todd
Date: Mon Jul 30 23:31:42 2012
New Revision: 1367365
URL: http://svn.apache.org/viewvc?rev=1367365&view=rev
Log:
Merge trunk into QJM branch
Added:
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/ (props changed)
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/resources/
- copied from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/resources/
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/resources/yarn-site.xml
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/resources/yarn-site.xml
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CLIMiniCluster.apt.vm
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CLIMiniCluster.apt.vm
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/EncryptedShuffle.apt.vm
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/EncryptedShuffle.apt.vm
Modified:
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/conf/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (contents, props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/c++/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/block_forensics/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/build-contrib.xml (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/build.xml (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/data_join/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/eclipse-plugin/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/index/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/vaidya/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/examples/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/java/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/webapps/job/ (props changed)
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1363593-1367364
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt Mon Jul 30 23:31:42 2012
@@ -132,6 +132,11 @@ Branch-2 ( Unreleased changes )
NEW FEATURES
+ MAPREDUCE-987. Exposing MiniDFS and MiniMR clusters as a single process
+ command-line. (ahmed via tucu)
+
+ MAPREDUCE-4417. add support for encrypted shuffle (tucu)
+
IMPROVEMENTS
MAPREDUCE-4157. ResourceManager should not kill apps that are well behaved
@@ -148,6 +153,12 @@ Branch-2 ( Unreleased changes )
MAPREDUCE-4407. Add hadoop-yarn-server-tests-<version>-tests.jar to hadoop dist
package. (ahmed via tucu)
+ MAPREDUCE-4465. Update description of yarn.nodemanager.address property.
+ (bowang via tucu)
+
+ MAPREDUCE-4342. Distributed Cache gives inconsistent result if cache files
+ get deleted from tasktracker. (mayank_bansal via tucu)
+
Release 2.1.0-alpha - Unreleased
INCOMPATIBLE CHANGES
@@ -158,6 +169,9 @@ Release 2.1.0-alpha - Unreleased
MAPREDUCE-3451. Port Fair Scheduler to MR2 (pwendell via tucu)
+ MAPREDUCE-4438. Add a simple, generic client to run 'easy' AMs in YARN.
+ (Bikas Saha via acmurthy)
+
IMPROVEMENTS
MAPREDUCE-4440. Changed SchedulerApp and SchedulerNode to be a minimal
@@ -739,6 +753,13 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4448. Fix NM crash during app cleanup if aggregation didn't
init. (Jason Lowe via daryn)
+ MAPREDUCE-3893. allow capacity scheduler configs maximum-applications and
+ maximum-am-resource-percent configurable on a per queue basis (tgraves via
+ bobby)
+
+ MAPREDUCE-4467. IndexCache failures due to missing synchronization
+ (Kihwal Lee via tgraves)
+
MAPREDUCE-4423. Potential infinite fetching of map output (Robert Evans
via tgraves)
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1363593-1367364
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1363593-1367364
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml Mon Jul 30 23:31:42 2012
@@ -473,5 +473,10 @@
<!--
The above 2 fields are accessed locally and only via methods that are synchronized.
-->
-
+
+ <Match>
+ <Class name="org.apache.hadoop.mapred.ShuffleHandler" />
+ <Field name="sslFileBufferSize" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC" />
+ </Match>
</FindBugsFilter>
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java Mon Jul 30 23:31:42 2012
@@ -34,6 +34,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TypeConverter;
@@ -43,6 +44,7 @@ import org.apache.hadoop.mapreduce.jobhi
import org.apache.hadoop.mapreduce.jobhistory.TaskFailedEvent;
import org.apache.hadoop.mapreduce.jobhistory.TaskFinishedEvent;
import org.apache.hadoop.mapreduce.jobhistory.TaskStartedEvent;
+import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent;
@@ -108,7 +110,8 @@ public abstract class TaskImpl implement
private long scheduledTime;
private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
-
+
+ protected boolean encryptedShuffle;
protected Credentials credentials;
protected Token<JobTokenIdentifier> jobToken;
@@ -274,6 +277,8 @@ public abstract class TaskImpl implement
this.jobToken = jobToken;
this.metrics = metrics;
this.appContext = appContext;
+ this.encryptedShuffle = conf.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY,
+ MRConfig.SHUFFLE_SSL_ENABLED_DEFAULT);
// See if this is from a previous generation.
if (completedTasksFromPreviousRun != null
@@ -637,9 +642,10 @@ public abstract class TaskImpl implement
TaskAttemptCompletionEvent tce = recordFactory
.newRecordInstance(TaskAttemptCompletionEvent.class);
tce.setEventId(-1);
- tce.setMapOutputServerAddress("http://"
- + attempt.getNodeHttpAddress().split(":")[0] + ":"
- + attempt.getShufflePort());
+ String scheme = (encryptedShuffle) ? "https://" : "http://";
+ tce.setMapOutputServerAddress(scheme
+ + attempt.getNodeHttpAddress().split(":")[0] + ":"
+ + attempt.getShufflePort());
tce.setStatus(status);
tce.setAttemptId(attempt.getID());
int runTime = 0;
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java Mon Jul 30 23:31:42 2012
@@ -67,13 +67,13 @@ class IndexCache {
if (info == null) {
info = readIndexFileToCache(fileName, mapId, expectedIndexOwner);
} else {
- while (isUnderConstruction(info)) {
- try {
- // In case the entry is ready after the above check but
- // before the following wait, we do timed wait.
- info.wait(200);
- } catch (InterruptedException e) {
- throw new IOException("Interrupted waiting for construction", e);
+ synchronized(info) {
+ while (isUnderConstruction(info)) {
+ try {
+ info.wait();
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted waiting for construction", e);
+ }
}
}
LOG.debug("IndexCache HIT: MapId " + mapId + " found");
@@ -101,13 +101,13 @@ class IndexCache {
IndexInformation info;
IndexInformation newInd = new IndexInformation();
if ((info = cache.putIfAbsent(mapId, newInd)) != null) {
- while (isUnderConstruction(info)) {
- try {
- // In case the entry is ready after the above check but
- // before the following wait, we do timed wait.
- info.wait(200);
- } catch (InterruptedException e) {
- throw new IOException("Interrupted waiting for construction", e);
+ synchronized(info) {
+ while (isUnderConstruction(info)) {
+ try {
+ info.wait();
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted waiting for construction", e);
+ }
}
}
LOG.debug("IndexCache HIT: MapId " + mapId + " found");
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java Mon Jul 30 23:31:42 2012
@@ -79,4 +79,9 @@ public interface MRConfig {
public static final int MAX_BLOCK_LOCATIONS_DEFAULT = 10;
public static final String MAX_BLOCK_LOCATIONS_KEY =
"mapreduce.job.max.split.locations";
+
+ public static final String SHUFFLE_SSL_ENABLED_KEY =
+ "mapreduce.shuffle.ssl.enabled";
+
+ public static final boolean SHUFFLE_SSL_ENABLED_DEFAULT = false;
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java Mon Jul 30 23:31:42 2012
@@ -21,15 +21,18 @@ import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
-import java.net.HttpURLConnection;
import java.net.URLConnection;
+import java.security.GeneralSecurityException;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import javax.crypto.SecretKey;
+import javax.net.ssl.HttpsURLConnection;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -42,9 +45,11 @@ import org.apache.hadoop.mapred.Counters
import org.apache.hadoop.mapred.IFileInputStream;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.security.SecureShuffleUtils;
+import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.mapreduce.task.reduce.MapOutput.Type;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
@@ -93,6 +98,9 @@ class Fetcher<K,V> extends Thread {
private volatile boolean stopped = false;
+ private static boolean sslShuffle;
+ private static SSLFactory sslFactory;
+
public Fetcher(JobConf job, TaskAttemptID reduceId,
ShuffleScheduler<K,V> scheduler, MergeManager<K,V> merger,
Reporter reporter, ShuffleClientMetrics metrics,
@@ -136,6 +144,20 @@ class Fetcher<K,V> extends Thread {
setName("fetcher#" + id);
setDaemon(true);
+
+ synchronized (Fetcher.class) {
+ sslShuffle = job.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY,
+ MRConfig.SHUFFLE_SSL_ENABLED_DEFAULT);
+ if (sslShuffle && sslFactory == null) {
+ sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, job);
+ try {
+ sslFactory.init();
+ } catch (Exception ex) {
+ sslFactory.destroy();
+ throw new RuntimeException(ex);
+ }
+ }
+ }
}
public void run() {
@@ -174,11 +196,24 @@ class Fetcher<K,V> extends Thread {
} catch (InterruptedException ie) {
LOG.warn("Got interrupt while joining " + getName(), ie);
}
+ if (sslFactory != null) {
+ sslFactory.destroy();
+ }
}
@VisibleForTesting
protected HttpURLConnection openConnection(URL url) throws IOException {
- return (HttpURLConnection)url.openConnection();
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ if (sslShuffle) {
+ HttpsURLConnection httpsConn = (HttpsURLConnection) conn;
+ try {
+ httpsConn.setSSLSocketFactory(sslFactory.createSSLSocketFactory());
+ } catch (GeneralSecurityException ex) {
+ throw new IOException(ex);
+ }
+ httpsConn.setHostnameVerifier(sslFactory.getHostnameVerifier());
+ }
+ return conn;
}
/**
@@ -187,6 +222,7 @@ class Fetcher<K,V> extends Thread {
* @param host {@link MapHost} from which we need to
* shuffle available map-outputs.
*/
+ @VisibleForTesting
protected void copyFromHost(MapHost host) throws IOException {
// Get completed maps on 'host'
List<TaskAttemptID> maps = scheduler.getMapsForHost(host);
@@ -198,10 +234,8 @@ class Fetcher<K,V> extends Thread {
}
if(LOG.isDebugEnabled()) {
- LOG.debug("Fetcher " + id + " going to fetch from " + host);
- for (TaskAttemptID tmp: maps) {
- LOG.debug(tmp);
- }
+ LOG.debug("Fetcher " + id + " going to fetch from " + host + " for: "
+ + maps);
}
// List of maps to be fetched yet
@@ -282,7 +316,8 @@ class Fetcher<K,V> extends Thread {
failedTasks = copyMapOutput(host, input, remaining);
}
- if(failedTasks != null) {
+ if(failedTasks != null && failedTasks.length > 0) {
+ LOG.warn("copyMapOutput failed for tasks "+Arrays.toString(failedTasks));
for(TaskAttemptID left: failedTasks) {
scheduler.copyFailed(left, host, true);
}
@@ -302,6 +337,8 @@ class Fetcher<K,V> extends Thread {
}
}
+ private static TaskAttemptID[] EMPTY_ATTEMPT_ID_ARRAY = new TaskAttemptID[0];
+
private TaskAttemptID[] copyMapOutput(MapHost host,
DataInputStream input,
Set<TaskAttemptID> remaining) {
@@ -335,8 +372,10 @@ class Fetcher<K,V> extends Thread {
return new TaskAttemptID[] {mapId};
}
- LOG.debug("header: " + mapId + ", len: " + compressedLength +
- ", decomp len: " + decompressedLength);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("header: " + mapId + ", len: " + compressedLength +
+ ", decomp len: " + decompressedLength);
+ }
// Get the location for the map output - either in-memory or on-disk
mapOutput = merger.reserve(mapId, decompressedLength, id);
@@ -344,7 +383,8 @@ class Fetcher<K,V> extends Thread {
// Check if we can shuffle *now* ...
if (mapOutput.getType() == Type.WAIT) {
LOG.info("fetcher#" + id + " - MergerManager returned Status.WAIT ...");
- return new TaskAttemptID[] {mapId};
+ //Not an error but wait to process data.
+ return EMPTY_ATTEMPT_ID_ARRAY;
}
// Go!
@@ -380,7 +420,7 @@ class Fetcher<K,V> extends Thread {
}
}
- LOG.info("Failed to shuffle output of " + mapId +
+ LOG.warn("Failed to shuffle output of " + mapId +
" from " + host.getHostName(), ioe);
// Inform the shuffle-scheduler
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml Mon Jul 30 23:31:42 2012
@@ -513,6 +513,21 @@
</property>
<property>
+ <name>mapreduce.shuffle.ssl.enabled</name>
+ <value>false</value>
+ <description>
+ Whether to use SSL for for the Shuffle HTTP endpoints.
+ </description>
+</property>
+
+<property>
+ <name>mapreduce.shuffle.ssl.file.buffer.size</name>
+ <value>65536</value>
+ <description>Buffer size for reading spills from file when using SSL.
+ </description>
+</property>
+
+<property>
<name>mapreduce.reduce.markreset.buffer.percent</name>
<value>0.0</value>
<description>The percentage of memory -relative to the maximum heap size- to
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1363593-1367364
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java Mon Jul 30 23:31:42 2012
@@ -29,6 +29,7 @@ import org.apache.hadoop.mapred.TestSequ
import org.apache.hadoop.mapred.TestTextInputFormat;
import org.apache.hadoop.mapred.ThreadedMapBenchmark;
import org.apache.hadoop.mapreduce.FailJob;
+import org.apache.hadoop.mapreduce.MiniHadoopClusterManager;
import org.apache.hadoop.mapreduce.SleepJob;
import org.apache.hadoop.util.ProgramDriver;
@@ -101,6 +102,8 @@ public class MapredTestDriver {
"Job History Log analyzer.");
pgd.addClass(SliveTest.class.getSimpleName(), SliveTest.class,
"HDFS Stress Test and Live Data Verification.");
+ pgd.addClass("minicluster", MiniHadoopClusterManager.class,
+ "Single process HDFS and MR cluster.");
} catch(Throwable e) {
e.printStackTrace();
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java Mon Jul 30 23:31:42 2012
@@ -55,7 +55,9 @@ import org.apache.hadoop.fs.LocalDirAllo
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputByteBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.security.SecureShuffleUtils;
+import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
import org.apache.hadoop.mapreduce.task.reduce.ShuffleHeader;
@@ -101,6 +103,8 @@ import org.jboss.netty.handler.codec.htt
import org.jboss.netty.handler.codec.http.HttpResponseEncoder;
import org.jboss.netty.handler.codec.http.HttpResponseStatus;
import org.jboss.netty.handler.codec.http.QueryStringDecoder;
+import org.jboss.netty.handler.ssl.SslHandler;
+import org.jboss.netty.handler.stream.ChunkedFile;
import org.jboss.netty.handler.stream.ChunkedWriteHandler;
import org.jboss.netty.util.CharsetUtil;
@@ -114,6 +118,8 @@ public class ShuffleHandler extends Abst
private int port;
private ChannelFactory selector;
private final ChannelGroup accepted = new DefaultChannelGroup();
+ private HttpPipelineFactory pipelineFact;
+ private int sslFileBufferSize;
public static final String MAPREDUCE_SHUFFLE_SERVICEID =
"mapreduce.shuffle";
@@ -126,6 +132,11 @@ public class ShuffleHandler extends Abst
public static final String SHUFFLE_PORT_CONFIG_KEY = "mapreduce.shuffle.port";
public static final int DEFAULT_SHUFFLE_PORT = 8080;
+ public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY =
+ "mapreduce.shuffle.ssl.file.buffer.size";
+
+ public static final int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE = 60 * 1024;
+
@Metrics(about="Shuffle output metrics", context="mapred")
static class ShuffleMetrics implements ChannelFutureListener {
@Metric("Shuffle output in bytes")
@@ -249,7 +260,11 @@ public class ShuffleHandler extends Abst
public synchronized void start() {
Configuration conf = getConfig();
ServerBootstrap bootstrap = new ServerBootstrap(selector);
- HttpPipelineFactory pipelineFact = new HttpPipelineFactory(conf);
+ try {
+ pipelineFact = new HttpPipelineFactory(conf);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
bootstrap.setPipelineFactory(pipelineFact);
port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT);
Channel ch = bootstrap.bind(new InetSocketAddress(port));
@@ -259,6 +274,9 @@ public class ShuffleHandler extends Abst
pipelineFact.SHUFFLE.setPort(port);
LOG.info(getName() + " listening on port " + port);
super.start();
+
+ sslFileBufferSize = conf.getInt(SUFFLE_SSL_FILE_BUFFER_SIZE_KEY,
+ DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE);
}
@Override
@@ -266,6 +284,7 @@ public class ShuffleHandler extends Abst
accepted.close().awaitUninterruptibly(10, TimeUnit.SECONDS);
ServerBootstrap bootstrap = new ServerBootstrap(selector);
bootstrap.releaseExternalResources();
+ pipelineFact.destroy();
super.stop();
}
@@ -283,22 +302,38 @@ public class ShuffleHandler extends Abst
class HttpPipelineFactory implements ChannelPipelineFactory {
final Shuffle SHUFFLE;
+ private SSLFactory sslFactory;
- public HttpPipelineFactory(Configuration conf) {
+ public HttpPipelineFactory(Configuration conf) throws Exception {
SHUFFLE = new Shuffle(conf);
+ if (conf.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY,
+ MRConfig.SHUFFLE_SSL_ENABLED_DEFAULT)) {
+ sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf);
+ sslFactory.init();
+ }
+ }
+
+ public void destroy() {
+ if (sslFactory != null) {
+ sslFactory.destroy();
+ }
}
@Override
public ChannelPipeline getPipeline() throws Exception {
- return Channels.pipeline(
- new HttpRequestDecoder(),
- new HttpChunkAggregator(1 << 16),
- new HttpResponseEncoder(),
- new ChunkedWriteHandler(),
- SHUFFLE);
- // TODO factor security manager into pipeline
- // TODO factor out encode/decode to permit binary shuffle
- // TODO factor out decode of index to permit alt. models
+ ChannelPipeline pipeline = Channels.pipeline();
+ if (sslFactory != null) {
+ pipeline.addLast("ssl", new SslHandler(sslFactory.createSSLEngine()));
+ }
+ pipeline.addLast("decoder", new HttpRequestDecoder());
+ pipeline.addLast("aggregator", new HttpChunkAggregator(1 << 16));
+ pipeline.addLast("encoder", new HttpResponseEncoder());
+ pipeline.addLast("chunking", new ChunkedWriteHandler());
+ pipeline.addLast("shuffle", SHUFFLE);
+ return pipeline;
+ // TODO factor security manager into pipeline
+ // TODO factor out encode/decode to permit binary shuffle
+ // TODO factor out decode of index to permit alt. models
}
}
@@ -483,17 +518,25 @@ public class ShuffleHandler extends Abst
LOG.info(spillfile + " not found");
return null;
}
- final FileRegion partition = new DefaultFileRegion(
- spill.getChannel(), info.startOffset, info.partLength);
- ChannelFuture writeFuture = ch.write(partition);
- writeFuture.addListener(new ChannelFutureListener() {
- // TODO error handling; distinguish IO/connection failures,
- // attribute to appropriate spill output
- @Override
- public void operationComplete(ChannelFuture future) {
- partition.releaseExternalResources();
- }
- });
+ ChannelFuture writeFuture;
+ if (ch.getPipeline().get(SslHandler.class) == null) {
+ final FileRegion partition = new DefaultFileRegion(
+ spill.getChannel(), info.startOffset, info.partLength);
+ writeFuture = ch.write(partition);
+ writeFuture.addListener(new ChannelFutureListener() {
+ // TODO error handling; distinguish IO/connection failures,
+ // attribute to appropriate spill output
+ @Override
+ public void operationComplete(ChannelFuture future) {
+ partition.releaseExternalResources();
+ }
+ });
+ } else {
+ // HTTPS cannot be done with zero copy.
+ writeFuture = ch.write(new ChunkedFile(spill, info.startOffset,
+ info.partLength,
+ sslFileBufferSize));
+ }
metrics.shuffleConnections.incr();
metrics.shuffleOutputBytes.incr(info.partLength); // optimistic
return writeFuture;
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java Mon Jul 30 23:31:42 2012
@@ -290,7 +290,10 @@ public class ApplicationMaster {
Map<String, String> envs = System.getenv();
appAttemptID = Records.newRecord(ApplicationAttemptId.class);
- if (!envs.containsKey(ApplicationConstants.AM_CONTAINER_ID_ENV)) {
+ if (envs.containsKey(ApplicationConstants.AM_APP_ATTEMPT_ID_ENV)) {
+ appAttemptID = ConverterUtils.toApplicationAttemptId(envs
+ .get(ApplicationConstants.AM_APP_ATTEMPT_ID_ENV));
+ } else if (!envs.containsKey(ApplicationConstants.AM_CONTAINER_ID_ENV)) {
if (cliParser.hasOption("app_attempt_id")) {
String appIdStr = cliParser.getOptionValue("app_attempt_id", "");
appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Mon Jul 30 23:31:42 2012
@@ -0,0 +1 @@
+target
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml Mon Jul 30 23:31:42 2012
@@ -30,6 +30,7 @@
<modules>
<module>hadoop-yarn-applications-distributedshell</module>
+ <module>hadoop-yarn-applications-unmanaged-am-launcher</module>
</modules>
<profiles>
<profile>
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Mon Jul 30 23:31:42 2012
@@ -241,7 +241,7 @@
<!-- Node Manager Configs -->
<property>
- <description>address of node manager IPC.</description>
+ <description>The address of the container manager in the NM.</description>
<name>yarn.nodemanager.address</name>
<value>0.0.0.0:0</value>
</property>
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java Mon Jul 30 23:31:42 2012
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
+import java.io.File;
import java.util.Iterator;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -30,6 +31,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEventType;
/**
* A collection of {@link LocalizedResource}s all of same
@@ -67,6 +69,12 @@ class LocalResourcesTrackerImpl implemen
switch (event.getType()) {
case REQUEST:
case LOCALIZED:
+ if (rsrc != null && (!isResourcePresent(rsrc))) {
+ LOG.info("Resource " + rsrc.getLocalPath()
+ + " is missing, localizing it again");
+ localrsrc.remove(req);
+ rsrc = null;
+ }
if (null == rsrc) {
rsrc = new LocalizedResource(req, dispatcher);
localrsrc.put(req, rsrc);
@@ -82,6 +90,24 @@ class LocalResourcesTrackerImpl implemen
rsrc.handle(event);
}
+ /**
+ * This module checks if the resource which was localized is already present
+ * or not
+ *
+ * @param rsrc
+ * @return true/false based on resource is present or not
+ */
+ public boolean isResourcePresent(LocalizedResource rsrc) {
+ boolean ret = true;
+ if (rsrc.getState() == ResourceState.LOCALIZED) {
+ File file = new File(rsrc.getLocalPath().toUri().getRawPath().toString());
+ if (!file.exists()) {
+ ret = false;
+ }
+ }
+ return ret;
+ }
+
@Override
public boolean contains(LocalResourceRequest resource) {
return localrsrc.containsKey(resource);
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java Mon Jul 30 23:31:42 2012
@@ -5,6 +5,8 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import java.io.File;
+import java.io.IOException;
import java.util.Iterator;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -30,6 +32,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.Test;
+import org.mortbay.log.Log;
public class TestLocalResourcesTrackerImpl {
@@ -131,6 +134,86 @@ public class TestLocalResourcesTrackerIm
}
}
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testConsistency() {
+ String user = "testuser";
+ DrainDispatcher dispatcher = null;
+ try {
+ dispatcher = createDispatcher(new Configuration());
+ EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
+ EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
+ dispatcher.register(LocalizerEventType.class, localizerEventHandler);
+ dispatcher.register(ContainerEventType.class, containerEventHandler);
+
+ ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
+ LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
+ LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1,
+ LocalResourceVisibility.PUBLIC);
+ LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
+ ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
+ localrsrc.put(req1, lr1);
+ LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
+ dispatcher, localrsrc);
+
+ ResourceEvent req11Event = new ResourceRequestEvent(req1,
+ LocalResourceVisibility.PUBLIC, lc1);
+
+ ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);
+
+ // Localize R1 for C1
+ tracker.handle(req11Event);
+
+ dispatcher.await();
+
+ // Verify refCount for R1 is 1
+ Assert.assertEquals(1, lr1.getRefCount());
+
+ dispatcher.await();
+ verifyTrackedResourceCount(tracker, 1);
+
+ // Localize resource1
+ ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path(
+ "file:///tmp/r1"), 1);
+ lr1.handle(rle);
+ Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
+ Assert.assertTrue(createdummylocalizefile(new Path("file:///tmp/r1")));
+ LocalizedResource rsrcbefore = tracker.iterator().next();
+ File resFile = new File(lr1.getLocalPath().toUri().getRawPath()
+ .toString());
+ Assert.assertTrue(resFile.exists());
+ Assert.assertTrue(resFile.delete());
+
+ // Localize R1 for C1
+ tracker.handle(req11Event);
+
+ dispatcher.await();
+ lr1.handle(rle);
+ Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
+ LocalizedResource rsrcafter = tracker.iterator().next();
+ if (rsrcbefore == rsrcafter) {
+ Assert.fail("Localized resource should not be equal");
+ }
+ // Release resource1
+ tracker.handle(rel11Event);
+ } finally {
+ if (dispatcher != null) {
+ dispatcher.stop();
+ }
+ }
+ }
+
+ private boolean createdummylocalizefile(Path path) {
+ boolean ret = false;
+ File file = new File(path.toUri().getRawPath().toString());
+ try {
+ ret = file.createNewFile();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return ret;
+ }
+
private void verifyTrackedResourceCount(LocalResourcesTracker tracker,
int expected) {
int count = 0;
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java Mon Jul 30 23:31:42 2012
@@ -47,12 +47,20 @@ public class CapacitySchedulerConfigurat
public static final String DOT = ".";
@Private
+ public static final String MAXIMUM_APPLICATIONS_SUFFIX =
+ "maximum-applications";
+
+ @Private
public static final String MAXIMUM_SYSTEM_APPLICATIONS =
- PREFIX + "maximum-applications";
+ PREFIX + MAXIMUM_APPLICATIONS_SUFFIX;
+
+ @Private
+ public static final String MAXIMUM_AM_RESOURCE_SUFFIX =
+ "maximum-am-resource-percent";
@Private
public static final String MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT =
- PREFIX + "maximum-am-resource-percent";
+ PREFIX + MAXIMUM_AM_RESOURCE_SUFFIX;
@Private
public static final String QUEUES = "queues";
@@ -131,6 +139,30 @@ public class CapacitySchedulerConfigurat
return getFloat(MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT);
}
+
+
+ /**
+ * Get the maximum applications per queue setting.
+ * @param queue name of the queue
+ * @return setting specified or -1 if not set
+ */
+ public int getMaximumApplicationsPerQueue(String queue) {
+ int maxApplicationsPerQueue =
+ getInt(getQueuePrefix(queue) + MAXIMUM_APPLICATIONS_SUFFIX,
+ (int)UNDEFINED);
+ return maxApplicationsPerQueue;
+ }
+
+ /**
+ * Get the maximum am resource percent per queue setting.
+ * @param queue name of the queue
+ * @return per queue setting or defaults to the global am-resource-percent
+ * setting if per queue setting not present
+ */
+ public float getMaximumApplicationMasterResourcePerQueuePercent(String queue) {
+ return getFloat(getQueuePrefix(queue) + MAXIMUM_AM_RESOURCE_SUFFIX,
+ getMaximumApplicationMasterResourcePercent());
+ }
public float getCapacity(String queue) {
float capacity = getFloat(getQueuePrefix(queue) + CAPACITY, UNDEFINED);
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Mon Jul 30 23:31:42 2012
@@ -85,7 +85,7 @@ public class LeafQueue implements CSQueu
private int maxApplications;
private int maxApplicationsPerUser;
- private float maxAMResourcePercent;
+ private float maxAMResourcePerQueuePercent;
private int maxActiveApplications; // Based on absolute max capacity
private int maxActiveAppsUsingAbsCap; // Based on absolute capacity
private int maxActiveApplicationsPerUser;
@@ -156,21 +156,25 @@ public class LeafQueue implements CSQueu
float userLimitFactor =
cs.getConfiguration().getUserLimitFactor(getQueuePath());
- int maxSystemJobs = cs.getConfiguration().getMaximumSystemApplications();
- int maxApplications = (int)(maxSystemJobs * absoluteCapacity);
- int maxApplicationsPerUser =
+ int maxApplications = cs.getConfiguration().getMaximumApplicationsPerQueue(getQueuePath());
+ if (maxApplications < 0) {
+ int maxSystemApps = cs.getConfiguration().getMaximumSystemApplications();
+ maxApplications = (int)(maxSystemApps * absoluteCapacity);
+ }
+ maxApplicationsPerUser =
(int)(maxApplications * (userLimit / 100.0f) * userLimitFactor);
- this.maxAMResourcePercent =
- cs.getConfiguration().getMaximumApplicationMasterResourcePercent();
+ this.maxAMResourcePerQueuePercent =
+ cs.getConfiguration().
+ getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
int maxActiveApplications =
CSQueueUtils.computeMaxActiveApplications(
cs.getClusterResources(), this.minimumAllocation,
- maxAMResourcePercent, absoluteMaxCapacity);
+ maxAMResourcePerQueuePercent, absoluteMaxCapacity);
this.maxActiveAppsUsingAbsCap =
CSQueueUtils.computeMaxActiveApplications(
cs.getClusterResources(), this.minimumAllocation,
- maxAMResourcePercent, absoluteCapacity);
+ maxAMResourcePerQueuePercent, absoluteCapacity);
int maxActiveApplicationsPerUser =
CSQueueUtils.computeMaxActiveApplicationsPerUser(maxActiveAppsUsingAbsCap, userLimit,
userLimitFactor);
@@ -265,15 +269,16 @@ public class LeafQueue implements CSQueu
"userLimitFactor = " + userLimitFactor +
" [= configuredUserLimitFactor ]" + "\n" +
"maxApplications = " + maxApplications +
- " [= (int)(configuredMaximumSystemApplications * absoluteCapacity) ]" +
+ " [= configuredMaximumSystemApplicationsPerQueue or" +
+ " (int)(configuredMaximumSystemApplications * absoluteCapacity)]" +
"\n" +
"maxApplicationsPerUser = " + maxApplicationsPerUser +
" [= (int)(maxApplications * (userLimit / 100.0f) * " +
"userLimitFactor) ]" + "\n" +
"maxActiveApplications = " + maxActiveApplications +
" [= max(" +
- "(int)ceil((clusterResourceMemory / minimumAllocation) *" +
- "maxAMResourcePercent * absoluteMaxCapacity)," +
+ "(int)ceil((clusterResourceMemory / minimumAllocation) * " +
+ "maxAMResourcePerQueuePercent * absoluteMaxCapacity)," +
"1) ]" + "\n" +
"maxActiveAppsUsingAbsCap = " + maxActiveAppsUsingAbsCap +
" [= max(" +
@@ -290,7 +295,7 @@ public class LeafQueue implements CSQueu
"(clusterResourceMemory * absoluteCapacity)]" + "\n" +
"absoluteUsedCapacity = " + absoluteUsedCapacity +
" [= usedResourcesMemory / clusterResourceMemory]" + "\n" +
- "maxAMResourcePercent = " + maxAMResourcePercent +
+ "maxAMResourcePerQueuePercent = " + maxAMResourcePerQueuePercent +
" [= configuredMaximumAMResourcePercent ]" + "\n" +
"minimumAllocationFactor = " + minimumAllocationFactor +
" [= (float)(maximumAllocationMemory - minimumAllocationMemory) / " +
@@ -1387,11 +1392,11 @@ public class LeafQueue implements CSQueu
maxActiveApplications =
CSQueueUtils.computeMaxActiveApplications(
clusterResource, minimumAllocation,
- maxAMResourcePercent, absoluteMaxCapacity);
+ maxAMResourcePerQueuePercent, absoluteMaxCapacity);
maxActiveAppsUsingAbsCap =
- CSQueueUtils.computeMaxActiveApplications(
- clusterResource, minimumAllocation,
- maxAMResourcePercent, absoluteCapacity);
+ CSQueueUtils.computeMaxActiveApplications(
+ clusterResource, minimumAllocation,
+ maxAMResourcePerQueuePercent, absoluteCapacity);
maxActiveApplicationsPerUser =
CSQueueUtils.computeMaxActiveApplicationsPerUser(
maxActiveAppsUsingAbsCap, userLimit, userLimitFactor);
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java Mon Jul 30 23:31:42 2012
@@ -158,7 +158,9 @@ public class TestApplicationLimits {
int expectedMaxActiveApps =
Math.max(1,
(int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) *
- csConf.getMaximumApplicationMasterResourcePercent() *
+ csConf.
+ getMaximumApplicationMasterResourcePerQueuePercent(
+ queue.getQueuePath()) *
queue.getAbsoluteMaximumCapacity()));
assertEquals(expectedMaxActiveApps,
queue.getMaximumActiveApplications());
@@ -183,7 +185,9 @@ public class TestApplicationLimits {
expectedMaxActiveApps =
Math.max(1,
(int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) *
- csConf.getMaximumApplicationMasterResourcePercent() *
+ csConf.
+ getMaximumApplicationMasterResourcePerQueuePercent(
+ queue.getQueuePath()) *
queue.getAbsoluteMaximumCapacity()));
assertEquals(expectedMaxActiveApps,
queue.getMaximumActiveApplications());
@@ -200,6 +204,72 @@ public class TestApplicationLimits {
(int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()),
queue.getMetrics().getAvailableMB()
);
+
+ // should return -1 if per queue setting not set
+ assertEquals((int)csConf.UNDEFINED, csConf.getMaximumApplicationsPerQueue(queue.getQueuePath()));
+ int expectedMaxApps = (int)(csConf.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS *
+ queue.getAbsoluteCapacity());
+ assertEquals(expectedMaxApps, queue.getMaxApplications());
+
+ int expectedMaxAppsPerUser = (int)(expectedMaxApps *
+ (queue.getUserLimit()/100.0f) * queue.getUserLimitFactor());
+ assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser());
+
+ // should default to global setting if per queue setting not set
+ assertEquals((long) csConf.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT,
+ (long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(queue.getQueuePath()));
+
+ // Change the per-queue max AM resources percentage.
+ csConf.setFloat(
+ "yarn.scheduler.capacity." +
+ queue.getQueuePath() +
+ ".maximum-am-resource-percent",
+ 0.5f);
+ // Re-create queues to get new configs.
+ queues = new HashMap<String, CSQueue>();
+ root =
+ CapacityScheduler.parseQueue(csContext, csConf, null, "root",
+ queues, queues,
+ CapacityScheduler.queueComparator,
+ CapacityScheduler.applicationComparator,
+ TestUtils.spyHook);
+ clusterResource = Resources.createResource(100 * 16 * GB);
+
+ queue = (LeafQueue)queues.get(A);
+ expectedMaxActiveApps =
+ Math.max(1,
+ (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) *
+ csConf.
+ getMaximumApplicationMasterResourcePerQueuePercent(
+ queue.getQueuePath()) *
+ queue.getAbsoluteMaximumCapacity()));
+
+ assertEquals((long) 0.5,
+ (long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(queue.getQueuePath()));
+ assertEquals(expectedMaxActiveApps,
+ queue.getMaximumActiveApplications());
+
+ // Change the per-queue max applications.
+ csConf.setInt(
+ "yarn.scheduler.capacity." +
+ queue.getQueuePath() +
+ ".maximum-applications", 9999);
+ // Re-create queues to get new configs.
+ queues = new HashMap<String, CSQueue>();
+ root =
+ CapacityScheduler.parseQueue(csContext, csConf, null, "root",
+ queues, queues,
+ CapacityScheduler.queueComparator,
+ CapacityScheduler.applicationComparator,
+ TestUtils.spyHook);
+
+ queue = (LeafQueue)queues.get(A);
+ assertEquals(9999, (int)csConf.getMaximumApplicationsPerQueue(queue.getQueuePath()));
+ assertEquals(9999, queue.getMaxApplications());
+
+ expectedMaxAppsPerUser = (int)(9999 *
+ (queue.getUserLimit()/100.0f) * queue.getUserLimitFactor());
+ assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser());
}
@Test
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm Mon Jul 30 23:31:42 2012
@@ -240,17 +240,24 @@ Hadoop MapReduce Next Generation - Capac
*--------------------------------------+--------------------------------------+
|| Property || Description |
*--------------------------------------+--------------------------------------+
-| <<<yarn.scheduler.capacity.maximum-applications>>> | |
+| <<<yarn.scheduler.capacity.maximum-applications>>> / |
+| <<<yarn.scheduler.capacity.<queue-path>.maximum-applications>>> | |
| | Maximum number of applications in the system which can be concurrently |
| | active both running and pending. Limits on each queue are directly |
| | proportional to their queue capacities and user limits. This is a
| | hard limit and any applications submitted when this limit is reached will |
-| | be rejected. Default is 10000.|
+| | be rejected. Default is 10000. This can be set for all queues with |
+| | <<<yarn.scheduler.capacity.maximum-applications>>> and can also be overridden on a |
+| | per queue basis by setting <<<yarn.scheduler.capacity.<queue-path>.maximum-applications>>>. |
*--------------------------------------+--------------------------------------+
-| yarn.scheduler.capacity.maximum-am-resource-percent | |
+| <<<yarn.scheduler.capacity.maximum-am-resource-percent>>> / |
+| <<<yarn.scheduler.capacity.<queue-path>.maximum-am-resource-percent>>> | |
| | Maximum percent of resources in the cluster which can be used to run |
-| | application masters - controls number of concurrent running applications. |
-| | Specified as a float - ie 0.5 = 50%. Default is 10%. |
+| | application masters - controls number of concurrent active applications. Limits on each |
+| | queue are directly proportional to their queue capacities and user limits. |
+| | Specified as a float - ie 0.5 = 50%. Default is 10%. This can be set for all queues with |
+| | <<<yarn.scheduler.capacity.maximum-am-resource-percent>>> and can also be overridden on a |
+| | per queue basis by setting <<<yarn.scheduler.capacity.<queue-path>.maximum-am-resource-percent>>> |
*--------------------------------------+--------------------------------------+
* Queue Administration & Permissions
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm Mon Jul 30 23:31:42 2012
@@ -49,4 +49,7 @@ MapReduce NextGen aka YARN aka MRv2
* {{{./WebApplicationProxy.html}Web Application Proxy}}
+ * {{{./CLIMiniCluster.html}CLI MiniCluster}}
+
+ * {{{./EncryptedShuffle.html}Encrypted Shuffle}}
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/c++/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/c++:r1363593-1367364
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:r1363593-1367364
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/block_forensics/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:r1363593-1367364
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/build-contrib.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:r1363593-1367364
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/build.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:r1363593-1367364
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/data_join/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:r1363593-1367364
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/eclipse-plugin/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:r1363593-1367364
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/index/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:r1363593-1367364
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/vaidya/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:r1363593-1367364
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/examples/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/examples:r1363593-1367364
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/java:r1363593-1367364
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:r1363593-1367364
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:r1363593-1367364
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:r1363593-1367364
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:r1363593-1367364
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/webapps/job/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:r1363593-1367364