You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sz...@apache.org on 2012/01/17 03:28:52 UTC
svn commit: r1232275 - in
/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project: ./ bin/
conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/
hadoop-mapreduce-client/hadoop-mapreduce-clie...
Author: szetszwo
Date: Tue Jan 17 02:28:49 2012
New Revision: 1232275
URL: http://svn.apache.org/viewvc?rev=1232275&view=rev
Log:
Merge r1231828 through r1232270 from 0.23.
Added:
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSAssignment.java
- copied unchanged from r1232270, hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSAssignment.java
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/.gitignore (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/bin/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/conf/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-mapreduce-examples/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/Federation.apt.vm
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/c++/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/contrib/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/contrib/block_forensics/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/contrib/build-contrib.xml (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/contrib/build.xml (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/contrib/data_join/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/contrib/eclipse-plugin/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/contrib/index/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/contrib/vaidya/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/examples/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/java/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/test/mapred/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/ (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java (props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/webapps/job/ (props changed)
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,3 +1,3 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1164771,1166402,1167001,1167318,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1210208,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1214066,1214106,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1230886,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred:713112
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/.gitignore
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1 +1,2 @@
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/.gitignore:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/.gitignore:1182205,1204177,1205697,1225192,1225456,1225489,1225591,1226239,1227165,1229347,1230398,1231569,1231572,1231627,1231640
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/CHANGES.txt?rev=1232275&r1=1232274&r2=1232275&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/CHANGES.txt Tue Jan 17 02:28:49 2012
@@ -131,6 +131,10 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3553. Add support for data returned when exceptions thrown from web
service apis to be in either xml or in JSON. (Thomas Graves via mahadev)
+ MAPREDUCE-3641. Making CapacityScheduler more conservative so as to
+ assign only one off-switch container in a single scheduling
+ iteration. (Arun C Murthy via vinodkv)
+
OPTIMIZATIONS
MAPREDUCE-3567. Extraneous JobConf objects in AM heap. (Vinod Kumar
@@ -452,6 +456,15 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3404. Corrected MR AM to honor speculative configuration and enable
speculating either maps or reduces. (Eric Payne via vinodkv)
+ MAPREDUCE-3664. Federation Documentation has incorrect configuration example.
+ (Brandon Li via jitendra)
+
+ MAPREDUCE-3649. Job End notification gives an error on calling back.
+ (Ravi Prakash via mahadev)
+
+ MAPREDUCE-3657. State machine visualize build fails. (Jason Lowe
+ via mahadev)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:1161777,1161781-1161782,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1164771,1166402,1167001,1167318,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205519,1205697,1206786,1206830,1207694,1207714,1208153,1208313,1209231,1210208,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1214066,1214106,1215528,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1230886,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/CHANGES.txt:713112
/hadoop/mapreduce/branches/HDFS-641/CHANGES.txt:817878-835964
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/bin/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,3 +1,3 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/bin:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/bin:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/bin:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1164771,1166402,1167001,1167318,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/bin:713112
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/conf:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/conf:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/conf:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1164771,1166402,1167318,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/conf:713112
/hadoop/core/trunk/conf:784664-785643
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java?rev=1232275&r1=1232274&r2=1232275&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java Tue Jan 17 02:28:49 2012
@@ -19,12 +19,11 @@
package org.apache.hadoop.mapreduce.v2.app;
import java.io.IOException;
-import java.io.InputStream;
+import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLConnection;
import java.net.Proxy;
+import java.net.URL;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -40,7 +39,8 @@ import org.mortbay.log.Log;
* User can specify number of retry attempts and a time interval at which to
* attempt retries</li><li>
* Cluster administrators can set final parameters to set maximum number of
- * tries (0 would disable job end notification) and max time interval</li><li>
+ * tries (0 would disable job end notification) and max time interval and a
+ * proxy if needed</li><li>
* The URL may contain sentinels which will be replaced by jobId and jobStatus
* (eg. SUCCEEDED/KILLED/FAILED) </li> </ul>
* </p>
@@ -59,8 +59,8 @@ public class JobEndNotifier implements C
/**
* Parse the URL that needs to be notified of the end of the job, along
- * with the number of retries in case of failure and the amount of time to
- * wait between retries
+ * with the number of retries in case of failure, the amount of time to
+ * wait between retries and proxy settings
* @param conf the configuration
*/
public void setConf(Configuration conf) {
@@ -119,15 +119,19 @@ public class JobEndNotifier implements C
boolean success = false;
try {
Log.info("Job end notification trying " + urlToNotify);
- URLConnection conn = urlToNotify.openConnection(proxyToUse);
+ HttpURLConnection conn = (HttpURLConnection) urlToNotify.openConnection();
conn.setConnectTimeout(5*1000);
conn.setReadTimeout(5*1000);
conn.setAllowUserInteraction(false);
- InputStream is = conn.getInputStream();
- conn.getContent();
- is.close();
- success = true;
- Log.info("Job end notification to " + urlToNotify + " succeeded");
+ if(conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
+ Log.warn("Job end notification to " + urlToNotify +" failed with code: "
+ + conn.getResponseCode() + " and message \"" + conn.getResponseMessage()
+ +"\"");
+ }
+ else {
+ success = true;
+ Log.info("Job end notification to " + urlToNotify + " succeeded");
+ }
} catch(IOException ioe) {
Log.warn("Job end notification to " + urlToNotify + " failed", ioe);
}
@@ -135,8 +139,8 @@ public class JobEndNotifier implements C
}
/**
- * Notify a server of the completion of a submitted job. The server must have
- * configured MRConfig.JOB_END_NOTIFICATION_URLS
+ * Notify a server of the completion of a submitted job. The user must have
+ * configured MRJobConfig.MR_JOB_END_NOTIFICATION_URL
* @param jobReport JobReport used to read JobId and JobStatus
* @throws InterruptedException
*/
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:1164771,1167318,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1214066,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/src/java/mapred-default.xml:713112
/hadoop/core/trunk/src/mapred/mapred-default.xml:776175-785643
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-mapreduce-examples/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,3 +1,3 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-examples:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1164771,1166402,1167001,1167318,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213630,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/hadoop-mapreduce-examples:713112
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml?rev=1232275&r1=1232274&r2=1232275&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml Tue Jan 17 02:28:49 2012
@@ -109,6 +109,7 @@
</goals>
<configuration>
<mainClass>org.apache.hadoop.yarn.util.VisualizeStateMachine</mainClass>
+ <classpathScope>compile</classpathScope>
<arguments>
<argument>NodeManager</argument>
<argument>org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl,
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml?rev=1232275&r1=1232274&r2=1232275&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml Tue Jan 17 02:28:49 2012
@@ -147,6 +147,7 @@
</goals>
<configuration>
<mainClass>org.apache.hadoop.yarn.util.VisualizeStateMachine</mainClass>
+ <classpathScope>compile</classpathScope>
<arguments>
<argument>ResourceManager</argument>
<argument>org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl,
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java?rev=1232275&r1=1232274&r2=1232275&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java Tue Jan 17 02:28:49 2012
@@ -26,8 +26,26 @@ import org.apache.hadoop.yarn.util.Recor
@Private
@Evolving
public class Resources {
+
// Java doesn't have const :(
- private static final Resource NONE = createResource(0);
+ private static final Resource NONE = new Resource() {
+
+ @Override
+ public int getMemory() {
+ return 0;
+ }
+
+ @Override
+ public void setMemory(int memory) {
+ throw new RuntimeException("NONE cannot be modified!");
+ }
+
+ @Override
+ public int compareTo(Resource o) {
+ return (0 - o.getMemory());
+ }
+
+ };
public static Resource createResource(int memory) {
Resource resource = Records.newRecord(Resource.class);
@@ -36,7 +54,6 @@ public class Resources {
}
public static Resource none() {
- assert NONE.getMemory() == 0 : "NONE should be empty";
return NONE;
}
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java?rev=1232275&r1=1232274&r2=1232275&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java Tue Jan 17 02:28:49 2012
@@ -155,9 +155,10 @@ extends org.apache.hadoop.yarn.server.re
* Assign containers to applications in the queue or it's children (if any).
* @param clusterResource the resource of the cluster.
* @param node node on which resources are available
- * @return the resource that is being assigned.
+ * @return the assignment
*/
- public Resource assignContainers(Resource clusterResource, SchedulerNode node);
+ public CSAssignment assignContainers(
+ Resource clusterResource, SchedulerNode node);
/**
* A container assigned to the queue has completed.
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1232275&r1=1232274&r2=1232275&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Tue Jan 17 02:28:49 2012
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
-import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
@@ -35,7 +34,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
@@ -703,8 +701,11 @@ public class LeafQueue implements CSQueu
return applicationsMap.get(applicationAttemptId);
}
+ private static final CSAssignment NULL_ASSIGNMENT =
+ new CSAssignment(Resources.createResource(0), NodeType.NODE_LOCAL);
+
@Override
- public synchronized Resource
+ public synchronized CSAssignment
assignContainers(Resource clusterResource, SchedulerNode node) {
if(LOG.isDebugEnabled()) {
@@ -717,8 +718,11 @@ public class LeafQueue implements CSQueu
if (reservedContainer != null) {
SchedulerApp application =
getApplication(reservedContainer.getApplicationAttemptId());
- return assignReservedContainer(application, node, reservedContainer,
- clusterResource);
+ return new CSAssignment(
+ assignReservedContainer(application, node, reservedContainer,
+ clusterResource),
+ NodeType.NODE_LOCAL); // Don't care about locality constraints
+ // for reserved containers
}
// Try to assign containers to applications in order
@@ -746,7 +750,7 @@ public class LeafQueue implements CSQueu
// Are we going over limits by allocating to this application?
// Maximum Capacity of the queue
if (!assignToQueue(clusterResource, required)) {
- return Resources.none();
+ return NULL_ASSIGNMENT;
}
// User limits
@@ -760,24 +764,23 @@ public class LeafQueue implements CSQueu
application.addSchedulingOpportunity(priority);
// Try to schedule
- Resource assigned =
+ CSAssignment assignment =
assignContainersOnNode(clusterResource, node, application, priority,
null);
-
+
+ Resource assigned = assignment.getResource();
+
// Did we schedule or reserve a container?
if (Resources.greaterThan(assigned, Resources.none())) {
- Resource assignedResource =
- application.getResourceRequest(priority, RMNode.ANY).getCapability();
// Book-keeping
- allocateResource(clusterResource,
- application, assignedResource);
+ allocateResource(clusterResource, application, assigned);
// Reset scheduling opportunities
application.resetSchedulingOpportunities(priority);
// Done
- return assignedResource;
+ return assignment;
} else {
// Do not assign out of order w.r.t priorities
break;
@@ -792,7 +795,7 @@ public class LeafQueue implements CSQueu
application.showRequests();
}
- return Resources.none();
+ return NULL_ASSIGNMENT;
}
@@ -809,11 +812,12 @@ public class LeafQueue implements CSQueu
container.getId(),
SchedulerUtils.UNRESERVED_CONTAINER),
RMContainerEventType.RELEASED);
- return container.getResource();
+ return container.getResource(); // Ugh, return resource to force re-sort
}
// Try to assign if we have sufficient resources
- assignContainersOnNode(clusterResource, node, application, priority, rmContainer);
+ assignContainersOnNode(clusterResource, node, application, priority,
+ rmContainer);
// Doesn't matter... since it's already charged for at time of reservation
// "re-reservation" is *free*
@@ -966,7 +970,7 @@ public class LeafQueue implements CSQueu
return (((starvation + requiredContainers) - reservedContainers) > 0);
}
- private Resource assignContainersOnNode(Resource clusterResource,
+ private CSAssignment assignContainersOnNode(Resource clusterResource,
SchedulerNode node, SchedulerApp application,
Priority priority, RMContainer reservedContainer) {
@@ -977,7 +981,7 @@ public class LeafQueue implements CSQueu
assignNodeLocalContainers(clusterResource, node, application, priority,
reservedContainer);
if (Resources.greaterThan(assigned, Resources.none())) {
- return assigned;
+ return new CSAssignment(assigned, NodeType.NODE_LOCAL);
}
// Rack-local
@@ -985,12 +989,14 @@ public class LeafQueue implements CSQueu
assignRackLocalContainers(clusterResource, node, application, priority,
reservedContainer);
if (Resources.greaterThan(assigned, Resources.none())) {
- return assigned;
+ return new CSAssignment(assigned, NodeType.RACK_LOCAL);
}
// Off-switch
- return assignOffSwitchContainers(clusterResource, node, application,
- priority, reservedContainer);
+ return new CSAssignment(
+ assignOffSwitchContainers(clusterResource, node, application,
+ priority, reservedContainer),
+ NodeType.OFF_SWITCH);
}
private Resource assignNodeLocalContainers(Resource clusterResource,
@@ -1272,7 +1278,7 @@ public class LeafQueue implements CSQueu
metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
LOG.info(getQueueName() +
" used=" + usedResources + " numContainers=" + numContainers +
- " user=" + userName + " resources=" + user.getConsumedResources());
+ " user=" + userName + " user-resources=" + user.getConsumedResources());
}
synchronized void releaseResource(Resource clusterResource,
@@ -1290,7 +1296,7 @@ public class LeafQueue implements CSQueu
LOG.info(getQueueName() +
" used=" + usedResources + " numContainers=" + numContainers +
- " user=" + userName + " resources=" + user.getConsumedResources());
+ " user=" + userName + " user-resources=" + user.getConsumedResources());
}
@Override
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1232275&r1=1232274&r2=1232275&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java Tue Jan 17 02:28:49 2012
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.factory.pr
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
@@ -500,10 +501,12 @@ public class ParentQueue implements CSQu
}
@Override
- public synchronized Resource assignContainers(
+ public synchronized CSAssignment assignContainers(
Resource clusterResource, SchedulerNode node) {
- Resource assigned = Resources.createResource(0);
-
+ CSAssignment assignment =
+ new CSAssignment(Resources.createResource(0), NodeType.NODE_LOCAL);
+ boolean assignedOffSwitch = false;
+
while (canAssign(node)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Trying to assign containers to child-queue of "
@@ -516,16 +519,18 @@ public class ParentQueue implements CSQu
}
// Schedule
- Resource assignedToChild =
+ CSAssignment assignedToChild =
assignContainersToChildQueues(clusterResource, node);
+ assignedOffSwitch = (assignedToChild.getType() == NodeType.OFF_SWITCH);
// Done if no child-queue assigned anything
- if (Resources.greaterThan(assignedToChild, Resources.none())) {
+ if (Resources.greaterThan(assignedToChild.getResource(),
+ Resources.none())) {
// Track resource utilization for the parent-queue
- allocateResource(clusterResource, assignedToChild);
+ allocateResource(clusterResource, assignedToChild.getResource());
// Track resource utilization in this pass of the scheduler
- Resources.addTo(assigned, assignedToChild);
+ Resources.addTo(assignment.getResource(), assignedToChild.getResource());
LOG.info("assignedContainer" +
" queue=" + getQueueName() +
@@ -539,17 +544,26 @@ public class ParentQueue implements CSQu
if (LOG.isDebugEnabled()) {
LOG.debug("ParentQ=" + getQueueName()
- + " assignedSoFarInThisIteration=" + assigned
+ + " assignedSoFarInThisIteration=" + assignment.getResource()
+ " utilization=" + getUtilization());
}
// Do not assign more than one container if this isn't the root queue
- if (!rootQueue) {
+ // or if we've already assigned an off-switch container
+ if (rootQueue) {
+ if (assignedOffSwitch) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Not assigning more than one off-switch container," +
+ " assignments so far: " + assignment);
+ }
+ break;
+ }
+ } else {
break;
}
}
- return assigned;
+ return assignment;
}
private synchronized boolean assignToQueue(Resource clusterResource) {
@@ -573,9 +587,10 @@ public class ParentQueue implements CSQu
minimumAllocation);
}
- synchronized Resource assignContainersToChildQueues(Resource cluster,
+ synchronized CSAssignment assignContainersToChildQueues(Resource cluster,
SchedulerNode node) {
- Resource assigned = Resources.createResource(0);
+ CSAssignment assignment =
+ new CSAssignment(Resources.createResource(0), NodeType.NODE_LOCAL);
printChildQueues();
@@ -586,25 +601,28 @@ public class ParentQueue implements CSQu
LOG.debug("Trying to assign to queue: " + childQueue.getQueuePath()
+ " stats: " + childQueue);
}
- assigned = childQueue.assignContainers(cluster, node);
+ assignment = childQueue.assignContainers(cluster, node);
if(LOG.isDebugEnabled()) {
- LOG.debug("Assignedto queue: " + childQueue.getQueuePath()
- + " stats: " + childQueue + " --> " + assigned.getMemory());
+ LOG.debug("Assigned to queue: " + childQueue.getQueuePath() +
+ " stats: " + childQueue + " --> " +
+ assignment.getResource().getMemory() + ", " + assignment.getType());
}
// If we do assign, remove the queue and re-insert in-order to re-sort
- if (Resources.greaterThan(assigned, Resources.none())) {
+ if (Resources.greaterThan(assignment.getResource(), Resources.none())) {
// Remove and re-insert to sort
iter.remove();
LOG.info("Re-sorting queues since queue: " + childQueue.getQueuePath() +
" stats: " + childQueue);
childQueues.add(childQueue);
- printChildQueues();
+ if (LOG.isDebugEnabled()) {
+ printChildQueues();
+ }
break;
}
}
- return assigned;
+ return assignment;
}
String getChildQueuesToPrint() {
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1232275&r1=1232274&r2=1232275&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Tue Jan 17 02:28:49 2012
@@ -811,49 +811,56 @@ public class TestLeafQueue {
app_0.updateResourceRequests(app_0_requests_0);
// Start testing...
+ CSAssignment assignment = null;
// Start with off switch, shouldn't allocate due to delay scheduling
- a.assignContainers(clusterResource, node_2);
+ assignment = a.assignContainers(clusterResource, node_2);
verify(app_0, never()).allocate(any(NodeType.class), eq(node_2),
any(Priority.class), any(ResourceRequest.class), any(Container.class));
assertEquals(1, app_0.getSchedulingOpportunities(priority));
assertEquals(3, app_0.getTotalRequiredResources(priority));
+ assertEquals(NodeType.NODE_LOCAL, assignment.getType()); // None->NODE_LOCAL
// Another off switch, shouldn't allocate due to delay scheduling
- a.assignContainers(clusterResource, node_2);
+ assignment = a.assignContainers(clusterResource, node_2);
verify(app_0, never()).allocate(any(NodeType.class), eq(node_2),
any(Priority.class), any(ResourceRequest.class), any(Container.class));
assertEquals(2, app_0.getSchedulingOpportunities(priority));
assertEquals(3, app_0.getTotalRequiredResources(priority));
+ assertEquals(NodeType.NODE_LOCAL, assignment.getType()); // None->NODE_LOCAL
// Another off switch, shouldn't allocate due to delay scheduling
- a.assignContainers(clusterResource, node_2);
+ assignment = a.assignContainers(clusterResource, node_2);
verify(app_0, never()).allocate(any(NodeType.class), eq(node_2),
any(Priority.class), any(ResourceRequest.class), any(Container.class));
assertEquals(3, app_0.getSchedulingOpportunities(priority));
assertEquals(3, app_0.getTotalRequiredResources(priority));
+ assertEquals(NodeType.NODE_LOCAL, assignment.getType()); // None->NODE_LOCAL
// Another off switch, now we should allocate
// since missedOpportunities=3 and reqdContainers=3
- a.assignContainers(clusterResource, node_2);
+ assignment = a.assignContainers(clusterResource, node_2);
verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_2),
any(Priority.class), any(ResourceRequest.class), any(Container.class));
assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should reset
assertEquals(2, app_0.getTotalRequiredResources(priority));
+ assertEquals(NodeType.OFF_SWITCH, assignment.getType());
// NODE_LOCAL - node_0
- a.assignContainers(clusterResource, node_0);
+ assignment = a.assignContainers(clusterResource, node_0);
verify(app_0).allocate(eq(NodeType.NODE_LOCAL), eq(node_0),
any(Priority.class), any(ResourceRequest.class), any(Container.class));
assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should reset
assertEquals(1, app_0.getTotalRequiredResources(priority));
+ assertEquals(NodeType.NODE_LOCAL, assignment.getType());
// NODE_LOCAL - node_1
- a.assignContainers(clusterResource, node_1);
+ assignment = a.assignContainers(clusterResource, node_1);
verify(app_0).allocate(eq(NodeType.NODE_LOCAL), eq(node_1),
any(Priority.class), any(ResourceRequest.class), any(Container.class));
assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should reset
assertEquals(0, app_0.getTotalRequiredResources(priority));
+ assertEquals(NodeType.NODE_LOCAL, assignment.getType());
// Add 1 more request to check for RACK_LOCAL
app_0_requests_0.clear();
@@ -872,11 +879,12 @@ public class TestLeafQueue {
String host_3 = "host_3"; // on rack_1
SchedulerNode node_3 = TestUtils.getMockNode(host_3, rack_1, 0, 8*GB);
- a.assignContainers(clusterResource, node_3);
+ assignment = a.assignContainers(clusterResource, node_3);
verify(app_0).allocate(eq(NodeType.RACK_LOCAL), eq(node_3),
any(Priority.class), any(ResourceRequest.class), any(Container.class));
assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should reset
assertEquals(0, app_0.getTotalRequiredResources(priority));
+ assertEquals(NodeType.RACK_LOCAL, assignment.getType());
}
@Test
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java?rev=1232275&r1=1232274&r2=1232275&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java Tue Jan 17 02:28:49 2012
@@ -29,6 +29,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
@@ -92,11 +93,18 @@ public class TestParentQueue {
private void stubQueueAllocation(final CSQueue queue,
final Resource clusterResource, final SchedulerNode node,
final int allocation) {
+ stubQueueAllocation(queue, clusterResource, node, allocation,
+ NodeType.NODE_LOCAL);
+ }
+
+ private void stubQueueAllocation(final CSQueue queue,
+ final Resource clusterResource, final SchedulerNode node,
+ final int allocation, final NodeType type) {
// Simulate the queue allocation
- doAnswer(new Answer<Resource>() {
+ doAnswer(new Answer<CSAssignment>() {
@Override
- public Resource answer(InvocationOnMock invocation) throws Throwable {
+ public CSAssignment answer(InvocationOnMock invocation) throws Throwable {
try {
throw new Exception();
} catch (Exception e) {
@@ -115,8 +123,8 @@ public class TestParentQueue {
// Next call - nothing
if (allocation > 0) {
- doReturn(Resources.none()).when(queue).assignContainers(
- eq(clusterResource), eq(node));
+ doReturn(new CSAssignment(Resources.none(), type)).
+ when(queue).assignContainers(eq(clusterResource), eq(node));
// Mock the node's resource availability
Resource available = node.getAvailableResource();
@@ -124,7 +132,7 @@ public class TestParentQueue {
when(node).getAvailableResource();
}
- return allocatedResource;
+ return new CSAssignment(allocatedResource, type);
}
}).
when(queue).assignContainers(eq(clusterResource), eq(node));
@@ -401,6 +409,78 @@ public class TestParentQueue {
}
+ @Test
+ public void testOffSwitchScheduling() throws Exception {
+ // Setup queue configs
+ setupSingleLevelQueues(csConf);
+
+ Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
+ CSQueue root =
+ CapacityScheduler.parseQueue(csContext, csConf, null,
+ CapacitySchedulerConfiguration.ROOT, queues, queues,
+ CapacityScheduler.queueComparator,
+ CapacityScheduler.applicationComparator,
+ TestUtils.spyHook);
+
+ // Setup some nodes
+ final int memoryPerNode = 10;
+ final int numNodes = 2;
+
+ SchedulerNode node_0 =
+ TestUtils.getMockNode("host_0", DEFAULT_RACK, 0, memoryPerNode*GB);
+ SchedulerNode node_1 =
+ TestUtils.getMockNode("host_1", DEFAULT_RACK, 0, memoryPerNode*GB);
+
+ final Resource clusterResource =
+ Resources.createResource(numNodes * (memoryPerNode*GB));
+ when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+
+ // Start testing
+ LeafQueue a = (LeafQueue)queues.get(A);
+ LeafQueue b = (LeafQueue)queues.get(B);
+ final float delta = 0.0001f;
+
+ // Simulate B returning a container on node_0
+ stubQueueAllocation(a, clusterResource, node_0, 0*GB, NodeType.OFF_SWITCH);
+ stubQueueAllocation(b, clusterResource, node_0, 1*GB, NodeType.OFF_SWITCH);
+ root.assignContainers(clusterResource, node_0);
+ assertEquals(0.0f, a.getUtilization(), delta);
+ assertEquals(computeQueueUtilization(b, 1*GB, clusterResource),
+ b.getUtilization(), delta);
+
+ // Now, A should get the scheduling opportunity since A=0G/6G, B=1G/14G
+ // also, B gets a scheduling opportunity since A allocates RACK_LOCAL
+ stubQueueAllocation(a, clusterResource, node_1, 2*GB, NodeType.RACK_LOCAL);
+ stubQueueAllocation(b, clusterResource, node_1, 1*GB, NodeType.OFF_SWITCH);
+ root.assignContainers(clusterResource, node_1);
+ InOrder allocationOrder = inOrder(a, b);
+ allocationOrder.verify(a).assignContainers(eq(clusterResource),
+ any(SchedulerNode.class));
+ allocationOrder.verify(b).assignContainers(eq(clusterResource),
+ any(SchedulerNode.class));
+ assertEquals(computeQueueUtilization(a, 2*GB, clusterResource),
+ a.getUtilization(), delta);
+ assertEquals(computeQueueUtilization(b, 2*GB, clusterResource),
+ b.getUtilization(), delta);
+
+ // Now, B should get the scheduling opportunity
+ // since A has 2/6G while B has 2/14G,
+ // However, since B returns off-switch, A won't get an opportunity
+ stubQueueAllocation(a, clusterResource, node_0, 1*GB, NodeType.NODE_LOCAL);
+ stubQueueAllocation(b, clusterResource, node_0, 2*GB, NodeType.OFF_SWITCH);
+ root.assignContainers(clusterResource, node_0);
+ allocationOrder = inOrder(b, a);
+ allocationOrder.verify(b).assignContainers(eq(clusterResource),
+ any(SchedulerNode.class));
+ allocationOrder.verify(a).assignContainers(eq(clusterResource),
+ any(SchedulerNode.class));
+ assertEquals(computeQueueUtilization(a, 2*GB, clusterResource),
+ a.getUtilization(), delta);
+ assertEquals(computeQueueUtilization(b, 4*GB, clusterResource),
+ b.getUtilization(), delta);
+
+ }
+
@After
public void tearDown() throws Exception {
}
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,3 +1,3 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1166402,1167001,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1212299,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/hadoop-yarn/hadoop-yarn-site:713112
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,3 +1,3 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1166402,1167001,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205626,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1212299,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/hadoop-yarn/hadoop-yarn-site/src/site/apt:713112
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/Federation.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/Federation.apt.vm?rev=1232275&r1=1232274&r2=1232275&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/Federation.apt.vm (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/Federation.apt.vm Tue Jan 17 02:28:49 2012
@@ -169,7 +169,7 @@ HDFS Federation
</property>
<property>
<name>dfs.namenode.rpc-address.ns1</name>
- <value>hdfs://nn-host1:rpc-port</value>
+ <value>nn-host1:rpc-port</value>
</property>
<property>
<name>dfs.namenode.http-address.ns1</name>
@@ -181,7 +181,7 @@ HDFS Federation
</property>
<property>
<name>dfs.namenode.rpc-address.ns2</name>
- <value>hdfs://nn-host2:rpc-port</value>
+ <value>nn-host2:rpc-port</value>
</property>
<property>
<name>dfs.namenode.http-address.ns2</name>
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/c++/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/c++:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/c++:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/c++:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/src/c++:713112
/hadoop/core/trunk/src/c++:776175-784663
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1166402,1167001,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/src/contrib:713112
/hadoop/core/trunk/src/contrib:784664-785643
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/contrib/block_forensics/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,3 +1,3 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/block_forensics:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/block_forensics:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/hdfs/src/contrib/block_forensics:713112
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/contrib/build-contrib.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/build-contrib.xml:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/build-contrib.xml:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/src/contrib/build-contrib.xml:713112
/hadoop/core/trunk/src/contrib/build-contrib.xml:776175-786373
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/contrib/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/build.xml:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/build.xml:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/src/contrib/build.xml:713112
/hadoop/core/trunk/src/contrib/build.xml:776175-786373
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/contrib/data_join/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/data_join:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/data_join:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/src/contrib/data_join:713112
/hadoop/core/trunk/src/contrib/data_join:776175-786373
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/contrib/eclipse-plugin/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/eclipse-plugin:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/eclipse-plugin:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/core/src/contrib/eclipse-plugin:713112
/hadoop/core/trunk/src/contrib/eclipse-plugin:776175-784663
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/contrib/index/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/index:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/index:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/src/contrib/index:713112
/hadoop/core/trunk/src/contrib/index:776175-786373
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/contrib/vaidya/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/vaidya:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/vaidya:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/src/contrib/vaidya:713112
/hadoop/core/trunk/src/contrib/vaidya:776175-786373
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/examples/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/examples:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/examples:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/examples:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1164771,1166402,1167318,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/src/examples:713112
/hadoop/core/trunk/src/examples:776175-784663
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/java:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1164771,1166402,1167318,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/src/java:713112
/hadoop/core/trunk/src/mapred:776175-785643
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/test/mapred/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1164771,1166402,1167318,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred:713112
/hadoop/core/trunk/src/test/mapred:776175-785643
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1164771,1166402,1167318,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/fs:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/fs:776175-785643
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1164771,1166402,1167318,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/hdfs:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/hdfs:776175-785643
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1 +1,2 @@
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java:1182205,1204177,1205697,1225192,1225456,1225489,1225591,1226239,1227165,1229347,1230398,1231569,1231572,1231627,1231640
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1 +1,2 @@
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:1182205,1204177,1205697,1225192,1225456,1225489,1225591,1226239,1227165,1229347,1230398,1231569,1231572,1231627,1231640
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1164771,1166402,1167318,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs-with-mr/org/apache/hadoop/ipc:713112
/hadoop/core/trunk/src/test/hdfs-with-mr/org/apache/hadoop/ipc:776175-784663
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1 +1,2 @@
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:1182205,1204177,1205697,1225192,1225456,1225489,1225591,1226239,1227165,1229347,1230398,1231569,1231572,1231627,1231640
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1 +1,2 @@
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:1182205,1204177,1205697,1225192,1225456,1225489,1225591,1226239,1227165,1229347,1230398,1231569,1231572,1231627,1231640
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/src/webapps/job/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 17 02:28:49 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/webapps/job:1227776-1231827
+/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/webapps/job:1227776-1232270
/hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1164255,1164301,1164339,1164771,1166402,1167318,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640
/hadoop/core/branches/branch-0.19/mapred/src/webapps/job:713112
/hadoop/core/trunk/src/webapps/job:776175-785643