You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sz...@apache.org on 2013/04/26 02:55:48 UTC
svn commit: r1476006 - in
/hadoop/common/branches/HDFS-2802/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/
hadoop-yarn/hadoop-y...
Author: szetszwo
Date: Fri Apr 26 00:55:45 2013
New Revision: 1476006
URL: http://svn.apache.org/r1476006
Log:
Merge r1471229 through r1476005 from trunk.
Added:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/
- copied from r1476005, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResources.java
- copied unchanged from r1476005, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResources.java
Removed:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Resources.java
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestApplicationTokens.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt Fri Apr 26 00:55:45 2013
@@ -98,6 +98,10 @@ Release 2.0.5-beta - UNRELEASED
YARN-561. Modified NodeManager to set key information into the environment
of every container that it launches. (Xuan Gong via vinodkv)
+ YARN-579. Stop setting the Application Token in the AppMaster env, in
+ favour of the copy present in the container token field.
+ (Vinod Kumar Vavilapalli via sseth)
+
NEW FEATURES
YARN-482. FS: Extend SchedulingMode to intermediate queues.
@@ -177,6 +181,12 @@ Release 2.0.5-beta - UNRELEASED
YARN-581. Added a test to verify that app delegation tokens are restored
after RM restart. (Jian He via vinodkv)
+ YARN-577. Add application-progress also to ApplicationReport. (Hitesh Shah
+ via vinodkv)
+
+ YARN-595. Refactor fair scheduler to use common Resources. (Sandy Ryza
+ via tomwhite)
+
OPTIMIZATIONS
BUG FIXES
@@ -290,6 +300,12 @@ Release 2.0.5-beta - UNRELEASED
YARN-549. YarnClient.submitApplication should wait for application to be
accepted by the RM (Zhijie Shen via bikas)
+ YARN-605. Fix failing unit test in TestNMWebServices when versionInfo has
+ parantheses like when running on a git checkout. (Hitesh Shah via vinodkv)
+
+ YARN-289. Fair scheduler allows reservations that won't fit on node.
+ (Sandy Ryza via tomwhite)
+
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java Fri Apr 26 00:55:45 2013
@@ -30,10 +30,6 @@ import org.apache.hadoop.util.Shell;
public interface ApplicationConstants {
// TODO: They say tokens via env isn't good.
- public static final String APPLICATION_MASTER_TOKEN_ENV_NAME =
- "AppMasterTokenEnv";
-
- // TODO: They say tokens via env isn't good.
public static final String APPLICATION_CLIENT_SECRET_ENV_NAME =
"AppClientSecretEnv";
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java Fri Apr 26 00:55:45 2013
@@ -253,4 +253,16 @@ public interface ApplicationReport {
@Private
@Unstable
void setApplicationResourceUsageReport(ApplicationResourceUsageReport appResources);
+
+ /**
+ * Get the application's progress ( range 0.0 to 1.0 )
+ * @return application's progress
+ */
+ @Public
+ @Stable
+ float getProgress();
+
+ @Private
+ @Unstable
+ void setProgress(float progress);
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java Fri Apr 26 00:55:45 2013
@@ -214,6 +214,12 @@ implements ApplicationReport {
}
@Override
+ public float getProgress() {
+ ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getProgress();
+ }
+
+ @Override
public void setApplicationId(ApplicationId applicationId) {
maybeInitBuilder();
if (applicationId == null)
@@ -346,6 +352,12 @@ implements ApplicationReport {
}
@Override
+ public void setProgress(float progress) {
+ maybeInitBuilder();
+ builder.setProgress(progress);
+ }
+
+ @Override
public ApplicationReportProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Fri Apr 26 00:55:45 2013
@@ -162,6 +162,7 @@ message ApplicationReportProto {
optional ApplicationResourceUsageReportProto app_resource_Usage = 16;
optional string originalTrackingUrl = 17;
optional ApplicationAttemptIdProto currentApplicationAttemptId = 18;
+ optional float progress = 19;
}
enum NodeStateProto {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java Fri Apr 26 00:55:45 2013
@@ -33,13 +33,9 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.AMRMProtocol;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
@@ -115,24 +111,7 @@ public class AMRMClientImpl extends Abst
throw new YarnException(e);
}
- if (UserGroupInformation.isSecurityEnabled()) {
- String tokenURLEncodedStr = System.getenv().get(
- ApplicationConstants.APPLICATION_MASTER_TOKEN_ENV_NAME);
- Token<? extends TokenIdentifier> token = new Token<TokenIdentifier>();
-
- try {
- token.decodeFromUrlString(tokenURLEncodedStr);
- } catch (IOException e) {
- throw new YarnException(e);
- }
-
- SecurityUtil.setTokenService(token, rmAddress);
- if (LOG.isDebugEnabled()) {
- LOG.debug("AppMasterToken is " + token);
- }
- currentUser.addToken(token);
- }
-
+ // CurrentUser should already have AMToken loaded.
rmClient = currentUser.doAs(new PrivilegedAction<AMRMProtocol>() {
@Override
public AMRMProtocol run() {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java Fri Apr 26 00:55:45 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.client.cl
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
+import java.text.DecimalFormat;
import java.util.List;
import org.apache.commons.cli.CommandLine;
@@ -34,7 +35,7 @@ import org.apache.hadoop.yarn.util.Conve
public class ApplicationCLI extends YarnCLI {
private static final String APPLICATIONS_PATTERN =
- "%30s\t%20s\t%10s\t%10s\t%18s\t%18s\t%35s" +
+ "%30s\t%20s\t%10s\t%10s\t%18s\t%18s\t%15s\t%35s" +
System.getProperty("line.separator");
public static void main(String[] args) throws Exception {
@@ -98,12 +99,15 @@ public class ApplicationCLI extends Yarn
writer.println("Total Applications:" + appsReport.size());
writer.printf(APPLICATIONS_PATTERN, "Application-Id",
"Application-Name", "User", "Queue", "State", "Final-State",
- "Tracking-URL");
+ "Progress", "Tracking-URL");
for (ApplicationReport appReport : appsReport) {
+ DecimalFormat formatter = new DecimalFormat("###.##%");
+ String progress = formatter.format(appReport.getProgress());
writer.printf(APPLICATIONS_PATTERN, appReport.getApplicationId(),
appReport.getName(), appReport.getUser(), appReport.getQueue(),
appReport.getYarnApplicationState(), appReport
- .getFinalApplicationStatus(), appReport.getOriginalTrackingUrl());
+ .getFinalApplicationStatus(),
+ progress, appReport.getOriginalTrackingUrl());
}
writer.flush();
}
@@ -147,6 +151,10 @@ public class ApplicationCLI extends Yarn
appReportStr.println(appReport.getStartTime());
appReportStr.print("\tFinish-Time : ");
appReportStr.println(appReport.getFinishTime());
+ appReportStr.print("\tProgress : ");
+ DecimalFormat formatter = new DecimalFormat("###.##%");
+ String progress = formatter.format(appReport.getProgress());
+ appReportStr.println(progress);
appReportStr.print("\tState : ");
appReportStr.println(appReport.getYarnApplicationState());
appReportStr.print("\tFinal-State : ");
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java Fri Apr 26 00:55:45 2013
@@ -76,7 +76,7 @@ public class TestYarnCLI {
applicationId, BuilderUtils.newApplicationAttemptId(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A");
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport);
int result = cli.run(new String[] { "-status", applicationId.toString() });
@@ -91,6 +91,7 @@ public class TestYarnCLI {
pw.println("\tQueue : queue");
pw.println("\tStart-Time : 0");
pw.println("\tFinish-Time : 0");
+ pw.println("\tProgress : 53.79%");
pw.println("\tState : FINISHED");
pw.println("\tFinal-State : SUCCEEDED");
pw.println("\tTracking-URL : N/A");
@@ -111,7 +112,7 @@ public class TestYarnCLI {
applicationId, BuilderUtils.newApplicationAttemptId(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A");
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f);
List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
applicationReports.add(newApplicationReport);
when(client.getApplicationList()).thenReturn(applicationReports);
@@ -124,10 +125,12 @@ public class TestYarnCLI {
pw.println("Total Applications:1");
pw.print(" Application-Id\t Application-Name");
pw.print("\t User\t Queue\t State\t ");
- pw.println("Final-State\t Tracking-URL");
+ pw.print("Final-State\t Progress");
+ pw.println("\t Tracking-URL");
pw.print(" application_1234_0005\t ");
pw.print("appname\t user\t queue\t FINISHED\t ");
- pw.println("SUCCEEDED\t N/A");
+ pw.print("SUCCEEDED\t 53.79%");
+ pw.println("\t N/A");
pw.close();
String appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString());
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java Fri Apr 26 00:55:45 2013
@@ -333,7 +333,8 @@ public class BuilderUtils {
ClientToken clientToken, YarnApplicationState state, String diagnostics,
String url, long startTime, long finishTime,
FinalApplicationStatus finalStatus,
- ApplicationResourceUsageReport appResources, String origTrackingUrl) {
+ ApplicationResourceUsageReport appResources, String origTrackingUrl,
+ float progress) {
ApplicationReport report = recordFactory
.newRecordInstance(ApplicationReport.class);
report.setApplicationId(applicationId);
@@ -352,6 +353,7 @@ public class BuilderUtils {
report.setFinalApplicationStatus(finalStatus);
report.setApplicationResourceUsageReport(appResources);
report.setOriginalTrackingUrl(origTrackingUrl);
+ report.setProgress(progress);
return report;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java Fri Apr 26 00:55:45 2013
@@ -85,5 +85,10 @@ public class WebServicesTestUtils {
got.contains(expected));
}
+ public static void checkStringEqual(String print, String expected, String got) {
+ assertTrue(
+ print + " is not equal, got: " + got + " expected: " + expected,
+ got.equals(expected));
+ }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java Fri Apr 26 00:55:45 2013
@@ -360,14 +360,14 @@ public class TestNMWebServices extends J
WebServicesTestUtils.checkStringMatch("hadoopVersionBuiltOn",
VersionInfo.getDate(), hadoopVersionBuiltOn);
- WebServicesTestUtils.checkStringMatch("hadoopBuildVersion",
+ WebServicesTestUtils.checkStringEqual("hadoopBuildVersion",
VersionInfo.getBuildVersion(), hadoopBuildVersion);
WebServicesTestUtils.checkStringMatch("hadoopVersion",
VersionInfo.getVersion(), hadoopVersion);
WebServicesTestUtils.checkStringMatch("resourceManagerVersionBuiltOn",
YarnVersionInfo.getDate(), resourceManagerVersionBuiltOn);
- WebServicesTestUtils.checkStringMatch("resourceManagerBuildVersion",
+ WebServicesTestUtils.checkStringEqual("resourceManagerBuildVersion",
YarnVersionInfo.getBuildVersion(), resourceManagerBuildVersion);
WebServicesTestUtils.checkStringMatch("resourceManagerVersion",
YarnVersionInfo.getVersion(), resourceManagerVersion);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java Fri Apr 26 00:55:45 2013
@@ -204,7 +204,7 @@ public class AMLauncher implements Runna
ApplicationTokenIdentifier id = new ApplicationTokenIdentifier(
application.getAppAttemptId());
- Token<ApplicationTokenIdentifier> token =
+ Token<ApplicationTokenIdentifier> appMasterToken =
new Token<ApplicationTokenIdentifier>(id,
this.rmContext.getApplicationTokenSecretManager());
InetSocketAddress serviceAddr = conf.getSocketAddr(
@@ -212,16 +212,11 @@ public class AMLauncher implements Runna
YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
// normally the client should set the service after acquiring the token,
- // but this token is directly provided to the tasks
- SecurityUtil.setTokenService(token, serviceAddr);
- String appMasterTokenEncoded = token.encodeToUrlString();
- LOG.debug("Putting appMaster token in env : " + token);
- environment.put(
- ApplicationConstants.APPLICATION_MASTER_TOKEN_ENV_NAME,
- appMasterTokenEncoded);
+ // but this token is directly provided to the AMs
+ SecurityUtil.setTokenService(appMasterToken, serviceAddr);
- // Add the RM token
- credentials.addToken(token.getService(), token);
+ // Add the ApplicationMaster token
+ credentials.addToken(appMasterToken.getService(), appMasterToken);
DataOutputBuffer dob = new DataOutputBuffer();
credentials.writeTokenStorageToStream(dob);
container.setContainerTokens(
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java Fri Apr 26 00:55:45 2013
@@ -208,4 +208,14 @@ public class Resources {
Resource lhs, Resource rhs) {
return resourceCalculator.compare(clusterResource, lhs, rhs) >= 0 ? lhs : rhs;
}
+
+ public static boolean fitsIn(Resource smaller, Resource bigger) {
+ return smaller.getMemory() <= bigger.getMemory() &&
+ smaller.getVirtualCores() <= bigger.getVirtualCores();
+ }
+
+ public static Resource componentwiseMin(Resource lhs, Resource rhs) {
+ return createResource(Math.min(lhs.getMemory(), rhs.getMemory()),
+ Math.min(lhs.getVirtualCores(), rhs.getVirtualCores()));
+ }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Fri Apr 26 00:55:45 2013
@@ -437,6 +437,7 @@ public class RMAppImpl implements RMApp,
DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
FinalApplicationStatus finishState = getFinalApplicationStatus();
String diags = UNAVAILABLE;
+ float progress = 0.0f;
if (allowAccess) {
if (this.currentAttempt != null) {
currentApplicationAttemptId = this.currentAttempt.getAppAttemptId();
@@ -446,8 +447,8 @@ public class RMAppImpl implements RMApp,
host = this.currentAttempt.getHost();
rpcPort = this.currentAttempt.getRpcPort();
appUsageReport = currentAttempt.getApplicationResourceUsageReport();
+ progress = currentAttempt.getProgress();
}
-
diags = this.diagnostics.toString();
}
@@ -462,7 +463,7 @@ public class RMAppImpl implements RMApp,
this.name, host, rpcPort, clientToken,
createApplicationState(this.stateMachine.getCurrentState()), diags,
trackingUrl, this.startTime, this.finishTime, finishState,
- appUsageReport, origTrackingUrl);
+ appUsageReport, origTrackingUrl, progress);
} finally {
this.readLock.unlock();
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java Fri Apr 26 00:55:45 2013
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
@@ -45,6 +46,9 @@ import org.apache.hadoop.yarn.util.Build
@Private
@Unstable
public class AppSchedulable extends Schedulable {
+ private static final DefaultResourceCalculator RESOURCE_CALCULATOR
+ = new DefaultResourceCalculator();
+
private FairScheduler scheduler;
private FSSchedulerApp app;
private Resource demand = Resources.createResource(0);
@@ -180,15 +184,15 @@ public class AppSchedulable extends Sche
* update relevant bookeeping. This dispatches ro relevant handlers
* in the {@link FSSchedulerNode} and {@link SchedulerApp} classes.
*/
- private void reserve(FSSchedulerApp application, Priority priority,
- FSSchedulerNode node, Container container, boolean alreadyReserved) {
+ private void reserve(Priority priority, FSSchedulerNode node,
+ Container container, boolean alreadyReserved) {
LOG.info("Making reservation: node=" + node.getHostName() +
" app_id=" + app.getApplicationId());
if (!alreadyReserved) {
- getMetrics().reserveResource(application.getUser(), container.getResource());
- RMContainer rmContainer = application.reserve(node, priority, null,
+ getMetrics().reserveResource(app.getUser(), container.getResource());
+ RMContainer rmContainer = app.reserve(node, priority, null,
container);
- node.reserveResource(application, priority, rmContainer);
+ node.reserveResource(app, priority, rmContainer);
getMetrics().reserveResource(app.getUser(),
container.getResource());
scheduler.getRootQueueMetrics().reserveResource(app.getUser(),
@@ -197,25 +201,24 @@ public class AppSchedulable extends Sche
else {
RMContainer rmContainer = node.getReservedContainer();
- application.reserve(node, priority, rmContainer, container);
- node.reserveResource(application, priority, rmContainer);
+ app.reserve(node, priority, rmContainer, container);
+ node.reserveResource(app, priority, rmContainer);
}
}
/**
- * Remove the reservation on {@code node} for {@ application} at the given
+ * Remove the reservation on {@code node} at the given
* {@link Priority}. This dispatches to the SchedulerApp and SchedulerNode
* handlers for an unreservation.
*/
- private void unreserve(FSSchedulerApp application, Priority priority,
- FSSchedulerNode node) {
+ public void unreserve(Priority priority, FSSchedulerNode node) {
RMContainer rmContainer = node.getReservedContainer();
- application.unreserve(node, priority);
- node.unreserveResource(application);
+ app.unreserve(node, priority);
+ node.unreserveResource(app);
getMetrics().unreserveResource(
- application.getUser(), rmContainer.getContainer().getResource());
+ app.getUser(), rmContainer.getContainer().getResource());
scheduler.getRootQueueMetrics().unreserveResource(
- application.getUser(), rmContainer.getContainer().getResource());
+ app.getUser(), rmContainer.getContainer().getResource());
}
/**
@@ -224,8 +227,8 @@ public class AppSchedulable extends Sche
* sure the particular request should be facilitated by this node.
*/
private Resource assignContainer(FSSchedulerNode node,
- FSSchedulerApp application, Priority priority,
- ResourceRequest request, NodeType type, boolean reserved) {
+ Priority priority, ResourceRequest request, NodeType type,
+ boolean reserved) {
// How much does this request need?
Resource capability = request.getCapability();
@@ -237,7 +240,7 @@ public class AppSchedulable extends Sche
if (reserved) {
container = node.getReservedContainer().getContainer();
} else {
- container = createContainer(application, node, capability, priority);
+ container = createContainer(app, node, capability, priority);
}
// Can we allocate a container on this node?
@@ -247,9 +250,12 @@ public class AppSchedulable extends Sche
if (availableContainers > 0) {
// Inform the application of the new container for this request
RMContainer allocatedContainer =
- application.allocate(type, node, priority, request, container);
+ app.allocate(type, node, priority, request, container);
if (allocatedContainer == null) {
// Did the application need this resource?
+ if (reserved) {
+ unreserve(priority, node);
+ }
return Resources.none();
}
else {
@@ -262,17 +268,17 @@ public class AppSchedulable extends Sche
// If we had previously made a reservation, delete it
if (reserved) {
- unreserve(application, priority, node);
+ unreserve(priority, node);
}
// Inform the node
- node.allocateContainer(application.getApplicationId(),
+ node.allocateContainer(app.getApplicationId(),
allocatedContainer);
return container.getResource();
} else {
// The desired container won't fit here, so reserve
- reserve(application, priority, node, container, reserved);
+ reserve(priority, node, container, reserved);
return FairScheduler.CONTAINER_RESERVED;
}
@@ -287,7 +293,7 @@ public class AppSchedulable extends Sche
// Make sure the application still needs requests at this priority
if (app.getTotalRequiredResources(priority) == 0) {
- unreserve(app, priority, node);
+ unreserve(priority, node);
return Resources.none();
}
} else {
@@ -304,7 +310,8 @@ public class AppSchedulable extends Sche
// (not scheduled) in order to promote better locality.
synchronized (app) {
for (Priority priority : prioritiesToTry) {
- if (app.getTotalRequiredResources(priority) <= 0) {
+ if (app.getTotalRequiredResources(priority) <= 0 ||
+ !hasContainerForNode(priority, node)) {
continue;
}
@@ -321,14 +328,14 @@ public class AppSchedulable extends Sche
if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0
&& localRequest != null && localRequest.getNumContainers() != 0) {
- return assignContainer(node, app, priority,
+ return assignContainer(node, priority,
localRequest, NodeType.NODE_LOCAL, reserved);
}
if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0
&& (allowedLocality.equals(NodeType.RACK_LOCAL) ||
allowedLocality.equals(NodeType.OFF_SWITCH))) {
- return assignContainer(node, app, priority, rackLocalRequest,
+ return assignContainer(node, priority, rackLocalRequest,
NodeType.RACK_LOCAL, reserved);
}
@@ -336,7 +343,7 @@ public class AppSchedulable extends Sche
ResourceRequest.ANY);
if (offSwitchRequest != null && offSwitchRequest.getNumContainers() != 0
&& allowedLocality.equals(NodeType.OFF_SWITCH)) {
- return assignContainer(node, app, priority, offSwitchRequest,
+ return assignContainer(node, priority, offSwitchRequest,
NodeType.OFF_SWITCH, reserved);
}
}
@@ -352,4 +359,16 @@ public class AppSchedulable extends Sche
public Resource assignContainer(FSSchedulerNode node) {
return assignContainer(node, false);
}
+
+ /**
+ * Whether this app has containers requests that could be satisfied on the
+ * given node, if the node had full space.
+ */
+ public boolean hasContainerForNode(Priority prio, FSSchedulerNode node) {
+ // TODO: add checks stuff about node specific scheduling here
+ ResourceRequest request = app.getResourceRequest(prio, ResourceRequest.ANY);
+ return request.getNumContainers() > 0 &&
+ Resources.lessThanOrEqual(RESOURCE_CALCULATOR, null,
+ request.getCapability(), node.getRMNode().getTotalCapability());
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java Fri Apr 26 00:55:45 2013
@@ -33,6 +33,7 @@ import org.apache.hadoop.security.author
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
public class FSLeafQueue extends FSQueue {
private static final Log LOG = LogFactory.getLog(
@@ -126,8 +127,8 @@ public class FSLeafQueue extends FSQueue
+ demand);
}
demand = Resources.add(demand, toAdd);
- if (Resources.greaterThanOrEqual(demand, maxRes)) {
- demand = maxRes;
+ demand = Resources.componentwiseMin(demand, maxRes);
+ if (Resources.equals(demand, maxRes)) {
break;
}
}
@@ -153,7 +154,7 @@ public class FSLeafQueue extends FSQueue
for (AppSchedulable sched : appScheds) {
if (sched.getRunnable()) {
assigned = sched.assignContainer(node);
- if (Resources.greaterThan(assigned, Resources.none())) {
+ if (!assigned.equals(Resources.none())) {
break;
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java Fri Apr 26 00:55:45 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
public class FSParentQueue extends FSQueue {
private static final Log LOG = LogFactory.getLog(
@@ -87,8 +88,8 @@ public class FSParentQueue extends FSQue
" now " + demand);
}
demand = Resources.add(demand, toAdd);
- if (Resources.greaterThanOrEqual(demand, maxRes)) {
- demand = maxRes;
+ demand = Resources.componentwiseMin(demand, maxRes);
+ if (Resources.equals(demand, maxRes)) {
break;
}
}
@@ -135,16 +136,14 @@ public class FSParentQueue extends FSQue
Resource assigned = Resources.none();
// If this queue is over its limit, reject
- if (Resources.greaterThan(getResourceUsage(),
- queueMgr.getMaxResources(getName()))) {
+ if (!assignContainerPreCheck(node)) {
return assigned;
}
Collections.sort(childQueues, policy.getComparator());
for (FSQueue child : childQueues) {
assigned = child.assignContainer(node);
- if (node.getReservedContainer() != null
- || Resources.greaterThan(assigned, Resources.none())) {
+ if (!Resources.equals(assigned, Resources.none())) {
break;
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java Fri Apr 26 00:55:45 2013
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
@@ -161,7 +162,7 @@ public abstract class FSQueue extends Sc
* @return true if check passes (can assign) or false otherwise
*/
protected boolean assignContainerPreCheck(FSSchedulerNode node) {
- if (Resources.greaterThan(getResourceUsage(),
+ if (!Resources.fitsIn(getResourceUsage(),
queueMgr.getMaxResources(getName()))
|| node.getReservedContainer() != null) {
return false;
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java Fri Apr 26 00:55:45 2013
@@ -46,7 +46,7 @@ public class FSSchedulerNode extends Sch
private static final RecordFactory recordFactory = RecordFactoryProvider
.getRecordFactory(null);
- private Resource availableResource = recordFactory.newRecordInstance(Resource.class);
+ private Resource availableResource;
private Resource usedResource = recordFactory.newRecordInstance(Resource.class);
private volatile int numContainers;
@@ -62,7 +62,7 @@ public class FSSchedulerNode extends Sch
public FSSchedulerNode(RMNode node) {
this.rmNode = node;
- this.availableResource.setMemory(node.getTotalCapability().getMemory());
+ this.availableResource = Resources.clone(node.getTotalCapability());
}
public RMNode getRMNode() {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Fri Apr 26 00:55:45 2013
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
@@ -52,6 +53,9 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -111,6 +115,9 @@ public class FairScheduler implements Re
private static final Log LOG = LogFactory.getLog(FairScheduler.class);
+ private static final ResourceCalculator RESOURCE_CALCULATOR =
+ new DefaultResourceCalculator();
+
// Value that container assignment methods return when a container is
// reserved
public static final Resource CONTAINER_RESERVED = Resources.createResource(-1);
@@ -246,8 +253,10 @@ public class FairScheduler implements Re
* Is a queue below its min share for the given task type?
*/
boolean isStarvedForMinShare(FSLeafQueue sched) {
- Resource desiredShare = Resources.min(sched.getMinShare(), sched.getDemand());
- return Resources.lessThan(sched.getResourceUsage(), desiredShare);
+ Resource desiredShare = Resources.min(RESOURCE_CALCULATOR, clusterCapacity,
+ sched.getMinShare(), sched.getDemand());
+ return Resources.lessThan(RESOURCE_CALCULATOR, clusterCapacity,
+ sched.getResourceUsage(), desiredShare);
}
/**
@@ -255,9 +264,10 @@ public class FairScheduler implements Re
* defined as being below half its fair share.
*/
boolean isStarvedForFairShare(FSLeafQueue sched) {
- Resource desiredFairShare = Resources.max(
+ Resource desiredFairShare = Resources.max(RESOURCE_CALCULATOR, clusterCapacity,
Resources.multiply(sched.getFairShare(), .5), sched.getDemand());
- return Resources.lessThan(sched.getResourceUsage(), desiredFairShare);
+ return Resources.lessThan(RESOURCE_CALCULATOR, clusterCapacity,
+ sched.getResourceUsage(), desiredFairShare);
}
/**
@@ -283,7 +293,8 @@ public class FairScheduler implements Re
for (FSLeafQueue sched : queueMgr.getLeafQueues()) {
resToPreempt = Resources.add(resToPreempt, resToPreempt(sched, curTime));
}
- if (Resources.greaterThan(resToPreempt, Resources.none())) {
+ if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterCapacity, resToPreempt,
+ Resources.none())) {
preemptResources(queueMgr.getLeafQueues(), resToPreempt);
}
}
@@ -309,7 +320,8 @@ public class FairScheduler implements Re
// Collect running containers from over-scheduled queues
List<RMContainer> runningContainers = new ArrayList<RMContainer>();
for (FSLeafQueue sched : scheds) {
- if (Resources.greaterThan(sched.getResourceUsage(), sched.getFairShare())) {
+ if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterCapacity,
+ sched.getResourceUsage(), sched.getFairShare())) {
for (AppSchedulable as : sched.getAppSchedulables()) {
for (RMContainer c : as.getApp().getLiveContainers()) {
runningContainers.add(c);
@@ -332,7 +344,8 @@ public class FairScheduler implements Re
// tasks, making sure we don't kill too many from any queue
for (RMContainer container : runningContainers) {
FSLeafQueue sched = queues.get(container);
- if (Resources.greaterThan(sched.getResourceUsage(), sched.getFairShare())) {
+ if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterCapacity,
+ sched.getResourceUsage(), sched.getFairShare())) {
LOG.info("Preempting container (prio=" + container.getContainer().getPriority() +
"res=" + container.getContainer().getResource() +
") from queue " + sched.getName());
@@ -345,7 +358,8 @@ public class FairScheduler implements Re
toPreempt = Resources.subtract(toPreempt,
container.getContainer().getResource());
- if (Resources.equals(toPreempt, Resources.none())) {
+ if (Resources.lessThanOrEqual(RESOURCE_CALCULATOR, clusterCapacity,
+ toPreempt, Resources.none())) {
break;
}
}
@@ -369,17 +383,21 @@ public class FairScheduler implements Re
Resource resDueToMinShare = Resources.none();
Resource resDueToFairShare = Resources.none();
if (curTime - sched.getLastTimeAtMinShare() > minShareTimeout) {
- Resource target = Resources.min(sched.getMinShare(), sched.getDemand());
- resDueToMinShare = Resources.max(Resources.none(),
- Resources.subtract(target, sched.getResourceUsage()));
+ Resource target = Resources.min(RESOURCE_CALCULATOR, clusterCapacity,
+ sched.getMinShare(), sched.getDemand());
+ resDueToMinShare = Resources.max(RESOURCE_CALCULATOR, clusterCapacity,
+ Resources.none(), Resources.subtract(target, sched.getResourceUsage()));
}
if (curTime - sched.getLastTimeAtHalfFairShare() > fairShareTimeout) {
- Resource target = Resources.min(sched.getFairShare(), sched.getDemand());
- resDueToFairShare = Resources.max(Resources.none(),
- Resources.subtract(target, sched.getResourceUsage()));
- }
- Resource resToPreempt = Resources.max(resDueToMinShare, resDueToFairShare);
- if (Resources.greaterThan(resToPreempt, Resources.none())) {
+ Resource target = Resources.min(RESOURCE_CALCULATOR, clusterCapacity,
+ sched.getFairShare(), sched.getDemand());
+ resDueToFairShare = Resources.max(RESOURCE_CALCULATOR, clusterCapacity,
+ Resources.none(), Resources.subtract(target, sched.getResourceUsage()));
+ }
+ Resource resToPreempt = Resources.max(RESOURCE_CALCULATOR, clusterCapacity,
+ resDueToMinShare, resDueToFairShare);
+ if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterCapacity,
+ resToPreempt, Resources.none())) {
String message = "Should preempt " + resToPreempt + " res for queue "
+ sched.getName() + ": resDueToMinShare = " + resDueToMinShare
+ ", resDueToFairShare = " + resDueToFairShare;
@@ -788,21 +806,32 @@ public class FairScheduler implements Re
AppSchedulable reservedAppSchedulable = node.getReservedAppSchedulable();
if (reservedAppSchedulable != null) {
- // Reservation exists; try to fulfill the reservation
- LOG.info("Trying to fulfill reservation for application "
- + reservedAppSchedulable.getApp().getApplicationAttemptId()
- + " on node: " + nm);
+ Priority reservedPriority = node.getReservedContainer().getReservedPriority();
+ if (reservedAppSchedulable != null &&
+ !reservedAppSchedulable.hasContainerForNode(reservedPriority, node)) {
+ // Don't hold the reservation if app can no longer use it
+ LOG.info("Releasing reservation that cannot be satisfied for application "
+ + reservedAppSchedulable.getApp().getApplicationAttemptId()
+ + " on node " + nm);
+ reservedAppSchedulable.unreserve(reservedPriority, node);
+ reservedAppSchedulable = null;
+ } else {
+ // Reservation exists; try to fulfill the reservation
+ LOG.info("Trying to fulfill reservation for application "
+ + reservedAppSchedulable.getApp().getApplicationAttemptId()
+ + " on node: " + nm);
- node.getReservedAppSchedulable().assignReservedContainer(node);
+ node.getReservedAppSchedulable().assignReservedContainer(node);
+ }
}
- else {
+ if (reservedAppSchedulable == null) {
// No reservation, schedule at queue which is farthest below fair share
int assignedContainers = 0;
while (node.getReservedContainer() == null) {
boolean assignedContainer = false;
- if (Resources.greaterThan(
- queueMgr.getRootQueue().assignContainer(node),
- Resources.none())) {
+ if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterCapacity,
+ queueMgr.getRootQueue().assignContainer(node),
+ Resources.none())) {
assignedContainer = true;
}
if (!assignedContainer) { break; }
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java Fri Apr 26 00:55:45 2013
@@ -42,6 +42,7 @@ import org.apache.hadoop.security.author
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
@@ -474,8 +475,8 @@ public class QueueManager {
}
queueAcls.put(queueName, acls);
if (maxQueueResources.containsKey(queueName) && minQueueResources.containsKey(queueName)
- && Resources.lessThan(maxQueueResources.get(queueName),
- minQueueResources.get(queueName))) {
+ && !Resources.fitsIn(minQueueResources.get(queueName),
+ maxQueueResources.get(queueName))) {
LOG.warn(String.format("Queue %s has max resources %d less than min resources %d",
queueName, maxQueueResources.get(queueName), minQueueResources.get(queueName)));
}
@@ -504,7 +505,7 @@ public class QueueManager {
if (maxQueueResource != null) {
return maxQueueResource;
} else {
- return Resources.createResource(Integer.MAX_VALUE);
+ return Resources.createResource(Integer.MAX_VALUE, Integer.MAX_VALUE);
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java Fri Apr 26 00:55:45 2013
@@ -22,7 +22,8 @@ import java.util.Collection;
import java.util.Comparator;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Resources;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SchedulingPolicy;
@@ -31,6 +32,8 @@ import com.google.common.annotations.Vis
public class FairSharePolicy extends SchedulingPolicy {
@VisibleForTesting
public static final String NAME = "Fairshare";
+ private static final DefaultResourceCalculator RESOURCE_CALCULATOR =
+ new DefaultResourceCalculator();
private FairShareComparator comparator = new FairShareComparator();
@Override
@@ -59,15 +62,19 @@ public class FairSharePolicy extends Sch
public int compare(Schedulable s1, Schedulable s2) {
double minShareRatio1, minShareRatio2;
double useToWeightRatio1, useToWeightRatio2;
- Resource minShare1 = Resources.min(s1.getMinShare(), s1.getDemand());
- Resource minShare2 = Resources.min(s2.getMinShare(), s2.getDemand());
- boolean s1Needy = Resources.lessThan(s1.getResourceUsage(), minShare1);
- boolean s2Needy = Resources.lessThan(s2.getResourceUsage(), minShare2);
+ Resource minShare1 = Resources.min(RESOURCE_CALCULATOR, null,
+ s1.getMinShare(), s1.getDemand());
+ Resource minShare2 = Resources.min(RESOURCE_CALCULATOR, null,
+ s2.getMinShare(), s2.getDemand());
+ boolean s1Needy = Resources.lessThan(RESOURCE_CALCULATOR, null,
+ s1.getResourceUsage(), minShare1);
+ boolean s2Needy = Resources.lessThan(RESOURCE_CALCULATOR, null,
+ s2.getResourceUsage(), minShare2);
Resource one = Resources.createResource(1);
minShareRatio1 = (double) s1.getResourceUsage().getMemory()
- / Resources.max(minShare1, one).getMemory();
+ / Resources.max(RESOURCE_CALCULATOR, null, minShare1, one).getMemory();
minShareRatio2 = (double) s2.getResourceUsage().getMemory()
- / Resources.max(minShare2, one).getMemory();
+ / Resources.max(RESOURCE_CALCULATOR, null, minShare2, one).getMemory();
useToWeightRatio1 = s1.getResourceUsage().getMemory() / s1.getWeight();
useToWeightRatio2 = s2.getResourceUsage().getMemory() / s2.getWeight();
int res = 0;
@@ -161,9 +168,11 @@ public class FairSharePolicy extends Sch
for (Schedulable sched : schedulables) {
Resources.addTo(totalDemand, sched.getDemand());
}
- Resource cap = Resources.min(totalDemand, totalResources);
+ Resource cap = Resources.min(RESOURCE_CALCULATOR, null, totalDemand,
+ totalResources);
double rMax = 1.0;
- while (Resources.lessThan(resUsedWithWeightToResRatio(rMax, schedulables),
+ while (Resources.lessThan(RESOURCE_CALCULATOR, null,
+ resUsedWithWeightToResRatio(rMax, schedulables),
cap)) {
rMax *= 2.0;
}
@@ -172,7 +181,8 @@ public class FairSharePolicy extends Sch
double right = rMax;
for (int i = 0; i < COMPUTE_FAIR_SHARES_ITERATIONS; i++) {
double mid = (left + right) / 2.0;
- if (Resources.lessThan(resUsedWithWeightToResRatio(mid, schedulables),
+ if (Resources.lessThan(RESOURCE_CALCULATOR, null,
+ resUsedWithWeightToResRatio(mid, schedulables),
cap)) {
left = mid;
} else {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java Fri Apr 26 00:55:45 2013
@@ -22,8 +22,8 @@ import java.util.Collection;
import java.util.Comparator;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SchedulingPolicy;
import com.google.common.annotations.VisibleForTesting;
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java Fri Apr 26 00:55:45 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.re
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
import java.security.PrivilegedAction;
import java.util.HashMap;
import java.util.Map;
@@ -28,11 +29,10 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.io.DataInputByteBuffer;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.api.AMRMProtocol;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ContainerManager;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse;
@@ -59,9 +59,17 @@ public class TestAMAuthorization {
private static final Log LOG = LogFactory.getLog(TestAMAuthorization.class);
+ private static final Configuration confWithSecurityEnabled =
+ new Configuration();
+ static {
+ confWithSecurityEnabled.set(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ UserGroupInformation.setConfiguration(confWithSecurityEnabled);
+ }
+
public static final class MyContainerManager implements ContainerManager {
- public Map<String, String> amContainerEnv;
+ public ByteBuffer amTokens;
public MyContainerManager() {
}
@@ -70,7 +78,7 @@ public class TestAMAuthorization {
public StartContainerResponse
startContainer(StartContainerRequest request)
throws YarnRemoteException {
- amContainerEnv = request.getContainerLaunchContext().getEnvironment();
+ amTokens = request.getContainerLaunchContext().getContainerTokens();
return null;
}
@@ -93,9 +101,6 @@ public class TestAMAuthorization {
public MockRMWithAMS(Configuration conf, ContainerManager containerManager) {
super(conf, containerManager);
- conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
- "kerberos");
- UserGroupInformation.setConfiguration(conf);
}
@Override
@@ -105,7 +110,6 @@ public class TestAMAuthorization {
@Override
protected ApplicationMasterService createApplicationMasterService() {
-
return new ApplicationMasterService(getRMContext(), this.scheduler);
}
}
@@ -113,7 +117,8 @@ public class TestAMAuthorization {
@Test
public void testAuthorizedAccess() throws Exception {
MyContainerManager containerManager = new MyContainerManager();
- final MockRM rm = new MockRMWithAMS(new Configuration(), containerManager);
+ final MockRM rm =
+ new MockRMWithAMS(confWithSecurityEnabled, containerManager);
rm.start();
MockNM nm1 = rm.registerNode("localhost:1234", 5120);
@@ -126,11 +131,11 @@ public class TestAMAuthorization {
nm1.nodeHeartbeat(true);
int waitCount = 0;
- while (containerManager.amContainerEnv == null && waitCount++ < 20) {
+ while (containerManager.amTokens == null && waitCount++ < 20) {
LOG.info("Waiting for AM Launch to happen..");
Thread.sleep(1000);
}
- Assert.assertNotNull(containerManager.amContainerEnv);
+ Assert.assertNotNull(containerManager.amTokens);
RMAppAttempt attempt = app.getCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
@@ -142,12 +147,12 @@ public class TestAMAuthorization {
UserGroupInformation currentUser = UserGroupInformation
.createRemoteUser(applicationAttemptId.toString());
- String tokenURLEncodedStr = containerManager.amContainerEnv
- .get(ApplicationConstants.APPLICATION_MASTER_TOKEN_ENV_NAME);
- LOG.info("AppMasterToken is " + tokenURLEncodedStr);
- Token<? extends TokenIdentifier> token = new Token<TokenIdentifier>();
- token.decodeFromUrlString(tokenURLEncodedStr);
- currentUser.addToken(token);
+ Credentials credentials = new Credentials();
+ DataInputByteBuffer buf = new DataInputByteBuffer();
+ containerManager.amTokens.rewind();
+ buf.reset(containerManager.amTokens);
+ credentials.readTokenStorageStream(buf);
+ currentUser.addCredentials(credentials);
AMRMProtocol client = currentUser
.doAs(new PrivilegedAction<AMRMProtocol>() {
@@ -172,7 +177,7 @@ public class TestAMAuthorization {
@Test
public void testUnauthorizedAccess() throws Exception {
MyContainerManager containerManager = new MyContainerManager();
- MockRM rm = new MockRMWithAMS(new Configuration(), containerManager);
+ MockRM rm = new MockRMWithAMS(confWithSecurityEnabled, containerManager);
rm.start();
MockNM nm1 = rm.registerNode("localhost:1234", 5120);
@@ -182,17 +187,16 @@ public class TestAMAuthorization {
nm1.nodeHeartbeat(true);
int waitCount = 0;
- while (containerManager.amContainerEnv == null && waitCount++ < 20) {
+ while (containerManager.amTokens == null && waitCount++ < 20) {
LOG.info("Waiting for AM Launch to happen..");
Thread.sleep(1000);
}
- Assert.assertNotNull(containerManager.amContainerEnv);
+ Assert.assertNotNull(containerManager.amTokens);
RMAppAttempt attempt = app.getCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
waitForLaunchedState(attempt);
- // Create a client to the RM.
final Configuration conf = rm.getConfig();
final YarnRPC rpc = YarnRPC.create(conf);
final InetSocketAddress serviceAddr = conf.getSocketAddr(
@@ -202,13 +206,8 @@ public class TestAMAuthorization {
UserGroupInformation currentUser = UserGroupInformation
.createRemoteUser(applicationAttemptId.toString());
- String tokenURLEncodedStr = containerManager.amContainerEnv
- .get(ApplicationConstants.APPLICATION_MASTER_TOKEN_ENV_NAME);
- LOG.info("AppMasterToken is " + tokenURLEncodedStr);
- Token<? extends TokenIdentifier> token = new Token<TokenIdentifier>();
- token.decodeFromUrlString(tokenURLEncodedStr);
- currentUser.addToken(token);
+ // First try contacting NM without tokens
AMRMProtocol client = currentUser
.doAs(new PrivilegedAction<AMRMProtocol>() {
@Override
@@ -217,9 +216,39 @@ public class TestAMAuthorization {
serviceAddr, conf);
}
});
-
RegisterApplicationMasterRequest request = Records
.newRecord(RegisterApplicationMasterRequest.class);
+ request.setApplicationAttemptId(applicationAttemptId);
+ try {
+ client.registerApplicationMaster(request);
+ Assert.fail("Should fail with authorization error");
+ } catch (Exception e) {
+ // Because there are no tokens, the request should be rejected as the
+ // server side will assume we are trying simple auth.
+ Assert.assertTrue(e.getCause().getMessage().contains(
+ "SIMPLE authentication is not enabled. "
+ + "Available:[KERBEROS, DIGEST]"));
+ }
+
+ // Now try to validate invalid authorization.
+ Credentials credentials = new Credentials();
+ DataInputByteBuffer buf = new DataInputByteBuffer();
+ containerManager.amTokens.rewind();
+ buf.reset(containerManager.amTokens);
+ credentials.readTokenStorageStream(buf);
+ currentUser.addCredentials(credentials);
+
+ // Create a client to the RM.
+ client = currentUser
+ .doAs(new PrivilegedAction<AMRMProtocol>() {
+ @Override
+ public AMRMProtocol run() {
+ return (AMRMProtocol) rpc.getProxy(AMRMProtocol.class,
+ serviceAddr, conf);
+ }
+ });
+
+ request = Records.newRecord(RegisterApplicationMasterRequest.class);
ApplicationAttemptId otherAppAttemptId = BuilderUtils
.newApplicationAttemptId(applicationAttemptId.getApplicationId(), 42);
request.setApplicationAttemptId(otherAppAttemptId);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Fri Apr 26 00:55:45 2013
@@ -1520,4 +1520,28 @@ public class TestFairScheduler {
}
assertEquals(FinalApplicationStatus.FAILED, application.getFinalApplicationStatus());
}
+
+ @Test
+ public void testReservationThatDoesntFit() {
+ RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024));
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ ApplicationAttemptId attId = createSchedulingRequest(2048, "queue1",
+ "user1", 1);
+ scheduler.update();
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
+ scheduler.handle(updateEvent);
+
+ FSSchedulerApp app = scheduler.applications.get(attId);
+ assertEquals(0, app.getLiveContainers().size());
+ assertEquals(0, app.getReservedContainers().size());
+
+ createSchedulingRequestExistingApplication(1024, 2, attId);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+
+ assertEquals(1, app.getLiveContainers().size());
+ assertEquals(0, app.getReservedContainers().size());
+ }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestApplicationTokens.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestApplicationTokens.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestApplicationTokens.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestApplicationTokens.java Fri Apr 26 00:55:45 2013
@@ -25,11 +25,11 @@ import javax.crypto.SecretKey;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.io.DataInputByteBuffer;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.api.AMRMProtocol;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
@@ -51,6 +51,14 @@ public class TestApplicationTokens {
private static final Log LOG = LogFactory.getLog(TestApplicationTokens.class);
+ private static final Configuration confWithSecurityEnabled =
+ new Configuration();
+ static {
+ confWithSecurityEnabled.set(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ UserGroupInformation.setConfiguration(confWithSecurityEnabled);
+ }
+
/**
* Validate that application tokens are unusable after the
* application-finishes.
@@ -61,7 +69,8 @@ public class TestApplicationTokens {
public void testTokenExpiry() throws Exception {
MyContainerManager containerManager = new MyContainerManager();
- final MockRM rm = new MockRMWithAMS(new Configuration(), containerManager);
+ final MockRM rm =
+ new MockRMWithAMS(confWithSecurityEnabled, containerManager);
rm.start();
final Configuration conf = rm.getConfig();
@@ -76,11 +85,11 @@ public class TestApplicationTokens {
nm1.nodeHeartbeat(true);
int waitCount = 0;
- while (containerManager.amContainerEnv == null && waitCount++ < 20) {
+ while (containerManager.amTokens == null && waitCount++ < 20) {
LOG.info("Waiting for AM Launch to happen..");
Thread.sleep(1000);
}
- Assert.assertNotNull(containerManager.amContainerEnv);
+ Assert.assertNotNull(containerManager.amTokens);
RMAppAttempt attempt = app.getCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
@@ -89,13 +98,12 @@ public class TestApplicationTokens {
UserGroupInformation currentUser =
UserGroupInformation
.createRemoteUser(applicationAttemptId.toString());
- String tokenURLEncodedStr =
- containerManager.amContainerEnv
- .get(ApplicationConstants.APPLICATION_MASTER_TOKEN_ENV_NAME);
- LOG.info("AppMasterToken is " + tokenURLEncodedStr);
- Token<? extends TokenIdentifier> token = new Token<TokenIdentifier>();
- token.decodeFromUrlString(tokenURLEncodedStr);
- currentUser.addToken(token);
+ Credentials credentials = new Credentials();
+ DataInputByteBuffer buf = new DataInputByteBuffer();
+ containerManager.amTokens.rewind();
+ buf.reset(containerManager.amTokens);
+ credentials.readTokenStorageStream(buf);
+ currentUser.addCredentials(credentials);
rmClient = createRMClient(rm, conf, rpc, currentUser);
@@ -152,9 +160,9 @@ public class TestApplicationTokens {
@Test
public void testMasterKeyRollOver() throws Exception {
- Configuration config = new Configuration();
MyContainerManager containerManager = new MyContainerManager();
- final MockRM rm = new MockRMWithAMS(config, containerManager);
+ final MockRM rm =
+ new MockRMWithAMS(confWithSecurityEnabled, containerManager);
rm.start();
final Configuration conf = rm.getConfig();
@@ -169,11 +177,11 @@ public class TestApplicationTokens {
nm1.nodeHeartbeat(true);
int waitCount = 0;
- while (containerManager.amContainerEnv == null && waitCount++ < 20) {
+ while (containerManager.amTokens == null && waitCount++ < 20) {
LOG.info("Waiting for AM Launch to happen..");
Thread.sleep(1000);
}
- Assert.assertNotNull(containerManager.amContainerEnv);
+ Assert.assertNotNull(containerManager.amTokens);
RMAppAttempt attempt = app.getCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
@@ -182,13 +190,12 @@ public class TestApplicationTokens {
UserGroupInformation currentUser =
UserGroupInformation
.createRemoteUser(applicationAttemptId.toString());
- String tokenURLEncodedStr =
- containerManager.amContainerEnv
- .get(ApplicationConstants.APPLICATION_MASTER_TOKEN_ENV_NAME);
- LOG.info("AppMasterToken is " + tokenURLEncodedStr);
- Token<? extends TokenIdentifier> token = new Token<TokenIdentifier>();
- token.decodeFromUrlString(tokenURLEncodedStr);
- currentUser.addToken(token);
+ Credentials credentials = new Credentials();
+ DataInputByteBuffer buf = new DataInputByteBuffer();
+ containerManager.amTokens.rewind();
+ buf.reset(containerManager.amTokens);
+ credentials.readTokenStorageStream(buf);
+ currentUser.addCredentials(credentials);
rmClient = createRMClient(rm, conf, rpc, currentUser);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java?rev=1476006&r1=1476005&r2=1476006&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java Fri Apr 26 00:55:45 2013
@@ -293,14 +293,14 @@ public class TestRMWebServices extends J
WebServicesTestUtils.checkStringMatch("hadoopVersionBuiltOn",
VersionInfo.getDate(), hadoopVersionBuiltOn);
- WebServicesTestUtils.checkStringMatch("hadoopBuildVersion",
+ WebServicesTestUtils.checkStringEqual("hadoopBuildVersion",
VersionInfo.getBuildVersion(), hadoopBuildVersion);
WebServicesTestUtils.checkStringMatch("hadoopVersion",
VersionInfo.getVersion(), hadoopVersion);
WebServicesTestUtils.checkStringMatch("resourceManagerVersionBuiltOn",
YarnVersionInfo.getDate(), resourceManagerVersionBuiltOn);
- WebServicesTestUtils.checkStringMatch("resourceManagerBuildVersion",
+ WebServicesTestUtils.checkStringEqual("resourceManagerBuildVersion",
YarnVersionInfo.getBuildVersion(), resourceManagerBuildVersion);
WebServicesTestUtils.checkStringMatch("resourceManagerVersion",
YarnVersionInfo.getVersion(), resourceManagerVersion);