You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2018/02/14 15:20:35 UTC
hadoop git commit: YARN-7789. Should fail RM if 3rd resource type is
configured but RM uses DefaultResourceCalculator. (Zian Chen via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk 60971b819 -> 042ef2fa7
YARN-7789. Should fail RM if 3rd resource type is configured but RM uses DefaultResourceCalculator. (Zian Chen via wangda)
Change-Id: I1f6a451fe16758def6f47c046a9b8a67ad7255af
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/042ef2fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/042ef2fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/042ef2fa
Branch: refs/heads/trunk
Commit: 042ef2fa7bcc22e3ca4eb8205c34d83e594bc7de
Parents: 60971b8
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Feb 14 23:11:10 2018 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Feb 14 23:11:10 2018 +0800
----------------------------------------------------------------------
.../scheduler/capacity/CapacityScheduler.java | 8 +++++
...CapacitySchedulerWithMultiResourceTypes.java | 37 ++++++++++++++++++++
2 files changed, 45 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042ef2fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index d3aa5cb..cd9d1373 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -338,6 +338,14 @@ public class CapacityScheduler extends
this.minimumAllocation = super.getMinimumAllocation();
initMaximumResourceCapability(super.getMaximumAllocation());
this.calculator = this.conf.getResourceCalculator();
+ if (this.calculator instanceof DefaultResourceCalculator
+ && ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
+ throw new YarnRuntimeException("RM uses DefaultResourceCalculator which"
+ + " used only memory as resource-type but invalid resource-types"
+ + " specified " + ResourceUtils.getResourceTypes() + ". Use"
+ + " DomainantResourceCalculator instead to make effective use of"
+ + " these resource-types");
+ }
this.usePortForNodeName = this.conf.getUsePortForNodeName();
this.applications = new ConcurrentHashMap<>();
this.labelManager = rmContext.getNodeLabelManager();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042ef2fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
index 2da2cdd..ea29f7f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -38,6 +39,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.junit.Assert;
@@ -286,4 +288,39 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
rm.close();
}
+
+ @Test
+ public void testDefaultResourceCalculatorWithThirdResourceTypes() throws Exception {
+
+ CapacitySchedulerConfiguration csconf =
+ new CapacitySchedulerConfiguration();
+ csconf.setResourceComparator(DefaultResourceCalculator.class);
+
+ YarnConfiguration conf = new YarnConfiguration(csconf);
+
+ String[] res1 = {"resource1", "M"};
+ String[] res2 = {"resource2", "G"};
+ String[] res3 = {"resource3", "H"};
+
+ String[][] test = {res1, res2, res3};
+
+ String resSt = "";
+ for (String[] resources : test) {
+ resSt += (resources[0] + ",");
+ }
+ resSt = resSt.substring(0, resSt.length() - 1);
+ conf.set(YarnConfiguration.RESOURCE_TYPES, resSt);
+
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ boolean exception = false;
+ try {
+ MockRM rm = new MockRM(conf);
+ } catch (YarnRuntimeException e) {
+ exception = true;
+ }
+
+ Assert.assertTrue("Should have exception in CS", exception);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org