You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2020/01/28 14:23:02 UTC
[hadoop] branch trunk updated: YARN-10085. FS-CS converter: remove
mixed ordering policy check. Contributed by Peter Bacsko
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new ca29768 YARN-10085. FS-CS converter: remove mixed ordering policy check. Contributed by Peter Bacsko
ca29768 is described below
commit ca297680352b7bf20ca6cd98ff81bb68e1358c5b
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Tue Jan 28 15:22:12 2020 +0100
YARN-10085. FS-CS converter: remove mixed ordering policy check. Contributed by Peter Bacsko
---
.../converter/FSConfigToCSConfigConverter.java | 61 +++++++++++++++-----
.../converter/FSConfigToCSConfigRuleHandler.java | 12 ++++
.../scheduler/fair/converter/FSQueueConverter.java | 27 +++------
.../fair/converter/FSQueueConverterBuilder.java | 6 ++
.../fair/converter/FSYarnSiteConverter.java | 9 ++-
.../fair/converter/QueuePlacementConverter.java | 3 -
.../converter/TestFSConfigToCSConfigConverter.java | 67 +++++++++++++++-------
.../TestFSConfigToCSConfigRuleHandler.java | 5 ++
.../fair/converter/TestFSQueueConverter.java | 20 +++++++
.../fair/converter/TestFSYarnSiteConverter.java | 33 +++++++++--
...mixed.xml => fair-scheduler-onlyfairpolicy.xml} | 19 +++---
.../fair-scheduler-orderingpolicy-mixed.xml | 8 +--
12 files changed, 195 insertions(+), 75 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java
index a8fdb51..eae952d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java
@@ -23,6 +23,7 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
+import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
@@ -39,9 +40,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationCo
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfigurationException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
-import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -73,6 +75,7 @@ public class FSConfigToCSConfigConverter {
private boolean sizeBasedWeight = false;
private boolean userAsDefaultQueue = false;
private ConversionOptions conversionOptions;
+ private boolean drfUsed = false;
private Configuration yarnSiteConfig;
private Configuration capacitySchedulerConfig;
@@ -198,7 +201,7 @@ public class FSConfigToCSConfigConverter {
@VisibleForTesting
void convert(Configuration conf) throws Exception {
System.out.println(WARNING_TEXT);
-
+
// initialize Fair Scheduler
RMContext ctx = new RMContextImpl();
PlacementManager placementManager = new PlacementManager();
@@ -208,6 +211,8 @@ public class FSConfigToCSConfigConverter {
fs.setRMContext(ctx);
fs.init(conf);
+ drfUsed = isDrfUsed(fs);
+
AllocationConfiguration allocConf = fs.getAllocationConfiguration();
queueMaxAppsDefault = allocConf.getQueueMaxAppsDefault();
queueMaxAMShareDefault = allocConf.getQueueMaxAMShareDefault();
@@ -246,7 +251,7 @@ public class FSConfigToCSConfigConverter {
private void convertYarnSiteXml(Configuration conf) {
FSYarnSiteConverter siteConverter =
new FSYarnSiteConverter();
- siteConverter.convertSiteProperties(conf, yarnSiteConfig);
+ siteConverter.convertSiteProperties(conf, yarnSiteConfig, drfUsed);
autoCreateChildQueues = siteConverter.isAutoCreateChildQueues();
preemptionEnabled = siteConverter.isPreemptionEnabled();
@@ -271,6 +276,7 @@ public class FSConfigToCSConfigConverter {
.withQueueMaxAMShareDefault(queueMaxAMShareDefault)
.withQueueMaxAppsDefault(queueMaxAppsDefault)
.withConversionOptions(conversionOptions)
+ .withDrfUsed(drfUsed)
.build();
queueConverter.convertQueueHierarchy(rootQueue);
@@ -287,18 +293,6 @@ public class FSConfigToCSConfigConverter {
ruleHandler, userAsDefaultQueue);
properties.forEach((k, v) -> capacitySchedulerConfig.set(k, v));
}
-
- // Validate ordering policy
- if (queueConverter.isDrfPolicyUsedOnQueueLevel()) {
- if (queueConverter.isFifoOrFairSharePolicyUsed()) {
- throw new ConversionException(
- "DRF ordering policy cannot be used together with fifo/fair");
- } else {
- capacitySchedulerConfig.set(
- CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
- DominantResourceCalculator.class.getCanonicalName());
- }
- }
}
private void emitDefaultMaxApplications() {
@@ -359,6 +353,38 @@ public class FSConfigToCSConfigConverter {
}
}
+ private boolean isDrfUsed(FairScheduler fs) {
+ FSQueue rootQueue = fs.getQueueManager().getRootQueue();
+ AllocationConfiguration allocConf = fs.getAllocationConfiguration();
+
+ String defaultPolicy = allocConf.getDefaultSchedulingPolicy().getName();
+
+ if (DominantResourceFairnessPolicy.NAME.equals(defaultPolicy)) {
+ return true;
+ } else {
+ return isDrfUsedOnQueueLevel(rootQueue);
+ }
+ }
+
+ private boolean isDrfUsedOnQueueLevel(FSQueue queue) {
+ String policy = queue.getPolicy().getName();
+ boolean usesDrf = DominantResourceFairnessPolicy.NAME.equals(policy);
+
+ if (usesDrf) {
+ return true;
+ } else {
+ List<FSQueue> children = queue.getChildQueues();
+
+ if (children != null) {
+ for (FSQueue child : children) {
+ usesDrf |= isDrfUsedOnQueueLevel(child);
+ }
+ }
+
+ return usesDrf;
+ }
+ }
+
@VisibleForTesting
Resource getClusterResource() {
return clusterResource;
@@ -373,4 +399,9 @@ public class FSConfigToCSConfigConverter {
FSConfigToCSConfigRuleHandler getRuleHandler() {
return ruleHandler;
}
+
+ @VisibleForTesting
+ Configuration getYarnSiteConfig() {
+ return yarnSiteConfig;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java
index 2088334..a1db393 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java
@@ -73,6 +73,9 @@ public class FSConfigToCSConfigRuleHandler {
public static final String QUEUE_AUTO_CREATE =
"queueAutoCreate.action";
+ public static final String FAIR_AS_DRF =
+ "fairAsDrf.action";
+
@VisibleForTesting
enum RuleAction {
WARNING,
@@ -119,6 +122,7 @@ public class FSConfigToCSConfigRuleHandler {
setActionForProperty(SPECIFIED_NOT_FIRST);
setActionForProperty(RESERVATION_SYSTEM);
setActionForProperty(QUEUE_AUTO_CREATE);
+ setActionForProperty(FAIR_AS_DRF);
}
public void handleMaxCapacityPercentage(String queueName) {
@@ -181,6 +185,14 @@ public class FSConfigToCSConfigRuleHandler {
placementRule));
}
+ public void handleFairAsDrf(String queueName) {
+ handle(FAIR_AS_DRF,
+ null,
+ format(
+ "Queue %s will use DRF policy instead of Fair",
+ queueName));
+ }
+
private void handle(String actionName, String fsSetting, String message) {
RuleAction action = actions.get(actionName);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java
index 1722302..44bdad4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java
@@ -55,9 +55,7 @@ public class FSQueueConverter {
private final float queueMaxAMShareDefault;
private final boolean autoCreateChildQueues;
private final int queueMaxAppsDefault;
-
- private boolean fifoOrFairSharePolicyUsed;
- private boolean drfPolicyUsedOnQueueLevel;
+ private final boolean drfUsed;
private ConversionOptions conversionOptions;
@@ -72,6 +70,7 @@ public class FSQueueConverter {
this.autoCreateChildQueues = builder.autoCreateChildQueues;
this.queueMaxAppsDefault = builder.queueMaxAppsDefault;
this.conversionOptions = builder.conversionOptions;
+ this.drfUsed = builder.drfUsed;
}
public void convertQueueHierarchy(FSQueue queue) {
@@ -105,14 +104,6 @@ public class FSQueueConverter {
}
}
- public boolean isFifoOrFairSharePolicyUsed() {
- return fifoOrFairSharePolicyUsed;
- }
-
- public boolean isDrfPolicyUsedOnQueueLevel() {
- return drfPolicyUsedOnQueueLevel;
- }
-
/**
* Generates yarn.scheduler.capacity.<queue-name>.queues.
* @param queueName
@@ -306,20 +297,20 @@ public class FSQueueConverter {
String policy = queue.getPolicy().getName();
switch (policy) {
+ case DominantResourceFairnessPolicy.NAME:
+ capacitySchedulerConfig.set(PREFIX + queueName
+ + ".ordering-policy", FairSharePolicy.NAME);
+ break;
case FairSharePolicy.NAME:
capacitySchedulerConfig.set(PREFIX + queueName
+ ".ordering-policy", FairSharePolicy.NAME);
- fifoOrFairSharePolicyUsed = true;
+ if (drfUsed) {
+ ruleHandler.handleFairAsDrf(queueName);
+ }
break;
case FifoPolicy.NAME:
capacitySchedulerConfig.set(PREFIX + queueName
+ ".ordering-policy", FifoPolicy.NAME);
- fifoOrFairSharePolicyUsed = true;
- break;
- case DominantResourceFairnessPolicy.NAME:
- // DRF is not supported on a queue level,
- // it has to be global
- drfPolicyUsedOnQueueLevel = true;
break;
default:
String msg = String.format("Unexpected ordering policy " +
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverterBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverterBuilder.java
index 615f1b9..dd5f05c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverterBuilder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverterBuilder.java
@@ -32,6 +32,7 @@ public final class FSQueueConverterBuilder {
float queueMaxAMShareDefault;
int queueMaxAppsDefault;
ConversionOptions conversionOptions;
+ boolean drfUsed;
private FSQueueConverterBuilder() {
}
@@ -94,6 +95,11 @@ public final class FSQueueConverterBuilder {
return this;
}
+ public FSQueueConverterBuilder withDrfUsed(boolean drfUsed) {
+ this.drfUsed = drfUsed;
+ return this;
+ }
+
public FSQueueConverter build() {
return new FSQueueConverter(this);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java
index f6d023d..25db4d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
/**
* Converts a Fair Scheduler site configuration to Capacity Scheduler
@@ -37,7 +38,7 @@ public class FSYarnSiteConverter {
@SuppressWarnings({"deprecation", "checkstyle:linelength"})
public void convertSiteProperties(Configuration conf,
- Configuration yarnSiteConfig) {
+ Configuration yarnSiteConfig, boolean drfUsed) {
yarnSiteConfig.set(YarnConfiguration.RM_SCHEDULER,
CapacityScheduler.class.getCanonicalName());
@@ -139,6 +140,12 @@ public class FSYarnSiteConverter {
FairSchedulerConfiguration.DEFAULT_USER_AS_DEFAULT_QUEUE)) {
userAsDefaultQueue = true;
}
+
+ if (drfUsed) {
+ yarnSiteConfig.set(
+ CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+ DominantResourceCalculator.class.getCanonicalName());
+ }
}
public boolean isPreemptionEnabled() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
index 4641dfc..8d3182f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
@@ -82,7 +82,6 @@ class QueuePlacementConverter {
}
mapping.append("u:" + USER + ":").append(defaultRule.defaultQueueName);
} else if (rule instanceof SecondaryGroupExistingPlacementRule) {
- // TODO: wait for YARN-9840
if (mapping.length() > 0) {
mapping.append(";");
}
@@ -107,10 +106,8 @@ class QueuePlacementConverter {
mapping.append(";");
}
if (pr instanceof PrimaryGroupPlacementRule) {
- // TODO: wait for YARN-9841
mapping.append("u:" + USER + ":" + PRIMARY_GROUP + "." + USER);
} else if (pr instanceof SecondaryGroupExistingPlacementRule) {
- // TODO: wait for YARN-9865
mapping.append("u:" + USER + ":" + SECONDARY_GROUP + "." + USER);
} else if (pr instanceof DefaultPlacementRule) {
DefaultPlacementRule defaultRule = (DefaultPlacementRule) pr;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
index 4525dae..dc51db9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
@@ -60,6 +61,8 @@ import org.mockito.junit.MockitoJUnitRunner;
*/
@RunWith(MockitoJUnitRunner.class)
public class TestFSConfigToCSConfigConverter {
+ private static final String CLUSTER_RESOURCE_STRING =
+ "vcores=20, memory-mb=240";
private static final Resource CLUSTER_RESOURCE =
Resource.newInstance(16384, 16);
private static final String FILE_PREFIX = "file:";
@@ -67,6 +70,10 @@ public class TestFSConfigToCSConfigConverter {
prepareFileName("fair-scheduler-conversion.xml");
private static final String FS_INVALID_PLACEMENT_RULES_XML =
prepareFileName("fair-scheduler-invalidplacementrules.xml");
+ private static final String FS_ONLY_FAIR_POLICY_XML =
+ prepareFileName("fair-scheduler-onlyfairpolicy.xml");
+ private static final String FS_MIXED_POLICY_XML =
+ prepareFileName("fair-scheduler-orderingpolicy-mixed.xml");
@Mock
private FSConfigToCSConfigRuleHandler ruleHandler;
@@ -216,20 +223,6 @@ public class TestFSConfigToCSConfigConverter {
}
@Test
- public void testMixedQueueOrderingPolicy() throws Exception {
- expectedException.expect(ConversionException.class);
- expectedException.expectMessage(
- "DRF ordering policy cannot be used together with fifo/fair");
- String absolutePath =
- new File("src/test/resources/fair-scheduler-orderingpolicy-mixed.xml")
- .getAbsolutePath();
- config.set(FairSchedulerConfiguration.ALLOCATION_FILE,
- FILE_PREFIX + absolutePath);
-
- converter.convert(config);
- }
-
- @Test
public void testQueueMaxChildCapacityNotSupported() throws Exception {
expectedException.expect(UnsupportedPropertyException.class);
expectedException.expectMessage("test");
@@ -277,7 +270,7 @@ public class TestFSConfigToCSConfigConverter {
@Test
public void testConvertFSConfigurationClusterResource() throws Exception {
FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder()
- .withClusterResource("vcores=20, memory-mb=240")
+ .withClusterResource(CLUSTER_RESOURCE_STRING)
.build();
converter.convert(params);
assertEquals("Resource", Resource.newInstance(240, 20),
@@ -288,7 +281,7 @@ public class TestFSConfigToCSConfigConverter {
public void testConvertFSConfigPctModeUsedAndClusterResourceDefined()
throws Exception {
FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder()
- .withClusterResource("vcores=20, memory-mb=240")
+ .withClusterResource(CLUSTER_RESOURCE_STRING)
.build();
converter.convert(params);
assertEquals("Resource", Resource.newInstance(240, 20),
@@ -394,7 +387,7 @@ public class TestFSConfigToCSConfigConverter {
@Test
public void testConvertCheckOutputDir() throws Exception {
FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder()
- .withClusterResource("vcores=20, memory-mb=240")
+ .withClusterResource(CLUSTER_RESOURCE_STRING)
.build();
converter.convert(params);
@@ -419,7 +412,7 @@ public class TestFSConfigToCSConfigConverter {
throws Exception {
FSConfigToCSConfigConverterParams params =
createParamsBuilder(YARN_SITE_XML_NO_REF_TO_FS_XML)
- .withClusterResource("vcores=20, memory-mb=240")
+ .withClusterResource(CLUSTER_RESOURCE_STRING)
.build();
expectedException.expect(PreconditionException.class);
@@ -430,7 +423,7 @@ public class TestFSConfigToCSConfigConverter {
@Test
public void testInvalidFairSchedulerXml() throws Exception {
FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder()
- .withClusterResource("vcores=20, memory-mb=240")
+ .withClusterResource(CLUSTER_RESOURCE_STRING)
.withFairSchedulerXmlConfig(FAIR_SCHEDULER_XML_INVALID)
.build();
@@ -442,7 +435,7 @@ public class TestFSConfigToCSConfigConverter {
public void testInvalidYarnSiteXml() throws Exception {
FSConfigToCSConfigConverterParams params =
createParamsBuilder(YARN_SITE_XML_INVALID)
- .withClusterResource("vcores=20, memory-mb=240")
+ .withClusterResource(CLUSTER_RESOURCE_STRING)
.build();
expectedException.expect(RuntimeException.class);
@@ -464,7 +457,7 @@ public class TestFSConfigToCSConfigConverter {
public void testConversionWhenInvalidPlacementRulesIgnored()
throws Exception {
FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder()
- .withClusterResource("vcores=20, memory-mb=240")
+ .withClusterResource(CLUSTER_RESOURCE_STRING)
.withFairSchedulerXmlConfig(FS_INVALID_PLACEMENT_RULES_XML)
.build();
@@ -479,6 +472,38 @@ public class TestFSConfigToCSConfigConverter {
// expected: no exception
}
+ @Test
+ public void testConversionWhenOnlyFairPolicyIsUsed() throws Exception {
+ FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder()
+ .withClusterResource(CLUSTER_RESOURCE_STRING)
+ .withFairSchedulerXmlConfig(FS_ONLY_FAIR_POLICY_XML)
+ .build();
+
+ converter.convert(params);
+
+ Configuration convertedConfig = converter.getYarnSiteConfig();
+
+ assertEquals("Resource calculator class shouldn't be set", null,
+ convertedConfig.getClass(
+ CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, null));
+ }
+
+ @Test
+ public void testConversionWhenMixedPolicyIsUsed() throws Exception {
+ FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder()
+ .withClusterResource(CLUSTER_RESOURCE_STRING)
+ .withFairSchedulerXmlConfig(FS_MIXED_POLICY_XML)
+ .build();
+
+ converter.convert(params);
+
+ Configuration convertedConfig = converter.getYarnSiteConfig();
+
+ assertEquals("Resource calculator type", DominantResourceCalculator.class,
+ convertedConfig.getClass(
+ CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, null));
+ }
+
private Configuration getConvertedCSConfig() {
ByteArrayInputStream input =
new ByteArrayInputStream(csConfigOut.toByteArray());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java
index b6caadd..4539e1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java
@@ -25,6 +25,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.conve
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.SPECIFIED_NOT_FIRST;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_APPS_DEFAULT;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_RUNNING_APPS;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.FAIR_AS_DRF;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -86,6 +87,7 @@ public class TestFSConfigToCSConfigRuleHandler {
rules.put(SPECIFIED_NOT_FIRST, WARNING);
rules.put(USER_MAX_APPS_DEFAULT, WARNING);
rules.put(USER_MAX_RUNNING_APPS, WARNING);
+ rules.put(FAIR_AS_DRF, WARNING);
ruleHandler = new FSConfigToCSConfigRuleHandler(rules,
createDefaultConversionOptions());
@@ -111,6 +113,8 @@ public class TestFSConfigToCSConfigRuleHandler {
rules.put(SPECIFIED_NOT_FIRST, ABORT);
rules.put(USER_MAX_APPS_DEFAULT, ABORT);
rules.put(USER_MAX_RUNNING_APPS, ABORT);
+ rules.put(USER_MAX_RUNNING_APPS, ABORT);
+ rules.put(FAIR_AS_DRF, ABORT);
rules.put(MAX_CHILD_QUEUE_LIMIT, "1");
ruleHandler = new FSConfigToCSConfigRuleHandler(rules,
@@ -126,6 +130,7 @@ public class TestFSConfigToCSConfigRuleHandler {
expectAbort(() -> ruleHandler.handleSpecifiedNotFirstRule());
expectAbort(() -> ruleHandler.handleUserMaxApps());
expectAbort(() -> ruleHandler.handleUserMaxAppsDefault());
+ expectAbort(() -> ruleHandler.handleFairAsDrf("test"));
}
@Test(expected = ConversionException.class)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java
index 3507c83..2221364 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java
@@ -20,6 +20,9 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.C
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
import java.io.File;
import java.io.IOException;
@@ -401,6 +404,23 @@ public class TestFSQueueConverter {
}
@Test
+ public void testQueueUnsupportedMixedOrderingPolicy() throws IOException {
+ converter = builder.withDrfUsed(true).build();
+ String absolutePath =
+ new File("src/test/resources/fair-scheduler-orderingpolicy-mixed.xml")
+ .getAbsolutePath();
+ config.set(FairSchedulerConfiguration.ALLOCATION_FILE,
+ FILE_PREFIX + absolutePath);
+ fs.close();
+ fs = createFairScheduler();
+ rootQueue = fs.getQueueManager().getRootQueue();
+
+ converter.convertQueueHierarchy(rootQueue);
+
+ verify(ruleHandler, times(6)).handleFairAsDrf(anyString());
+ }
+
+ @Test
public void testQueueMaxChildCapacityNotSupported() {
converter = builder.build();
expectedException.expect(UnsupportedPropertyException.class);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java
index edb9a4e..0bd8fb1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java
@@ -20,6 +20,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.junit.Before;
import org.junit.Test;
@@ -50,7 +52,7 @@ public class TestFSYarnSiteConverter {
yarnConfig.setInt(
FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_SLEEP_MS, 666);
- converter.convertSiteProperties(yarnConfig, yarnConvertedConfig);
+ converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false);
assertTrue("Cont. scheduling", yarnConvertedConfig.getBoolean(
CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, false));
@@ -65,7 +67,7 @@ public class TestFSYarnSiteConverter {
yarnConfig.setInt("yarn.resource-types.memory-mb.increment-allocation", 11);
yarnConfig.setInt("yarn.resource-types.vcores.increment-allocation", 5);
- converter.convertSiteProperties(yarnConfig, yarnConvertedConfig);
+ converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false);
assertEquals("Memory alloc increment", 11,
yarnConvertedConfig.getInt("yarn.scheduler.minimum-allocation-mb",
@@ -83,7 +85,7 @@ public class TestFSYarnSiteConverter {
FairSchedulerConfiguration.WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS,
321);
- converter.convertSiteProperties(yarnConfig, yarnConvertedConfig);
+ converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false);
assertTrue("Preemption enabled",
yarnConvertedConfig.getBoolean(
@@ -103,7 +105,7 @@ public class TestFSYarnSiteConverter {
public void testSiteAssignMultipleConversion() {
yarnConfig.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true);
- converter.convertSiteProperties(yarnConfig, yarnConvertedConfig);
+ converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false);
assertTrue("Assign multiple",
yarnConvertedConfig.getBoolean(
@@ -115,7 +117,7 @@ public class TestFSYarnSiteConverter {
public void testSiteMaxAssignConversion() {
yarnConfig.setInt(FairSchedulerConfiguration.MAX_ASSIGN, 111);
- converter.convertSiteProperties(yarnConfig, yarnConvertedConfig);
+ converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false);
assertEquals("Max assign", 111,
yarnConvertedConfig.getInt(
@@ -129,7 +131,7 @@ public class TestFSYarnSiteConverter {
yarnConfig.set(FairSchedulerConfiguration.LOCALITY_THRESHOLD_RACK,
"321.321");
- converter.convertSiteProperties(yarnConfig, yarnConvertedConfig);
+ converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false);
assertEquals("Locality threshold node", "123.123",
yarnConvertedConfig.get(
@@ -138,4 +140,23 @@ public class TestFSYarnSiteConverter {
yarnConvertedConfig.get(
CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY));
}
+
+ @Test
+ public void testSiteDrfEnabledConversion() {
+ converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, true);
+
+ assertEquals("Resource calculator type", DominantResourceCalculator.class,
+ yarnConvertedConfig.getClass(
+ CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, null));
+ }
+
+ @Test
+ public void testSiteDrfDisabledConversion() {
+ converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false);
+
+ assertEquals("Resource calculator type", DefaultResourceCalculator.class,
+ yarnConvertedConfig.getClass(
+ CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+ CapacitySchedulerConfiguration.DEFAULT_RESOURCE_CALCULATOR_CLASS));
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-onlyfairpolicy.xml
similarity index 86%
copy from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml
copy to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-onlyfairpolicy.xml
index 3a2a593..e03be5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-onlyfairpolicy.xml
@@ -19,7 +19,7 @@
<allocations>
<queue name="root">
<weight>1.0</weight>
- <schedulingPolicy>drf</schedulingPolicy>
+ <schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>alice,bob,joe,john hadoop_users</aclSubmitApps>
<aclAdministerApps>alice,bob,joe,john hadoop_users</aclAdministerApps>
<queue name="default">
@@ -28,10 +28,10 @@
</queue>
<queue name="users" type="parent">
<weight>1.0</weight>
- <schedulingPolicy>drf</schedulingPolicy>
+ <schedulingPolicy>fair</schedulingPolicy>
<queue name="john">
<weight>1.0</weight>
- <schedulingPolicy>drf</schedulingPolicy>
+ <schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>john </aclSubmitApps>
<aclAdministerApps>john </aclAdministerApps>
<maxContainerAllocation>vcores=2,memory-mb=8192</maxContainerAllocation>
@@ -40,7 +40,7 @@
<maxResources>memory-mb=50.0%, vcores=50.0%</maxResources>
<weight>3.0</weight>
<allowPreemptionFrom>false</allowPreemptionFrom>
- <schedulingPolicy>drf</schedulingPolicy>
+ <schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>joe </aclSubmitApps>
<aclAdministerApps>joe </aclAdministerApps>
</queue>
@@ -55,26 +55,31 @@
<maxRunningApps>2</maxRunningApps>
<weight>3.0</weight>
<allowPreemptionFrom>false</allowPreemptionFrom>
- <schedulingPolicy>drf</schedulingPolicy>
+ <schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>alice </aclSubmitApps>
<aclAdministerApps>alice </aclAdministerApps>
<maxAMShare>0.15</maxAMShare>
+ <reservation>memory-mb=16384, vcores=4</reservation>
</queue>
<queue name="bob">
<maxResources>memory-mb=8192, vcores=2</maxResources>
<weight>1.0</weight>
- <schedulingPolicy>drf</schedulingPolicy>
+ <schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>bob </aclSubmitApps>
<aclAdministerApps>bob </aclAdministerApps>
<maxAMShare>-1.0</maxAMShare>
</queue>
</queue>
</queue>
+ <user name="alice">
+ <maxRunningApps>30</maxRunningApps>
+ </user>
+ <userMaxAppsDefault>10</userMaxAppsDefault>
<defaultFairSharePreemptionTimeout>23</defaultFairSharePreemptionTimeout>
<defaultMinSharePreemptionTimeout>24</defaultMinSharePreemptionTimeout>
<defaultFairSharePreemptionThreshold>0.12</defaultFairSharePreemptionThreshold>
<queueMaxAppsDefault>15</queueMaxAppsDefault>
- <defaultQueueSchedulingPolicy>drf</defaultQueueSchedulingPolicy>
+ <defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>
<queueMaxAMShareDefault>0.16</queueMaxAMShareDefault>
<queuePlacementPolicy>
<rule name="nestedUserQueue" create="false">
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml
index 3a2a593..778fb86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml
@@ -31,7 +31,7 @@
<schedulingPolicy>drf</schedulingPolicy>
<queue name="john">
<weight>1.0</weight>
- <schedulingPolicy>drf</schedulingPolicy>
+ <schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>john </aclSubmitApps>
<aclAdministerApps>john </aclAdministerApps>
<maxContainerAllocation>vcores=2,memory-mb=8192</maxContainerAllocation>
@@ -40,7 +40,7 @@
<maxResources>memory-mb=50.0%, vcores=50.0%</maxResources>
<weight>3.0</weight>
<allowPreemptionFrom>false</allowPreemptionFrom>
- <schedulingPolicy>drf</schedulingPolicy>
+ <schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>joe </aclSubmitApps>
<aclAdministerApps>joe </aclAdministerApps>
</queue>
@@ -55,7 +55,7 @@
<maxRunningApps>2</maxRunningApps>
<weight>3.0</weight>
<allowPreemptionFrom>false</allowPreemptionFrom>
- <schedulingPolicy>drf</schedulingPolicy>
+ <schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>alice </aclSubmitApps>
<aclAdministerApps>alice </aclAdministerApps>
<maxAMShare>0.15</maxAMShare>
@@ -63,7 +63,7 @@
<queue name="bob">
<maxResources>memory-mb=8192, vcores=2</maxResources>
<weight>1.0</weight>
- <schedulingPolicy>drf</schedulingPolicy>
+ <schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>bob </aclSubmitApps>
<aclAdministerApps>bob </aclAdministerApps>
<maxAMShare>-1.0</maxAMShare>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org