You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2020/06/05 13:00:57 UTC
[hadoop] branch trunk updated: YARN-10292. FS-CS converter: add an
option to enable asynchronous scheduling in CapacityScheduler. Contributed
by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 8abff51 YARN-10292. FS-CS converter: add an option to enable asynchronous scheduling in CapacityScheduler. Contributed by Benjamin Teke
8abff51 is described below
commit 8abff5151a7c157355930475dbea2dc535d51c19
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Fri Jun 5 15:00:39 2020 +0200
YARN-10292. FS-CS converter: add an option to enable asynchronous scheduling in CapacityScheduler. Contributed by Benjamin Teke
---
.../fair/converter/ConversionOptions.java | 9 +++++
.../FSConfigToCSConfigArgumentHandler.java | 5 +++
.../converter/FSConfigToCSConfigConverter.java | 3 +-
.../fair/converter/FSYarnSiteConverter.java | 6 +++-
.../TestFSConfigToCSConfigArgumentHandler.java | 31 ++++++++++++++++
.../converter/TestFSConfigToCSConfigConverter.java | 35 ++++++++++++++++++
.../fair/converter/TestFSYarnSiteConverter.java | 42 ++++++++++++++++++----
7 files changed, 122 insertions(+), 9 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/ConversionOptions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/ConversionOptions.java
index 7fec0a8..aae1d55 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/ConversionOptions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/ConversionOptions.java
@@ -22,6 +22,7 @@ public class ConversionOptions {
private DryRunResultHolder dryRunResultHolder;
private boolean dryRun;
private boolean noTerminalRuleCheck;
+ private boolean enableAsyncScheduler;
public ConversionOptions(DryRunResultHolder dryRunResultHolder,
boolean dryRun) {
@@ -41,6 +42,14 @@ public class ConversionOptions {
return noTerminalRuleCheck;
}
+ public void setEnableAsyncScheduler(boolean enableAsyncScheduler) {
+ this.enableAsyncScheduler = enableAsyncScheduler;
+ }
+
+ public boolean isEnableAsyncScheduler() {
+ return enableAsyncScheduler;
+ }
+
public void handleWarning(String msg, Logger log) {
if (dryRun) {
dryRunResultHolder.addDryRunWarning(msg);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java
index 5bd3b1a..c2554a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java
@@ -109,6 +109,9 @@ public class FSConfigToCSConfigArgumentHandler {
SKIP_VERIFICATION("skip verification", "s",
"skip-verification",
"Skips the verification of the converted configuration", false),
+ ENABLE_ASYNC_SCHEDULER("enable asynchronous scheduler", "a", "enable-async-scheduler",
+ "Enables the Asynchronous scheduler which decouples the CapacityScheduler" +
+ " scheduling from Node Heartbeats.", false),
HELP("help", "h", "help", "Displays the list of options", false);
private final String name;
@@ -220,6 +223,8 @@ public class FSConfigToCSConfigArgumentHandler {
conversionOptions.setDryRun(dryRun);
conversionOptions.setNoTerminalRuleCheck(
cliParser.hasOption(CliOption.NO_TERMINAL_RULE_CHECK.shortSwitch));
+ conversionOptions.setEnableAsyncScheduler(
+ cliParser.hasOption(CliOption.ENABLE_ASYNC_SCHEDULER.shortSwitch));
checkOptionPresent(cliParser, CliOption.YARN_SITE);
checkOutputDefined(cliParser, dryRun);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java
index 368bc1f..5acf3567 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java
@@ -270,7 +270,8 @@ public class FSConfigToCSConfigConverter {
FSYarnSiteConverter siteConverter =
new FSYarnSiteConverter();
siteConverter.convertSiteProperties(inputYarnSiteConfig,
- convertedYarnSiteConfig, drfUsed);
+ convertedYarnSiteConfig, drfUsed,
+ conversionOptions.isEnableAsyncScheduler());
// See docs: "allow-undeclared-pools" and "user-as-default-queue" are
// ignored if we have placement rules
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java
index 8e5f92a..86e4cd3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java
@@ -38,7 +38,7 @@ public class FSYarnSiteConverter {
@SuppressWarnings({"deprecation", "checkstyle:linelength"})
public void convertSiteProperties(Configuration conf,
- Configuration yarnSiteConfig, boolean drfUsed) {
+ Configuration yarnSiteConfig, boolean drfUsed, boolean enableAsyncScheduler) {
yarnSiteConfig.set(YarnConfiguration.RM_SCHEDULER,
CapacityScheduler.class.getCanonicalName());
@@ -131,6 +131,10 @@ public class FSYarnSiteConverter {
CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
DominantResourceCalculator.class.getCanonicalName());
}
+
+ if (enableAsyncScheduler) {
+ yarnSiteConfig.setBoolean(CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, true);
+ }
}
public boolean isPreemptionEnabled() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java
index 9f41d31..c3f380e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java
@@ -651,4 +651,35 @@ public class TestFSConfigToCSConfigArgumentHandler {
verifyZeroInteractions(mockValidator);
}
+
+ @Test
+ public void testEnabledAsyncScheduling() throws Exception {
+ setupFSConfigConversionFiles(true);
+
+ FSConfigToCSConfigArgumentHandler argumentHandler =
+ new FSConfigToCSConfigArgumentHandler(conversionOptions, mockValidator);
+
+ String[] args = getArgumentsAsArrayWithDefaults("-f",
+ FSConfigConverterTestCommons.FS_ALLOC_FILE, "-p",
+ "-a");
+ argumentHandler.parseAndConvert(args);
+
+ assertTrue("-a switch had no effect",
+ conversionOptions.isEnableAsyncScheduler());
+ }
+
+ @Test
+ public void testDisabledAsyncScheduling() throws Exception {
+ setupFSConfigConversionFiles(true);
+
+ FSConfigToCSConfigArgumentHandler argumentHandler =
+ new FSConfigToCSConfigArgumentHandler(conversionOptions, mockValidator);
+
+ String[] args = getArgumentsAsArrayWithDefaults("-f",
+ FSConfigConverterTestCommons.FS_ALLOC_FILE, "-p");
+ argumentHandler.parseAndConvert(args);
+
+ assertFalse("-a switch wasn't provided but async scheduling option is true",
+ conversionOptions.isEnableAsyncScheduler());
+ }
}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
index dd3e6d4..46e1fb3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
@@ -717,6 +717,41 @@ public class TestFSConfigToCSConfigConverter {
any(Boolean.class));
}
+ @Test
+ public void testConversionWhenAsyncSchedulingIsEnabled()
+ throws Exception {
+ boolean schedulingEnabledValue = testConversionWithAsyncSchedulingOption(true);
+ assertTrue("Asynchronous scheduling should be true", schedulingEnabledValue);
+ }
+
+ @Test
+ public void testConversionWhenAsyncSchedulingIsDisabled() throws Exception {
+ boolean schedulingEnabledValue = testConversionWithAsyncSchedulingOption(false);
+ assertEquals("Asynchronous scheduling should be the default value",
+ CapacitySchedulerConfiguration.DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE,
+ schedulingEnabledValue);
+ }
+
+ private boolean testConversionWithAsyncSchedulingOption(boolean enabled) throws Exception {
+ FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder()
+ .withClusterResource(CLUSTER_RESOURCE_STRING)
+ .withFairSchedulerXmlConfig(FAIR_SCHEDULER_XML)
+ .build();
+
+ ConversionOptions conversionOptions = createDefaultConversionOptions();
+ conversionOptions.setEnableAsyncScheduler(enabled);
+
+ converter = new FSConfigToCSConfigConverter(ruleHandler,
+ conversionOptions);
+
+ converter.convert(params);
+
+ Configuration convertedConfig = converter.getYarnSiteConfig();
+
+ return convertedConfig.getBoolean(CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE,
+ CapacitySchedulerConfiguration.DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE);
+ }
+
private Configuration getConvertedCSConfig(String dir) throws IOException {
File capacityFile = new File(dir, "capacity-scheduler.xml");
ByteArrayInputStream input =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java
index 1597d50..9cebf16 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java
@@ -26,6 +26,7 @@ import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/**
@@ -52,7 +53,8 @@ public class TestFSYarnSiteConverter {
yarnConfig.setInt(
FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_SLEEP_MS, 666);
- converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false);
+ converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false,
+ false);
assertTrue("Cont. scheduling", yarnConvertedConfig.getBoolean(
CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, false));
@@ -70,7 +72,8 @@ public class TestFSYarnSiteConverter {
FairSchedulerConfiguration.WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS,
321);
- converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false);
+ converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false,
+ false);
assertTrue("Preemption enabled",
yarnConvertedConfig.getBoolean(
@@ -90,7 +93,8 @@ public class TestFSYarnSiteConverter {
public void testSiteAssignMultipleConversion() {
yarnConfig.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true);
- converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false);
+ converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false,
+ false);
assertTrue("Assign multiple",
yarnConvertedConfig.getBoolean(
@@ -102,7 +106,8 @@ public class TestFSYarnSiteConverter {
public void testSiteMaxAssignConversion() {
yarnConfig.setInt(FairSchedulerConfiguration.MAX_ASSIGN, 111);
- converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false);
+ converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false,
+ false);
assertEquals("Max assign", 111,
yarnConvertedConfig.getInt(
@@ -116,7 +121,8 @@ public class TestFSYarnSiteConverter {
yarnConfig.set(FairSchedulerConfiguration.LOCALITY_THRESHOLD_RACK,
"321.321");
- converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false);
+ converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false,
+ false);
assertEquals("Locality threshold node", "123.123",
yarnConvertedConfig.get(
@@ -128,7 +134,8 @@ public class TestFSYarnSiteConverter {
@Test
public void testSiteDrfEnabledConversion() {
- converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, true);
+ converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, true,
+ false);
assertEquals("Resource calculator type", DominantResourceCalculator.class,
yarnConvertedConfig.getClass(
@@ -137,11 +144,32 @@ public class TestFSYarnSiteConverter {
@Test
public void testSiteDrfDisabledConversion() {
- converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false);
+ converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false,
+ false);
assertEquals("Resource calculator type", DefaultResourceCalculator.class,
yarnConvertedConfig.getClass(
CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
CapacitySchedulerConfiguration.DEFAULT_RESOURCE_CALCULATOR_CLASS));
}
+
+ @Test
+ public void testAsyncSchedulingEnabledConversion() {
+ converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, true,
+ true);
+
+ assertTrue("Asynchronous scheduling", yarnConvertedConfig.getBoolean(
+ CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE,
+ CapacitySchedulerConfiguration.DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE));
+ }
+
+ @Test
+ public void testAsyncSchedulingDisabledConversion() {
+ converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false,
+ false);
+
+ assertFalse("Asynchronous scheduling", yarnConvertedConfig.getBoolean(
+ CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE,
+ CapacitySchedulerConfiguration.DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org