You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2021/01/08 12:41:43 UTC
[hadoop] branch trunk updated: YARN-10507. Add the capability to
fs2cs to write the converted placement rules inside capacity-scheduler.xml.
Contributed by Peter Bacsko
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4176759 YARN-10507. Add the capability to fs2cs to write the converted placement rules inside capacity-scheduler.xml. Contributed by Peter Bacsko
4176759 is described below
commit 41767599093f6a39e43be963336af24738616cb1
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Fri Jan 8 13:40:33 2021 +0100
YARN-10507. Add the capability to fs2cs to write the converted placement rules inside capacity-scheduler.xml. Contributed by Peter Bacsko
---
.../FSConfigToCSConfigArgumentHandler.java | 12 ++++
.../converter/FSConfigToCSConfigConverter.java | 78 +++++++++++++++-------
.../converter/FSConfigToCSConfigConverterMain.java | 20 ++++--
.../FSConfigToCSConfigConverterParams.java | 14 +++-
.../fair/converter/VerificationException.java | 3 +
.../TestFSConfigToCSConfigArgumentHandler.java | 37 +++++++++-
.../converter/TestFSConfigToCSConfigConverter.java | 8 +--
.../TestFSConfigToCSConfigConverterMain.java | 52 +++++++++++----
8 files changed, 177 insertions(+), 47 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java
index b6730c7..8f217e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java
@@ -112,6 +112,9 @@ public class FSConfigToCSConfigArgumentHandler {
ENABLE_ASYNC_SCHEDULER("enable asynchronous scheduler", "a", "enable-async-scheduler",
"Enables the Asynchronous scheduler which decouples the CapacityScheduler" +
" scheduling from Node Heartbeats.", false),
+ RULES_TO_FILE("rules to external file", "e", "rules-to-file",
+ "Generates the converted placement rules to an external JSON file " +
+ "called mapping-rules.json", false),
HELP("help", "h", "help", "Displays the list of options", false);
private final String name;
@@ -254,6 +257,13 @@ public class FSConfigToCSConfigArgumentHandler {
checkDirectory(CliOption.OUTPUT_DIR, outputDir);
checkOutputDirDoesNotContainXmls(yarnSiteXmlFile, outputDir);
+ // check mapping-rules.json if we intend to generate it
+ if (!cliParser.hasOption(CliOption.CONSOLE_MODE.shortSwitch) &&
+ cliParser.hasOption(CliOption.RULES_TO_FILE.shortSwitch)) {
+ checkFileNotInOutputDir(new File(outputDir),
+ FSConfigToCSConfigConverter.MAPPING_RULES_JSON);
+ }
+
return FSConfigToCSConfigConverterParams.Builder.create()
.withYarnSiteXmlConfig(yarnSiteXmlFile)
.withFairSchedulerXmlConfig(fairSchedulerXmlFile)
@@ -263,6 +273,8 @@ public class FSConfigToCSConfigArgumentHandler {
.withConsole(cliParser.hasOption(CliOption.CONSOLE_MODE.shortSwitch))
.withOutputDirectory(outputDir)
.withConvertPlacementRules(convertPlacementRules)
+ .withPlacementRulesToFile(
+ cliParser.hasOption(CliOption.RULES_TO_FILE.shortSwitch))
.build();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java
index ce05889..af89657 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java
@@ -17,13 +17,18 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAPPING_RULE_JSON;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT_JSON;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSQueueConverter.QUEUE_MAX_AM_SHARE_DISABLED;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
@@ -55,6 +60,7 @@ import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import com.fasterxml.jackson.core.JsonGenerator;
/**
* Converts Fair Scheduler configuration (site and fair-scheduler.xml)
@@ -64,13 +70,13 @@ import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTest
public class FSConfigToCSConfigConverter {
public static final Logger LOG = LoggerFactory.getLogger(
FSConfigToCSConfigConverter.class.getName());
+ public static final String MAPPING_RULES_JSON =
+ "mapping-rules.json";
private static final String YARN_SITE_XML = "yarn-site.xml";
private static final String CAPACITY_SCHEDULER_XML =
"capacity-scheduler.xml";
private static final String FAIR_SCHEDULER_XML =
"fair-scheduler.xml";
- private static final String MAPPING_RULES_JSON =
- "mapping-rules.json";
public static final String WARNING_TEXT =
"WARNING: This feature is experimental and not intended " +
@@ -99,6 +105,7 @@ public class FSConfigToCSConfigConverter {
private boolean consoleMode = false;
private boolean convertPlacementRules = true;
private String outputDirectory;
+ private boolean rulesToFile;
public FSConfigToCSConfigConverter(FSConfigToCSConfigRuleHandler
ruleHandler, ConversionOptions conversionOptions) {
@@ -106,7 +113,6 @@ public class FSConfigToCSConfigConverter {
this.conversionOptions = conversionOptions;
this.yarnSiteOutputStream = System.out;
this.capacitySchedulerOutputStream = System.out;
- this.mappingRulesOutputStream = System.out;
this.placementConverter = new QueuePlacementConverter();
}
@@ -116,6 +122,7 @@ public class FSConfigToCSConfigConverter {
this.clusterResource = getClusterResource(params);
this.convertPlacementRules = params.isConvertPlacementRules();
this.outputDirectory = params.getOutputDirectory();
+ this.rulesToFile = params.isPlacementRulesToFile();
prepareOutputFiles(params.isConsole());
loadConversionRules(params.getConversionRulesConfig());
Configuration inputYarnSiteConfig = getInputYarnSiteConfig(params);
@@ -127,9 +134,9 @@ public class FSConfigToCSConfigConverter {
private void prepareOutputFiles(boolean console)
throws FileNotFoundException {
if (console) {
- LOG.info("Console mode is enabled, " + YARN_SITE_XML + " and" +
- " " + CAPACITY_SCHEDULER_XML + " will be only emitted " +
- "to the console!");
+ LOG.info("Console mode is enabled, {}, {} and {} will be only emitted " +
+ "to the console!",
+ YARN_SITE_XML, CAPACITY_SCHEDULER_XML, MAPPING_RULES_JSON);
this.consoleMode = true;
return;
}
@@ -253,6 +260,10 @@ public class FSConfigToCSConfigConverter {
convertYarnSiteXml(inputYarnSiteConfig);
convertCapacitySchedulerXml(fs);
+ if (convertPlacementRules) {
+ performRuleConversion(fs);
+ }
+
if (consoleMode) {
System.out.println("======= " + CAPACITY_SCHEDULER_XML + " =======");
}
@@ -263,10 +274,6 @@ public class FSConfigToCSConfigConverter {
System.out.println("======= " + YARN_SITE_XML + " =======");
}
convertedYarnSiteConfig.writeXml(yarnSiteOutputStream);
-
- if (convertPlacementRules) {
- performRuleConversion(fs);
- }
}
private void convertYarnSiteXml(Configuration inputYarnSiteConfig) {
@@ -313,28 +320,58 @@ public class FSConfigToCSConfigConverter {
fs.getRMContext().getQueuePlacementManager();
if (placementManager.getPlacementRules().size() > 0) {
- if (!consoleMode) {
- File mappingRulesFile = new File(outputDirectory,
- MAPPING_RULES_JSON);
- this.mappingRulesOutputStream =
- new FileOutputStream(mappingRulesFile);
- } else {
- System.out.println("======= " + MAPPING_RULES_JSON + " =======");
- }
+ mappingRulesOutputStream = getOutputStreamForJson();
MappingRulesDescription desc =
placementConverter.convertPlacementPolicy(placementManager,
ruleHandler, capacitySchedulerConfig);
ObjectMapper mapper = new ObjectMapper();
+ // close output stream if we write to a file, leave it open otherwise
+ if (!consoleMode && rulesToFile) {
+ mapper.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, true);
+ } else {
+ mapper.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false);
+ }
ObjectWriter writer = mapper.writer(new DefaultPrettyPrinter());
+ if (consoleMode && rulesToFile) {
+ System.out.println("======= " + MAPPING_RULES_JSON + " =======");
+ }
writer.writeValue(mappingRulesOutputStream, desc);
+
+ capacitySchedulerConfig.set(MAPPING_RULE_FORMAT,
+ MAPPING_RULE_FORMAT_JSON);
+ if (!rulesToFile) {
+ String json =
+ ((ByteArrayOutputStream)mappingRulesOutputStream)
+ .toString(StandardCharsets.UTF_8.displayName());
+ capacitySchedulerConfig.set(MAPPING_RULE_JSON, json);
+ }
} else {
LOG.info("No rules to convert");
}
}
+ /*
+ * Console RulesToFile OutputStream
+ * true true System.out / PrintStream
+ * true false ByteArrayOutputStream
+ * false true FileOutputStream
+ * false false ByteArrayOutputStream
+ */
+ private OutputStream getOutputStreamForJson() throws FileNotFoundException {
+ if (consoleMode && rulesToFile) {
+ return System.out;
+ } else if (rulesToFile) {
+ File mappingRulesFile = new File(outputDirectory,
+ MAPPING_RULES_JSON);
+ return new FileOutputStream(mappingRulesFile);
+ } else {
+ return new ByteArrayOutputStream();
+ }
+ }
+
private void emitDefaultQueueMaxParallelApplications() {
if (queueMaxAppsDefault != Integer.MAX_VALUE) {
capacitySchedulerConfig.set(
@@ -468,11 +505,6 @@ public class FSConfigToCSConfigConverter {
}
@VisibleForTesting
- void setMappingRulesOutputStream(OutputStream outputStream) {
- this.mappingRulesOutputStream = outputStream;
- }
-
- @VisibleForTesting
void setConsoleMode(boolean console) {
this.consoleMode = console;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterMain.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterMain.java
index 3953dcd..f22a59a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterMain.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterMain.java
@@ -16,6 +16,9 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
+import java.util.function.Consumer;
+
+import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.Marker;
@@ -25,12 +28,16 @@ import org.slf4j.MarkerFactory;
* Main class that invokes the FS->CS converter.
*
*/
-@SuppressWarnings("checkstyle:hideutilityclassconstructor")
-public class FSConfigToCSConfigConverterMain {
+public final class FSConfigToCSConfigConverterMain {
+ private FSConfigToCSConfigConverterMain() {
+ // no instances
+ }
+
private static final Logger LOG =
LoggerFactory.getLogger(FSConfigToCSConfigConverterMain.class);
private static final Marker FATAL =
MarkerFactory.getMarker("FATAL");
+ private static Consumer<Integer> exitFunction = System::exit;
public static void main(String[] args) {
try {
@@ -44,11 +51,16 @@ public class FSConfigToCSConfigConverterMain {
"see previous error messages for details!");
}
- System.exit(exitCode);
+ exitFunction.accept(exitCode);
} catch (Throwable t) {
LOG.error(FATAL,
"Error while starting FS configuration conversion!", t);
- System.exit(-1);
+ exitFunction.accept(-1);
}
}
+
+ @VisibleForTesting
+ static void setExit(Consumer<Integer> exitFunc) {
+ exitFunction = exitFunc;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterParams.java
index 2073164..ad70dbf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterParams.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterParams.java
@@ -28,7 +28,7 @@ public final class FSConfigToCSConfigConverterParams {
private String clusterResource;
private String outputDirectory;
private boolean convertPlacementRules;
-
+ private boolean placementRulesToFile;
private FSConfigToCSConfigConverterParams() {
@@ -63,6 +63,10 @@ public final class FSConfigToCSConfigConverterParams {
return convertPlacementRules;
}
+ public boolean isPlacementRulesToFile() {
+ return placementRulesToFile;
+ }
+
@Override
public String toString() {
return "FSConfigToCSConfigConverterParams{" +
@@ -72,6 +76,7 @@ public final class FSConfigToCSConfigConverterParams {
", clusterResource='" + clusterResource + '\'' +
", console=" + console + '\'' +
", convertPlacementRules=" + convertPlacementRules +
+ ", placementRulesToFile=" + placementRulesToFile +
'}';
}
@@ -87,6 +92,7 @@ public final class FSConfigToCSConfigConverterParams {
private String clusterResource;
private String outputDirectory;
private boolean convertPlacementRules;
+ private boolean placementRulesToFile;
private Builder() {
}
@@ -130,6 +136,11 @@ public final class FSConfigToCSConfigConverterParams {
return this;
}
+ public Builder withPlacementRulesToFile(boolean rulesToFile) {
+ this.placementRulesToFile = rulesToFile;
+ return this;
+ }
+
public FSConfigToCSConfigConverterParams build() {
FSConfigToCSConfigConverterParams params =
new FSConfigToCSConfigConverterParams();
@@ -140,6 +151,7 @@ public final class FSConfigToCSConfigConverterParams {
params.conversionRulesConfig = this.conversionRulesConfig;
params.outputDirectory = this.outputDirectory;
params.convertPlacementRules = this.convertPlacementRules;
+ params.placementRulesToFile = this.placementRulesToFile;
return params;
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/VerificationException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/VerificationException.java
index 68ca37d..d9d9689 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/VerificationException.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/VerificationException.java
@@ -28,4 +28,7 @@ public class VerificationException extends RuntimeException {
public VerificationException(String message, Throwable cause) {
super(message, cause);
}
+
+ public VerificationException() {
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java
index addf95f..1068953 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java
@@ -19,7 +19,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
@@ -29,6 +31,7 @@ import java.util.Arrays;
import java.util.List;
import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.junit.After;
@@ -45,8 +48,6 @@ import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
-
/**
* Unit tests for FSConfigToCSConfigArgumentHandler.
*
@@ -230,6 +231,28 @@ public class TestFSConfigToCSConfigArgumentHandler {
}
@Test
+ public void testVerificationException() throws Exception {
+ setupFSConfigConversionFiles(true);
+ ConversionOptions mockOptions = Mockito.mock(ConversionOptions.class);
+ FSConfigToCSConfigArgumentHandler argumentHandler =
+ new FSConfigToCSConfigArgumentHandler(mockOptions, mockValidator);
+ argumentHandler.setConverterSupplier(this::getMockConverter);
+
+ String[] args = getArgumentsAsArrayWithDefaults("-f",
+ FSConfigConverterTestCommons.FS_ALLOC_FILE,
+ "-r", FSConfigConverterTestCommons.CONVERSION_RULES_FILE);
+
+ doThrow(new VerificationException("test", new Exception("test")))
+ .when(mockConverter)
+ .convert(any(FSConfigToCSConfigConverterParams.class));
+
+ argumentHandler.parseAndConvert(args);
+
+ verify(mockOptions).handleVerificationFailure(any(Exception.class),
+ any(String.class));
+ }
+
+ @Test
public void testFairSchedulerXmlIsNotDefinedIfItsDefinedInYarnSiteXml()
throws Exception {
setupFSConfigConversionFiles(true);
@@ -534,7 +557,8 @@ public class TestFSConfigToCSConfigArgumentHandler {
String[] args = new String[] {
"-y", FSConfigConverterTestCommons.YARN_SITE_XML,
- "-o", FSConfigConverterTestCommons.OUTPUT_DIR};
+ "-o", FSConfigConverterTestCommons.OUTPUT_DIR,
+ "-e"};
int retVal = argumentHandler.parseAndConvert(args);
assertEquals("Return value", -1, retVal);
@@ -565,6 +589,13 @@ public class TestFSConfigToCSConfigArgumentHandler {
}
@Test
+ public void testMappingRulesJsonExistsInOutputFolder()
+ throws Exception {
+ testFileExistsInOutputFolder(
+ "mapping-rules.json");
+ }
+
+ @Test
public void testPlacementRulesConversionEnabled() throws Exception {
testPlacementRuleConversion(true);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
index 4968956..070bd64 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
@@ -36,7 +36,6 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.util.Map;
@@ -463,6 +462,7 @@ public class TestFSConfigToCSConfigConverter {
FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder()
.withClusterResource(CLUSTER_RESOURCE_STRING)
.withConvertPlacementRules(true)
+ .withPlacementRulesToFile(true)
.build();
converter.convert(params);
@@ -612,17 +612,17 @@ public class TestFSConfigToCSConfigConverter {
config.setBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE,
true);
- ByteArrayOutputStream jsonOutStream = new ByteArrayOutputStream();
converter.setConvertPlacementRules(true);
- converter.setMappingRulesOutputStream(jsonOutStream);
converter.setConsoleMode(true);
converter.convert(config);
+ String json = converter.getCapacitySchedulerConfig()
+ .get(CapacitySchedulerConfiguration.MAPPING_RULE_JSON);
MappingRulesDescription description =
new ObjectMapper()
.reader()
.forType(MappingRulesDescription.class)
- .readValue(jsonOutStream.toByteArray());
+ .readValue(json);
if (hasPlacementRules) {
// fs.xml defines 5 rules
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverterMain.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverterMain.java
index 4d262bd..ad0f6e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverterMain.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverterMain.java
@@ -21,17 +21,17 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.conve
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.OUTPUT_DIR;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.YARN_SITE_XML;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.setupFSConfigConversionFiles;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
+import java.util.function.Consumer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.junit.After;
import org.junit.Before;
-import org.junit.Rule;
import org.junit.Test;
-import org.junit.contrib.java.lang.system.ExpectedSystemExit;
/**
@@ -40,19 +40,20 @@ import org.junit.contrib.java.lang.system.ExpectedSystemExit;
*/
public class TestFSConfigToCSConfigConverterMain {
private FSConfigConverterTestCommons converterTestCommons;
-
- @Rule
- public final ExpectedSystemExit exit = ExpectedSystemExit.none();
+ private ExitFunc exitFunc;
@Before
public void setUp() throws Exception {
+ exitFunc = new ExitFunc();
converterTestCommons = new FSConfigConverterTestCommons();
converterTestCommons.setUp();
+ FSConfigToCSConfigConverterMain.setExit(exitFunc);
}
@After
public void tearDown() throws Exception {
QueueMetrics.clearQueueMetrics();
+ FSConfigToCSConfigConverterMain.setExit(System::exit);
converterTestCommons.tearDown();
}
@@ -68,7 +69,6 @@ public class TestFSConfigToCSConfigConverterMain {
public void testConvertFSConfigurationDefaults()
throws Exception {
setupFSConfigConversionFiles();
- exit.expectSystemExitWithStatus(0);
FSConfigToCSConfigConverterMain.main(new String[] {
"-o", OUTPUT_DIR,
@@ -83,17 +83,18 @@ public class TestFSConfigToCSConfigConverterMain {
assertTrue("capacity-scheduler.xml was not generated", csConfigExists);
assertTrue("yarn-site.xml was not generated", yarnSiteConfigExists);
+ assertEquals("Exit code", 0, exitFunc.exitCode);
}
@Test
public void testConvertFSConfigurationWithConsoleParam()
throws Exception {
setupFSConfigConversionFiles();
- exit.expectSystemExitWithStatus(0);
FSConfigToCSConfigConverterMain.main(new String[] {
"-p",
"-m",
+ "-e",
"-y", YARN_SITE_XML,
"-f", FS_ALLOC_FILE,
"-r", CONVERSION_RULES_FILE});
@@ -105,35 +106,42 @@ public class TestFSConfigToCSConfigConverterMain {
stdout.contains("======= capacity-scheduler.xml ======="));
assertTrue("Stdout doesn't contain mapping-rules.json",
stdout.contains("======= mapping-rules.json ======="));
+ assertEquals("Exit code", 0, exitFunc.exitCode);
}
@Test
public void testShortHelpSwitch() {
- exit.expectSystemExitWithStatus(0);
-
FSConfigToCSConfigConverterMain.main(new String[] {"-h"});
verifyHelpText();
+ assertEquals("Exit code", 0, exitFunc.exitCode);
}
@Test
public void testLongHelpSwitch() {
- exit.expectSystemExitWithStatus(0);
-
FSConfigToCSConfigConverterMain.main(new String[] {"--help"});
verifyHelpText();
+ assertEquals("Exit code", 0, exitFunc.exitCode);
+ }
+
+ @Test
+ public void testHelpDisplayedWithoutArgs() {
+ FSConfigToCSConfigConverterMain.main(new String[] {});
+
+ verifyHelpText();
+ assertEquals("Exit code", 0, exitFunc.exitCode);
}
@Test
public void testConvertFSConfigurationWithLongSwitches()
throws IOException {
- exit.expectSystemExitWithStatus(0);
setupFSConfigConversionFiles();
FSConfigToCSConfigConverterMain.main(new String[] {
"--print",
"--convert-placement-rules",
+ "--rules-to-file",
"--yarnsiteconfig", YARN_SITE_XML,
"--fsconfig", FS_ALLOC_FILE,
"--rulesconfig", CONVERSION_RULES_FILE});
@@ -145,6 +153,16 @@ public class TestFSConfigToCSConfigConverterMain {
stdout.contains("======= capacity-scheduler.xml ======="));
assertTrue("Stdout doesn't contain mapping-rules.json",
stdout.contains("======= mapping-rules.json ======="));
+ assertEquals("Exit code", 0, exitFunc.exitCode);
+ }
+
+ @Test
+ public void testNegativeReturnValueOnError() {
+ FSConfigToCSConfigConverterMain.main(new String[] {
+ "--print",
+ "--yarnsiteconfig"});
+
+ assertEquals("Exit code", -1, exitFunc.exitCode);
}
private void verifyHelpText() {
@@ -152,4 +170,14 @@ public class TestFSConfigToCSConfigConverterMain {
assertTrue("Help was not displayed",
stdout.contains("General options are:"));
}
+
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ class ExitFunc implements Consumer<Integer> {
+ int exitCode;
+
+ @Override
+ public void accept(Integer t) {
+ this.exitCode = t.intValue();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org