You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2019/11/26 20:23:47 UTC
[hadoop] branch trunk updated: YARN-9899. Migration tool that help
to generate CS config based on FS config [Phase 2]. Contributed by Peter
Bacsko
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 8c9018d YARN-9899. Migration tool that help to generate CS config based on FS config [Phase 2]. Contributed by Peter Bacsko
8c9018d is described below
commit 8c9018d5c7830ae8ec85f446985cafbc8a14d688
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Tue Nov 26 21:22:35 2019 +0100
YARN-9899. Migration tool that help to generate CS config based on FS config [Phase 2]. Contributed by Peter Bacsko
---
hadoop-yarn-project/hadoop-yarn/bin/yarn | 4 +
.../server/resourcemanager/ResourceManager.java | 46 ----
.../FSConfigToCSConfigArgumentHandler.java | 68 ++++--
.../converter/FSConfigToCSConfigConverterMain.java | 57 +++++
.../fair/converter/QueuePlacementConverter.java | 15 +-
.../resourcemanager/TestResourceManager.java | 91 +------
.../converter/FSConfigConverterTestCommons.java | 6 +-
.../TestFSConfigToCSConfigArgumentHandler.java | 27 ++-
.../TestFSConfigToCSConfigConverterMain.java | 135 +++++++++++
.../converter/TestQueuePlacementConverter.java | 265 +++++++++++++++++++++
10 files changed, 547 insertions(+), 167 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 3ec0311..54daeaa 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -47,6 +47,7 @@ function hadoop_usage
hadoop_add_subcommand "queue" client "prints queue information"
hadoop_add_subcommand "registrydns" daemon "run the registry DNS server"
hadoop_add_subcommand "resourcemanager" daemon "run the ResourceManager"
+ hadoop_add_subcommand "fs2cs" client "converts Fair Scheduler configuration to Capacity Scheduler (EXPERIMENTAL)"
hadoop_add_subcommand "rmadmin" admin "admin tools"
hadoop_add_subcommand "router" daemon "run the Router daemon"
hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration"
@@ -165,6 +166,9 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
hadoop_translate_cygwin_path sld
hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}"
;;
+ fs2cs)
+ HADOOP_CLASSNAME="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverterMain"
+ ;;
rmadmin)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
;;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 9ba8f32..e71d3c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -21,10 +21,6 @@ package org.apache.hadoop.yarn.server.resourcemanager;
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.spi.container.servlet.ServletContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigArgumentHandler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigArgumentHandler.CliOption;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverter;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.Marker;
@@ -232,13 +228,6 @@ public class ResourceManager extends CompositeService
private Configuration conf;
private UserGroupInformation rmLoginUGI;
- private static FSConfigToCSConfigArgumentHandler
- fsConfigConversionArgumentHandler;
-
- static {
- FSConfigToCSConfigConverter converter = initFSConfigConverter();
- initFSArgumentHandler(converter);
- }
public ResourceManager() {
super("ResourceManager");
@@ -1576,22 +1565,6 @@ public class ResourceManager extends CompositeService
} else if (argv[0].equals("-remove-application-from-state-store")
&& argv.length == 2) {
removeApplication(conf, argv[1]);
- } else if (argv[0].equals("-convert-fs-configuration")) {
- String[] args = Arrays.copyOfRange(argv, 1, argv.length);
- try {
- int exitCode =
- fsConfigConversionArgumentHandler.parseAndConvert(args);
- if (exitCode != 0) {
- LOG.error(FATAL,
- "Error while starting FS configuration conversion, " +
- "see previous error messages for details!");
- System.exit(exitCode);
- }
- } catch (Throwable t) {
- LOG.error(FATAL,
- "Error while starting FS configuration conversion!", t);
- System.exit(-1);
- }
} else {
printUsage(System.err);
}
@@ -1744,12 +1717,6 @@ public class ResourceManager extends CompositeService
out.println(" "
+ "[-format-conf-store]" + "\n");
- out.println("[-convert-fs-configuration ");
- out.println(FSConfigToCSConfigConverter.WARNING_TEXT);
- for (CliOption cliOption : CliOption.values()) {
- out.println(" " + cliOption.getAsArgumentString());
- }
- out.println("]");
}
protected RMAppLifetimeMonitor createRMAppLifetimeMonitor() {
@@ -1767,17 +1734,4 @@ public class ResourceManager extends CompositeService
public boolean isSecurityEnabled() {
return UserGroupInformation.isSecurityEnabled();
}
-
- @VisibleForTesting
- static void initFSArgumentHandler(FSConfigToCSConfigConverter converter) {
- ResourceManager.fsConfigConversionArgumentHandler =
- new FSConfigToCSConfigArgumentHandler(converter);
- }
-
- private static FSConfigToCSConfigConverter initFSConfigConverter() {
- FSConfigToCSConfigRuleHandler ruleHandler =
- new FSConfigToCSConfigRuleHandler();
- return new FSConfigToCSConfigConverter(ruleHandler);
- }
-
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java
index 1641249..9e8f618 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java
@@ -18,9 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.MissingArgumentException;
+import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,69 +48,76 @@ public class FSConfigToCSConfigArgumentHandler {
*/
public enum CliOption {
YARN_SITE("yarn-site.xml", "y", "yarnsiteconfig",
- "Path to a valid yarn-site.xml config file", true, true),
+ "Path to a valid yarn-site.xml config file", true),
// fair-scheduler.xml is not mandatory
// if FairSchedulerConfiguration.ALLOCATION_FILE is defined in yarn-site.xml
FAIR_SCHEDULER("fair-scheduler.xml", "f", "fsconfig",
- "Path to a valid fair-scheduler.xml config file", false, true),
+ "Path to a valid fair-scheduler.xml config file", true),
CONVERSION_RULES("conversion rules config file", "r", "rulesconfig",
"Optional parameter. If given, should specify a valid path to the " +
- "conversion rules file (property format).", false, true),
+ "conversion rules file (property format).", true),
CONSOLE_MODE("console mode", "p", "print",
"If defined, the converted configuration will " +
- "only be emitted to the console.", false, false),
+ "only be emitted to the console.", false),
CLUSTER_RESOURCE("cluster resource", "c", "cluster-resource",
"Needs to be given if maxResources is defined as percentages " +
"for any queue, otherwise this parameter can be omitted.",
- false, true),
+ true),
OUTPUT_DIR("output directory", "o", "output-directory",
"Output directory for yarn-site.xml and" +
" capacity-scheduler.xml files." +
"Must have write permission for user who is running this script.",
- true, true);
+ true),
+ HELP("help", "h", "help", "Displays the list of options", false);
private final String name;
private final String shortSwitch;
private final String longSwitch;
private final String description;
- private final boolean required;
private final boolean hasArg;
CliOption(String name, String shortSwitch, String longSwitch,
- String description, boolean required, boolean hasArg) {
+ String description, boolean hasArg) {
this.name = name;
this.shortSwitch = shortSwitch;
this.longSwitch = longSwitch;
this.description = description;
- this.required = required;
this.hasArg = hasArg;
}
public Option createCommonsCliOption() {
Option option = new Option(shortSwitch, longSwitch, hasArg, description);
- option.setRequired(required);
return option;
}
-
- public String getAsArgumentString() {
- return shortSwitch + "|" + longSwitch + ": " + description;
- }
}
- public int parseAndConvert(String[] args) throws Exception {
+ int parseAndConvert(String[] args) throws Exception {
Options opts = createOptions();
try {
+ if (args.length == 0) {
+ LOG.info("Missing command line arguments");
+ printHelp(opts);
+ return 0;
+ }
+
CommandLine cliParser = new GnuParser().parse(opts, args);
+
+ if (cliParser.hasOption(CliOption.HELP.shortSwitch)) {
+ printHelp(opts);
+ return 0;
+ }
+
checkOptionPresent(cliParser, CliOption.YARN_SITE);
- checkOptionPresent(cliParser, CliOption.OUTPUT_DIR);
+ checkOutputDefined(cliParser);
FSConfigToCSConfigConverterParams params = validateInputFiles(cliParser);
converter.convert(params);
- } catch (MissingArgumentException e) {
- String msg = "Missing argument for options" + e.getMessage();
+ } catch (ParseException e) {
+ String msg = "Options parsing failed: " + e.getMessage();
logAndStdErr(e, msg);
+ printHelp(opts);
return -1;
} catch (PreconditionException e) {
String msg = "Cannot start FS config conversion due to the following"
@@ -131,7 +139,8 @@ public class FSConfigToCSConfigArgumentHandler {
}
private void logAndStdErr(Exception e, String msg) {
- LOG.error(msg, e);
+ LOG.debug("Stack trace", e);
+ LOG.error(msg);
System.err.println(msg);
}
@@ -172,6 +181,11 @@ public class FSConfigToCSConfigArgumentHandler {
.build();
}
+ private void printHelp(Options opts) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("General options are: ", opts);
+ }
+
private static void checkOptionPresent(CommandLine cliParser,
CliOption cliOption) {
if (!cliParser.hasOption(cliOption.shortSwitch)) {
@@ -181,6 +195,20 @@ public class FSConfigToCSConfigArgumentHandler {
}
}
+ private static void checkOutputDefined(CommandLine cliParser) {
+ boolean hasOutputDir =
+ cliParser.hasOption(CliOption.OUTPUT_DIR.shortSwitch);
+
+ boolean console =
+ cliParser.hasOption(CliOption.CONSOLE_MODE.shortSwitch);
+
+ if (!console && !hasOutputDir) {
+ throw new PreconditionException(
+ "Output directory or console mode was not defined. Please" +
+ " use -h or --help to see command line switches");
+ }
+ }
+
private static void checkFile(CliOption cliOption, String filePath) {
checkFileInternal(cliOption, filePath, true);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterMain.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterMain.java
new file mode 100644
index 0000000..bbf4fd2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterMain.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.Marker;
+import org.slf4j.MarkerFactory;
+
+/**
+ * Main class that invokes the FS->CS converter.
+ *
+ */
+@SuppressWarnings("checkstyle:hideutilityclassconstructor")
+public class FSConfigToCSConfigConverterMain {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FSConfigToCSConfigConverterMain.class);
+ private static final Marker FATAL =
+ MarkerFactory.getMarker("FATAL");
+
+ public static void main(String[] args) {
+ try {
+ FSConfigToCSConfigRuleHandler ruleHandler =
+ new FSConfigToCSConfigRuleHandler();
+ FSConfigToCSConfigConverter converter =
+ new FSConfigToCSConfigConverter(ruleHandler);
+ FSConfigToCSConfigArgumentHandler fsConfigConversionArgumentHandler =
+ new FSConfigToCSConfigArgumentHandler(converter);
+ int exitCode =
+ fsConfigConversionArgumentHandler.parseAndConvert(args);
+ if (exitCode != 0) {
+ LOG.error(FATAL,
+ "Error while starting FS configuration conversion, " +
+ "see previous error messages for details!");
+ System.exit(exitCode);
+ }
+ } catch (Throwable t) {
+ LOG.error(FATAL,
+ "Error while starting FS configuration conversion!", t);
+ System.exit(-1);
+ }
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
index 2db808f..4641dfc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.placement.FSPlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.PrimaryGroupPlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.RejectPlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.SecondaryGroupExistingPlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.SpecifiedPlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserPlacementRule;
@@ -51,15 +52,15 @@ class QueuePlacementConverter {
ruleCount++;
if (rule instanceof UserPlacementRule) {
UserPlacementRule userRule = (UserPlacementRule) rule;
- if (mapping.length() > 0) {
- mapping.append(";");
- }
// nested rule
if (userRule.getParentRule() != null) {
handleNestedRule(mapping, userRule);
} else {
if (!userAsDefaultQueue) {
+ if (mapping.length() > 0) {
+ mapping.append(";");
+ }
mapping.append("u:" + USER + ":" + USER);
}
}
@@ -82,8 +83,11 @@ class QueuePlacementConverter {
mapping.append("u:" + USER + ":").append(defaultRule.defaultQueueName);
} else if (rule instanceof SecondaryGroupExistingPlacementRule) {
// TODO: wait for YARN-9840
+ if (mapping.length() > 0) {
+ mapping.append(";");
+ }
mapping.append("u:" + USER + ":" + SECONDARY_GROUP);
- } else {
+ } else if (!(rule instanceof RejectPlacementRule)) {
throw new IllegalArgumentException("Unknown placement rule: " + rule);
}
}
@@ -99,6 +103,9 @@ class QueuePlacementConverter {
private void handleNestedRule(StringBuilder mapping,
UserPlacementRule userRule) {
PlacementRule pr = userRule.getParentRule();
+ if (mapping.length() > 0) {
+ mapping.append(";");
+ }
if (pr instanceof PrimaryGroupPlacementRule) {
// TODO: wait for YARN-9841
mapping.append("u:" + USER + ":" + PRIMARY_GROUP + "." + USER);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
index a857c27..b9ff588 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
@@ -18,6 +18,13 @@
package org.apache.hadoop.yarn.server.resourcemanager;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.TimeoutException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.http.lib.StaticUserWebFilter;
@@ -39,8 +46,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptR
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverter;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverterParams;
import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilterInitializer;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
@@ -49,23 +54,8 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import org.mockito.ArgumentCaptor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.CONVERSION_RULES_FILE;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.FS_ALLOC_FILE;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.OUTPUT_DIR;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.YARN_SITE_XML;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.setupFSConfigConversionFiles;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
public class TestResourceManager {
private static final Logger LOG =
@@ -372,71 +362,4 @@ public class TestResourceManager {
dummyResourceManager.stop();
}
}
-
- /**
- * Example command: <br>
- * opt/hadoop/bin/yarn resourcemanager -convert-fs-configuration<br>
- * -o /tmp/output<br>
- * -y /opt/hadoop/etc/hadoop/yarn-site.xml<br>
- * -f /opt/hadoop/etc/hadoop/fair-scheduler.xml<br>
- * -r /home/systest/sample-rules-config.properties<br>
- */
- @Test
- @SuppressWarnings("checkstyle:javadocstyle")
- public void testResourceManagerConvertFSConfigurationDefaults()
- throws Exception {
- setupFSConfigConversionFiles();
-
- ArgumentCaptor<FSConfigToCSConfigConverterParams> conversionParams =
- ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class);
-
- final String mainSwitch = "-convert-fs-configuration";
- FSConfigToCSConfigConverter mockConverter =
- mock(FSConfigToCSConfigConverter.class);
-
- ResourceManager.initFSArgumentHandler(mockConverter);
- ResourceManager.main(new String[] {mainSwitch, "-o", OUTPUT_DIR,
- "-y", YARN_SITE_XML, "-f", FS_ALLOC_FILE, "-r",
- CONVERSION_RULES_FILE});
-
- // validate params
- verify(mockConverter).convert(conversionParams.capture());
- FSConfigToCSConfigConverterParams params = conversionParams.getValue();
- LOG.info("FS config converter parameters: " + params);
-
- assertThat(params.getYarnSiteXmlConfig()).isEqualTo(YARN_SITE_XML);
- assertThat(params.getFairSchedulerXmlConfig()).isEqualTo(FS_ALLOC_FILE);
- assertThat(params.getConversionRulesConfig())
- .isEqualTo(CONVERSION_RULES_FILE);
- assertThat(params.isConsole()).isEqualTo(false);
- }
-
- @Test
- public void testResourceManagerConvertFSConfigurationWithConsoleParam()
- throws Exception {
- setupFSConfigConversionFiles();
-
- ArgumentCaptor<FSConfigToCSConfigConverterParams> conversionParams =
- ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class);
-
- final String mainSwitch = "-convert-fs-configuration";
- FSConfigToCSConfigConverter mockConverter =
- mock(FSConfigToCSConfigConverter.class);
-
- ResourceManager.initFSArgumentHandler(mockConverter);
- ResourceManager.main(new String[] {mainSwitch, "-o", OUTPUT_DIR,
- "-p", "-y", YARN_SITE_XML, "-f", FS_ALLOC_FILE, "-r",
- CONVERSION_RULES_FILE});
-
- // validate params
- verify(mockConverter).convert(conversionParams.capture());
- FSConfigToCSConfigConverterParams params = conversionParams.getValue();
- LOG.info("FS config converter parameters: " + params);
-
- assertThat(params.getYarnSiteXmlConfig()).isEqualTo(YARN_SITE_XML);
- assertThat(params.getFairSchedulerXmlConfig()).isEqualTo(FS_ALLOC_FILE);
- assertThat(params.getConversionRulesConfig())
- .isEqualTo(CONVERSION_RULES_FILE);
- assertThat(params.isConsole()).isEqualTo(true);
- }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java
index 5fcf379..0f57241 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java
@@ -88,7 +88,11 @@ public class FSConfigConverterTestCommons {
return errContent;
}
- private void deleteTestFiles() {
+ ByteArrayOutputStream getStdOutContent() {
+ return outContent;
+ }
+
+ public void deleteTestFiles() {
//Files may not be created so we are not strict here!
deleteFile(FS_ALLOC_FILE, false);
deleteFile(YARN_SITE_XML, false);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java
index 70224fc..66c5999 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
import com.google.common.collect.Lists;
-import org.apache.commons.cli.MissingOptionException;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
@@ -116,13 +115,13 @@ public class TestFSConfigToCSConfigArgumentHandler {
String[] args = new String[] {"-o",
FSConfigConverterTestCommons.OUTPUT_DIR};
- expectedException.expect(MissingOptionException.class);
- expectedException.expectMessage("Missing required option: y");
+ int retVal = argumentHandler.parseAndConvert(args);
+ assertEquals("Return value", -1, retVal);
- argumentHandler.parseAndConvert(args);
+ assertTrue("Error content missing", fsTestCommons.getErrContent()
+ .toString().contains("Missing yarn-site.xml parameter"));
}
-
@Test
public void testMissingFairSchedulerXmlArgument() throws Exception {
setupFSConfigConversionFiles(true);
@@ -142,10 +141,12 @@ public class TestFSConfigToCSConfigArgumentHandler {
String[] args = new String[] {"-y",
FSConfigConverterTestCommons.YARN_SITE_XML};
- expectedException.expect(MissingOptionException.class);
- expectedException.expectMessage("Missing required option: o");
+ int retVal = argumentHandler.parseAndConvert(args);
+ assertEquals("Return value", -1, retVal);
- argumentHandler.parseAndConvert(args);
+ assertTrue("Error content missing", fsTestCommons.getErrContent()
+ .toString()
+ .contains("Output directory or console mode was not defined"));
}
@Test
@@ -183,8 +184,8 @@ public class TestFSConfigToCSConfigArgumentHandler {
FSConfigConverterTestCommons.YARN_SITE_XML, "-o",
FSConfigConverterTestCommons.YARN_SITE_XML);
- argumentHandler.parseAndConvert(args);
- System.out.println(fsTestCommons.getErrContent());
+ int retVal = argumentHandler.parseAndConvert(args);
+ assertEquals("Return value", -1, retVal);
assertTrue("Error content missing", fsTestCommons.getErrContent()
.toString()
.contains("Cannot start FS config conversion due to the following " +
@@ -355,7 +356,8 @@ public class TestFSConfigToCSConfigArgumentHandler {
Mockito.doThrow(UnsupportedPropertyException.class)
.when(mockConverter)
.convert(ArgumentMatchers.any(FSConfigToCSConfigConverterParams.class));
- argumentHandler.parseAndConvert(args);
+ int retVal = argumentHandler.parseAndConvert(args);
+ assertEquals("Return value", -1, retVal);
assertTrue("Error content missing", fsTestCommons.getErrContent()
.toString().contains("Unsupported property/setting encountered"));
}
@@ -372,7 +374,8 @@ public class TestFSConfigToCSConfigArgumentHandler {
Mockito.doThrow(ConversionException.class).when(mockConverter)
.convert(ArgumentMatchers.any(FSConfigToCSConfigConverterParams.class));
- argumentHandler.parseAndConvert(args);
+ int retVal = argumentHandler.parseAndConvert(args);
+ assertEquals("Return value", -1, retVal);
assertTrue("Error content missing", fsTestCommons.getErrContent()
.toString().contains("Fatal error during FS config conversion"));
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverterMain.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverterMain.java
new file mode 100644
index 0000000..6ac0ef7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverterMain.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.CONVERSION_RULES_FILE;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.FS_ALLOC_FILE;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.OUTPUT_DIR;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.YARN_SITE_XML;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.setupFSConfigConversionFiles;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+
+/**
+ * Unit tests for TestFSConfigToCSConfigConverterMain.
+ *
+ */
+public class TestFSConfigToCSConfigConverterMain {
+ private FSConfigConverterTestCommons converterTestCommons;
+
+ @Before
+ public void setUp() throws Exception {
+ converterTestCommons = new FSConfigConverterTestCommons();
+ converterTestCommons.setUp();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ converterTestCommons.tearDown();
+ }
+
+ /*
+ * Example command:
+ * opt/hadoop/bin/yarn fs2cs
+ * -o /tmp/output
+ * -y /opt/hadoop/etc/hadoop/yarn-site.xml
+ * -f /opt/hadoop/etc/hadoop/fair-scheduler.xml
+ * -r /home/systest/sample-rules-config.properties
+ */
+ @Test
+ public void testConvertFSConfigurationDefaults()
+ throws Exception {
+ setupFSConfigConversionFiles();
+
+ FSConfigToCSConfigConverterMain.main(new String[] {
+ "-o", OUTPUT_DIR,
+ "-y", YARN_SITE_XML,
+ "-f", FS_ALLOC_FILE,
+ "-r", CONVERSION_RULES_FILE});
+
+ boolean csConfigExists =
+ new File(OUTPUT_DIR, "capacity-scheduler.xml").exists();
+ boolean yarnSiteConfigExists =
+ new File(OUTPUT_DIR, "yarn-site.xml").exists();
+
+ assertTrue("capacity-scheduler.xml was not generated", csConfigExists);
+ assertTrue("yarn-site.xml was not generated", yarnSiteConfigExists);
+ }
+
+ @Test
+ public void testConvertFSConfigurationWithConsoleParam()
+ throws Exception {
+ setupFSConfigConversionFiles();
+
+ FSConfigToCSConfigConverterMain.main(new String[] {
+ "-p",
+ "-y", YARN_SITE_XML,
+ "-f", FS_ALLOC_FILE,
+ "-r", CONVERSION_RULES_FILE});
+
+ String stdout = converterTestCommons.getStdOutContent().toString();
+ assertTrue("Stdout doesn't contain yarn-site.xml",
+ stdout.contains("======= yarn-site.xml ======="));
+ assertTrue("Stdout doesn't contain capacity-scheduler.xml",
+ stdout.contains("======= capacity-scheduler.xml ======="));
+ }
+
+ @Test
+ public void testShortHelpSwitch() {
+ FSConfigToCSConfigConverterMain.main(new String[] {"-h"});
+
+ verifyHelpText();
+ }
+
+ @Test
+ public void testLongHelpSwitch() {
+ FSConfigToCSConfigConverterMain.main(new String[] {"--help"});
+
+ verifyHelpText();
+ }
+
+ @Test
+ public void testConvertFSConfigurationWithLongSwitches()
+ throws IOException {
+ setupFSConfigConversionFiles();
+
+ FSConfigToCSConfigConverterMain.main(new String[] {
+ "--print",
+ "--yarnsiteconfig", YARN_SITE_XML,
+ "--fsconfig", FS_ALLOC_FILE,
+ "--rulesconfig", CONVERSION_RULES_FILE});
+
+ String stdout = converterTestCommons.getStdOutContent().toString();
+ assertTrue("Stdout doesn't contain yarn-site.xml",
+ stdout.contains("======= yarn-site.xml ======="));
+ assertTrue("Stdout doesn't contain capacity-scheduler.xml",
+ stdout.contains("======= capacity-scheduler.xml ======="));
+ }
+
+ private void verifyHelpText() {
+ String stdout = converterTestCommons.getStdOutContent().toString();
+ assertTrue("Help was not displayed",
+ stdout.contains("General options are:"));
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java
new file mode 100644
index 0000000..bd8ca5e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ENABLE_QUEUE_MAPPING_OVERRIDE;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.QUEUE_MAPPING;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.DefaultPlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.FSPlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.PrimaryGroupPlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.RejectPlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.SecondaryGroupExistingPlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.SpecifiedPlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.UserPlacementRule;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Unit tests for QueuePlacementConverter.
+ *
+ */
+@RunWith(MockitoJUnitRunner.class)
+public class TestQueuePlacementConverter {
+ @Mock
+ private PlacementManager placementManager;
+
+ @Mock
+ private FSConfigToCSConfigRuleHandler ruleHandler;
+
+ private QueuePlacementConverter converter;
+
+ @Before
+ public void setup() {
+ this.converter = new QueuePlacementConverter();
+ }
+
+ @Test
+ public void testConvertUserAsDefaultQueue() {
+ Map<String, String> properties = convert(true);
+
+ verifyMapping(properties, "u:%user:%user");
+ verifyZeroInteractions(ruleHandler);
+ }
+
+ @Test
+ public void testConvertUserPlacementRuleWithoutUserAsDefaultQueue() {
+ testConvertUserPlacementRule(false);
+ }
+
+ @Test
+ public void testConvertUserPlacementRuleWithUserAsDefaultQueue() {
+ testConvertUserPlacementRule(true);
+ }
+
+ private void testConvertUserPlacementRule(boolean userAsDefaultQueue) {
+ PlacementRule rule = mock(UserPlacementRule.class);
+ initPlacementManagerMock(rule);
+
+ Map<String, String> properties = convert(userAsDefaultQueue);
+
+ verifyMapping(properties, "u:%user:%user");
+ verifyZeroInteractions(ruleHandler);
+ }
+
+ @Test
+ public void testConvertSpecifiedPlacementRule() {
+ PlacementRule rule = mock(SpecifiedPlacementRule.class);
+ initPlacementManagerMock(rule);
+
+ Map<String, String> properties = convert(false);
+
+ verifyMappingNoOverride(properties, 1);
+ verifyZeroInteractions(ruleHandler);
+ }
+
+ @Test
+ public void testConvertSpecifiedPlacementRuleAtSecondPlace() {
+ PlacementRule rule = mock(UserPlacementRule.class);
+ PlacementRule rule2 = mock(SpecifiedPlacementRule.class);
+ initPlacementManagerMock(rule, rule2);
+
+ Map<String, String> properties = convert(false);
+
+ verifyMappingNoOverride(properties, 2);
+ verify(ruleHandler).handleSpecifiedNotFirstRule();
+ }
+
+ @Test
+ public void testConvertPrimaryGroupPlacementRule() {
+ PlacementRule rule = mock(PrimaryGroupPlacementRule.class);
+ initPlacementManagerMock(rule);
+
+ Map<String, String> properties = convert(false);
+
+ verifyMapping(properties, "u:%user:%primary_group");
+ verifyZeroInteractions(ruleHandler);
+ }
+
+ @Test
+ public void testConvertSecondaryGroupPlacementRule() {
+ PlacementRule rule = mock(SecondaryGroupExistingPlacementRule.class);
+ initPlacementManagerMock(rule);
+
+ Map<String, String> properties = convert(false);
+
+ verifyMapping(properties, "u:%user:%secondary_group");
+ verifyZeroInteractions(ruleHandler);
+ }
+
+ @Test
+ public void testConvertDefaultPlacementRule() {
+ DefaultPlacementRule rule = mock(DefaultPlacementRule.class);
+ rule.defaultQueueName = "abc";
+ initPlacementManagerMock(rule);
+
+ Map<String, String> properties = convert(false);
+
+ verifyMapping(properties, "u:%user:abc");
+ verifyZeroInteractions(ruleHandler);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testConvertUnsupportedPlacementRule() {
+ PlacementRule rule = mock(TestPlacementRule.class);
+ initPlacementManagerMock(rule);
+
+ // throws exception
+ convert(false);
+ }
+
+ @Test
+ public void testConvertRejectPlacementRule() {
+ PlacementRule rule = mock(RejectPlacementRule.class);
+ initPlacementManagerMock(rule);
+
+ Map<String, String> properties = convert(false);
+
+ assertEquals("Map is not empty", 0, properties.size());
+ }
+
+ @Test
+ public void testConvertNestedPrimaryGroupRule() {
+ UserPlacementRule rule = mock(UserPlacementRule.class);
+ PrimaryGroupPlacementRule parent = mock(PrimaryGroupPlacementRule.class);
+ when(rule.getParentRule()).thenReturn(parent);
+ initPlacementManagerMock(rule);
+
+ Map<String, String> properties = convert(false);
+
+ verifyMapping(properties, "u:%user:%primary_group.%user");
+ verifyZeroInteractions(ruleHandler);
+ }
+
+ @Test
+ public void testConvertNestedSecondaryGroupRule() {
+ UserPlacementRule rule = mock(UserPlacementRule.class);
+ SecondaryGroupExistingPlacementRule parent =
+ mock(SecondaryGroupExistingPlacementRule.class);
+ when(rule.getParentRule()).thenReturn(parent);
+ initPlacementManagerMock(rule);
+
+ Map<String, String> properties = convert(false);
+
+ verifyMapping(properties, "u:%user:%secondary_group.%user");
+ verifyZeroInteractions(ruleHandler);
+ }
+
+ @Test
+ public void testConvertNestedDefaultRule() {
+ UserPlacementRule rule = mock(UserPlacementRule.class);
+ DefaultPlacementRule parent =
+ mock(DefaultPlacementRule.class);
+ parent.defaultQueueName = "abc";
+ when(rule.getParentRule()).thenReturn(parent);
+ initPlacementManagerMock(rule);
+
+ Map<String, String> properties = convert(false);
+
+ verifyMapping(properties, "u:%user:abc.%user");
+ verifyZeroInteractions(ruleHandler);
+ }
+
+ @Test
+ public void testConvertMultiplePlacementRules() {
+ UserPlacementRule rule1 = mock(UserPlacementRule.class);
+ PrimaryGroupPlacementRule rule2 =
+ mock(PrimaryGroupPlacementRule.class);
+ SecondaryGroupExistingPlacementRule rule3 =
+ mock(SecondaryGroupExistingPlacementRule.class);
+ initPlacementManagerMock(rule1, rule2, rule3);
+
+ Map<String, String> properties = convert(false);
+
+ verifyMapping(properties,
+ "u:%user:%user;u:%user:%primary_group;u:%user:%secondary_group");
+ verifyZeroInteractions(ruleHandler);
+ }
+
+ private void initPlacementManagerMock(
+ PlacementRule... rules) {
+ List<PlacementRule> listOfRules = Lists.newArrayList(rules);
+ when(placementManager.getPlacementRules()).thenReturn(listOfRules);
+ }
+
+ private Map<String, String> convert(boolean userAsDefaultQueue) {
+ return converter.convertPlacementPolicy(placementManager, ruleHandler,
+ userAsDefaultQueue);
+ }
+
+ private void verifyMapping(Map<String, String> properties,
+ String expectedValue) {
+ assertEquals("Map size", 1, properties.size());
+ String value = properties.get(QUEUE_MAPPING);
+ assertNotNull("No mapping property found", value);
+ assertEquals("Mapping", expectedValue, value);
+ }
+
+ private void verifyMappingNoOverride(Map<String, String> properties,
+ int expectedSize) {
+ assertEquals("Map size", expectedSize, properties.size());
+ String value = properties.get(ENABLE_QUEUE_MAPPING_OVERRIDE);
+ assertNotNull("No mapping property found", value);
+ assertEquals("Override mapping", "false", value);
+ }
+
+ private class TestPlacementRule extends FSPlacementRule {
+ @Override
+ public ApplicationPlacementContext getPlacementForApp(
+ ApplicationSubmissionContext asc, String user) throws YarnException {
+ return null;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org