You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2020/02/25 20:30:17 UTC
[hadoop] branch trunk updated: YARN-10130. FS-CS converter: Do not
allow output dir to be the same as input dir. Contributed by Adam Antal
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new d68616b YARN-10130. FS-CS converter: Do not allow output dir to be the same as input dir. Contributed by Adam Antal
d68616b is described below
commit d68616b0453881bec8dcb917a0b7d3fbe0838965
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Tue Feb 25 21:30:04 2020 +0100
YARN-10130. FS-CS converter: Do not allow output dir to be the same as input dir. Contributed by Adam Antal
---
.../FSConfigToCSConfigArgumentHandler.java | 48 +++++++++++++++++
.../converter/FSConfigConverterTestCommons.java | 2 +-
.../TestFSConfigToCSConfigArgumentHandler.java | 63 ++++++++++++++++++++++
3 files changed, 112 insertions(+), 1 deletion(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java
index c751953..9121a6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java
@@ -25,6 +25,7 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,6 +40,13 @@ public class FSConfigToCSConfigArgumentHandler {
private static final Logger LOG =
LoggerFactory.getLogger(FSConfigToCSConfigArgumentHandler.class);
+ private static final String ALREADY_CONTAINS_EXCEPTION_MSG =
+ "The %s (provided with %s|%s arguments) contains " +
+ "the %s provided with the %s|%s options.";
+ private static final String ALREADY_CONTAINS_FILE_EXCEPTION_MSG =
+ "The %s %s (provided with %s|%s arguments) already contains a file " +
+ "or directory named %s which will be the output of the conversion!";
+
private FSConfigToCSConfigRuleHandler ruleHandler;
private FSConfigToCSConfigConverterParams converterParams;
private ConversionOptions conversionOptions;
@@ -213,6 +221,7 @@ public class FSConfigToCSConfigArgumentHandler {
checkFile(CliOption.FAIR_SCHEDULER, fairSchedulerXmlFile);
checkFile(CliOption.CONVERSION_RULES, conversionRulesFile);
checkDirectory(CliOption.OUTPUT_DIR, outputDir);
+ checkOutputDirDoesNotContainXmls(yarnSiteXmlFile, outputDir);
return FSConfigToCSConfigConverterParams.Builder.create()
.withYarnSiteXmlConfig(yarnSiteXmlFile)
@@ -225,6 +234,45 @@ public class FSConfigToCSConfigArgumentHandler {
.build();
}
+ private static void checkOutputDirDoesNotContainXmls(String yarnSiteXmlFile,
+ String outputDir) {
+ if (yarnSiteXmlFile == null || outputDir == null) {
+ return;
+ }
+
+ // check whether yarn-site.xml is not in the output folder
+ File xmlFile = new File(yarnSiteXmlFile);
+ File xmlParentFolder = xmlFile.getParentFile();
+ File output = new File(outputDir);
+ if (output.equals(xmlParentFolder)) {
+ throw new IllegalArgumentException(
+ String.format(ALREADY_CONTAINS_EXCEPTION_MSG,
+ CliOption.OUTPUT_DIR.name, CliOption.OUTPUT_DIR.shortSwitch,
+ CliOption.OUTPUT_DIR.longSwitch, CliOption.YARN_SITE.name,
+ CliOption.YARN_SITE.shortSwitch,
+ CliOption.YARN_SITE.longSwitch));
+ }
+
+ // check whether the output folder does not contain nor yarn-site.xml
+ // neither capacity-scheduler.xml
+ checkFileNotInOutputDir(output,
+ YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
+ checkFileNotInOutputDir(output,
+ YarnConfiguration.CS_CONFIGURATION_FILE);
+ }
+
+ private static void checkFileNotInOutputDir(File output, String fileName) {
+ File file = new File(output, fileName);
+ if (file.exists()) {
+ throw new IllegalArgumentException(
+ String.format(ALREADY_CONTAINS_FILE_EXCEPTION_MSG,
+ CliOption.OUTPUT_DIR.name, output,
+ CliOption.OUTPUT_DIR.shortSwitch,
+ CliOption.OUTPUT_DIR.longSwitch,
+ fileName));
+ }
+ }
+
private void printHelp(Options opts) {
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("General options are: ", opts);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java
index 4f67809..af915d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java
@@ -35,7 +35,7 @@ import static org.junit.Assert.assertTrue;
*
*/
public class FSConfigConverterTestCommons {
- private final static String TEST_DIR =
+ public final static String TEST_DIR =
new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath();
public final static String FS_ALLOC_FILE =
new File(TEST_DIR, "test-fair-scheduler.xml").getAbsolutePath();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java
index edf4c86..7a29c71 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java
@@ -20,10 +20,13 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
@@ -492,4 +495,64 @@ public class TestFSConfigToCSConfigArgumentHandler {
assertFalse("No terminal rule check was enabled",
conversionOptions.isNoRuleTerminalCheck());
}
+
+ @Test
+ public void testYarnSiteOptionInOutputFolder() throws Exception {
+ setupFSConfigConversionFiles(true);
+
+ FSConfigToCSConfigArgumentHandler argumentHandler =
+ createArgumentHandler();
+
+ String[] args = new String[] {
+ "-y", FSConfigConverterTestCommons.YARN_SITE_XML,
+ "-o", FSConfigConverterTestCommons.TEST_DIR};
+
+ int retVal = argumentHandler.parseAndConvert(args);
+ assertEquals("Return value", -1, retVal);
+
+ assertTrue(fsTestCommons.getErrContent()
+ .toString().contains("contains the yarn-site.xml"));
+ }
+
+ private void testFileExistsInOutputFolder(String file) throws Exception {
+ File testFile = new File(FSConfigConverterTestCommons.OUTPUT_DIR, file);
+ try {
+ FileUtils.touch(testFile);
+
+ setupFSConfigConversionFiles(true);
+
+ FSConfigToCSConfigArgumentHandler argumentHandler =
+ createArgumentHandler();
+
+ String[] args = new String[] {
+ "-y", FSConfigConverterTestCommons.YARN_SITE_XML,
+ "-o", FSConfigConverterTestCommons.OUTPUT_DIR};
+
+ int retVal = argumentHandler.parseAndConvert(args);
+ assertEquals("Return value", -1, retVal);
+
+ String expectedMessage = String.format(
+ "already contains a file or directory named %s", file);
+
+ assertTrue(fsTestCommons.getErrContent()
+ .toString().contains(expectedMessage));
+ } finally {
+ if (testFile.exists()) {
+ testFile.delete();
+ }
+ }
+ }
+
+ @Test
+ public void testYarnSiteExistsInOutputFolder() throws Exception {
+ testFileExistsInOutputFolder(
+ YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
+ }
+
+ @Test
+ public void testCapacitySchedulerXmlExistsInOutputFolder()
+ throws Exception {
+ testFileExistsInOutputFolder(
+ YarnConfiguration.CS_CONFIGURATION_FILE);
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org