You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ey...@apache.org on 2019/10/19 00:31:32 UTC
[hadoop] branch trunk updated: YARN-9875. Improve fair scheduler
configuration store on HDFS. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
eyang pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 155864d YARN-9875. Improve fair scheduler configuration store on HDFS. Contributed by Prabhu Joseph
155864d is described below
commit 155864da006346a500ff35c2f6b69281093195b1
Author: Eric Yang <ey...@apache.org>
AuthorDate: Fri Oct 18 20:30:11 2019 -0400
YARN-9875. Improve fair scheduler configuration store on HDFS.
Contributed by Prabhu Joseph
---
.../conf/FSSchedulerConfigurationStore.java | 12 ++++-
.../conf/TestFSSchedulerConfigurationStore.java | 51 ++++++++++++++++++++++
2 files changed, 62 insertions(+), 1 deletion(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
index 464ef14..855939e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
@@ -66,7 +66,7 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore {
private Path configVersionFile;
@Override
- public void initialize(Configuration conf, Configuration vSchedConf,
+ public void initialize(Configuration fsConf, Configuration vSchedConf,
RMContext rmContext) throws Exception {
this.configFilePathFilter = new PathFilter() {
@Override
@@ -80,6 +80,7 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore {
}
};
+ Configuration conf = new Configuration(fsConf);
String schedulerConfPathStr = conf.get(
YarnConfiguration.SCHEDULER_CONFIGURATION_FS_PATH);
if (schedulerConfPathStr == null || schedulerConfPathStr.isEmpty()) {
@@ -88,6 +89,15 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore {
+ " must be set");
}
this.schedulerConfDir = new Path(schedulerConfPathStr);
+ String scheme = schedulerConfDir.toUri().getScheme();
+ if (scheme == null) {
+ scheme = FileSystem.getDefaultUri(conf).getScheme();
+ }
+ if (scheme != null) {
+ String disableCacheName = String.format("fs.%s.impl.disable.cache",
+ scheme);
+ conf.setBoolean(disableCacheName, true);
+ }
this.fileSystem = this.schedulerConfDir.getFileSystem(conf);
this.maxVersion = conf.getInt(
YarnConfiguration.SCHEDULER_CONFIGURATION_FS_MAX_VERSION,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
index 33596c3..7968372 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
import org.junit.After;
@@ -138,6 +140,55 @@ public class TestFSSchedulerConfigurationStore {
}
@Test
+ public void testFileSystemClose() throws Exception {
+ MiniDFSCluster hdfsCluster = null;
+ FileSystem fs = null;
+ Path path = new Path("/tmp/confstore");
+ try {
+ HdfsConfiguration hdfsConfig = new HdfsConfiguration();
+ hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig)
+ .numDataNodes(1).build();
+
+ fs = hdfsCluster.getFileSystem();
+ if (!fs.exists(path)) {
+ fs.mkdirs(path);
+ }
+
+ FSSchedulerConfigurationStore configStore =
+ new FSSchedulerConfigurationStore();
+ hdfsConfig.set(YarnConfiguration.SCHEDULER_CONFIGURATION_FS_PATH,
+ path.toString());
+ configStore.initialize(hdfsConfig, hdfsConfig, null);
+
+ // Close the FileSystem object and validate
+ fs.close();
+
+ try {
+ Map<String, String> updates = new HashMap<>();
+ updates.put("testkey", "testvalue");
+ LogMutation logMutation = new LogMutation(updates, "test");
+ configStore.logMutation(logMutation);
+ configStore.confirmMutation(true);
+ } catch (IOException e) {
+ if (e.getMessage().contains("Filesystem closed")) {
+ fail("FSSchedulerConfigurationStore failed to handle " +
+ "FileSystem close");
+ } else {
+ fail("Should not get any exceptions");
+ }
+ }
+ } finally {
+ fs = hdfsCluster.getFileSystem();
+ if (fs.exists(path)) {
+ fs.delete(path, true);
+ }
+ if (hdfsCluster != null) {
+ hdfsCluster.shutdown();
+ }
+ }
+ }
+
+ @Test
public void testFormatConfiguration() throws Exception {
Configuration schedulerConf = new Configuration();
schedulerConf.set("a", "a");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org