You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/03/18 04:34:38 UTC
[46/50] hadoop git commit: YARN-3205. FileSystemRMStateStore should
disable FileSystem Cache to avoid get a Filesystem with an old configuration.
Contributed by Zhihai Xu.
YARN-3205. FileSystemRMStateStore should disable FileSystem Cache to avoid get a Filesystem with an old configuration. Contributed by Zhihai Xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3bc72cc1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3bc72cc1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3bc72cc1
Branch: refs/heads/YARN-2928
Commit: 3bc72cc16d8c7b8addd8f565523001dfcc32b891
Parents: fc90bf7
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Wed Mar 18 11:53:14 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Wed Mar 18 11:53:19 2015 +0900
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../recovery/FileSystemRMStateStore.java | 22 +++++++++++++++-----
.../recovery/TestFSRMStateStore.java | 5 +++++
3 files changed, 25 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bc72cc1/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index bb752ab..c869113 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -72,6 +72,9 @@ Release 2.8.0 - UNRELEASED
YARN-3305. Normalize AM resource request on app submission. (Rohith Sharmaks
via jianhe)
+ YARN-3205 FileSystemRMStateStore should disable FileSystem Cache to avoid
+ get a Filesystem with an old configuration. (Zhihai Xu via ozawa)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bc72cc1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
index 8147597..7652a07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
@@ -84,7 +84,10 @@ public class FileSystemRMStateStore extends RMStateStore {
protected static final String AMRMTOKEN_SECRET_MANAGER_NODE =
"AMRMTokenSecretManagerNode";
+ @VisibleForTesting
protected FileSystem fs;
+ @VisibleForTesting
+ protected Configuration fsConf;
private Path rootDirPath;
@Private
@@ -121,14 +124,23 @@ public class FileSystemRMStateStore extends RMStateStore {
// create filesystem only now, as part of service-start. By this time, RM is
// authenticated with kerberos so we are good to create a file-system
// handle.
- Configuration conf = new Configuration(getConfig());
- conf.setBoolean("dfs.client.retry.policy.enabled", true);
+ fsConf = new Configuration(getConfig());
+ fsConf.setBoolean("dfs.client.retry.policy.enabled", true);
String retryPolicy =
- conf.get(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC,
+ fsConf.get(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC,
YarnConfiguration.DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC);
- conf.set("dfs.client.retry.policy.spec", retryPolicy);
+ fsConf.set("dfs.client.retry.policy.spec", retryPolicy);
+
+ String scheme = fsWorkingPath.toUri().getScheme();
+ if (scheme == null) {
+ scheme = FileSystem.getDefaultUri(fsConf).getScheme();
+ }
+ if (scheme != null) {
+ String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme);
+ fsConf.setBoolean(disableCacheName, true);
+ }
- fs = fsWorkingPath.getFileSystem(conf);
+ fs = fsWorkingPath.getFileSystem(fsConf);
mkdirsWithRetries(rmDTSecretManagerRoot);
mkdirsWithRetries(rmAppRoot);
mkdirsWithRetries(amrmTokenSecretManagerRoot);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bc72cc1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
index d2eddd6..c5f5578 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
@@ -106,6 +106,11 @@ public class TestFSRMStateStore extends RMStateStoreTestBase {
this.store = new TestFileSystemRMStore(conf);
Assert.assertEquals(store.getNumRetries(), 8);
Assert.assertEquals(store.getRetryInterval(), 900L);
+ Assert.assertTrue(store.fs.getConf() == store.fsConf);
+ FileSystem previousFs = store.fs;
+ store.startInternal();
+ Assert.assertTrue(store.fs != previousFs);
+ Assert.assertTrue(store.fs.getConf() == store.fsConf);
return store;
}