You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/08/26 00:44:42 UTC
[30/50] [abbrv] hadoop git commit: HDFS-13821. RBF: Add
dfs.federation.router.mount-table.cache.enable so that users can disable
cache. Contributed by Fei Hui.
HDFS-13821. RBF: Add dfs.federation.router.mount-table.cache.enable so that users can disable cache. Contributed by Fei Hui.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81847392
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81847392
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81847392
Branch: refs/heads/HDFS-12943
Commit: 81847392badcd58d934333e7c3b5bf14b4fa1f3f
Parents: e557c6b
Author: Yiqun Lin <yq...@apache.org>
Authored: Wed Aug 22 11:43:40 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Wed Aug 22 11:43:40 2018 +0800
----------------------------------------------------------------------
.../federation/resolver/MountTableResolver.java | 37 ++++++++++++++------
.../server/federation/router/RBFConfigKeys.java | 4 +++
.../src/main/resources/hdfs-rbf-default.xml | 9 +++++
.../resolver/TestMountTableResolver.java | 31 ++++++++++++++++
4 files changed, 71 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81847392/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
index c264de3..d45441f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
@@ -22,6 +22,8 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DeprecatedKeys.
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_CACHE_ENABLE;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_CACHE_ENABLE_DEFAULT;
import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.isParentEntry;
import java.io.IOException;
@@ -124,12 +126,19 @@ public class MountTableResolver
this.stateStore = null;
}
- int maxCacheSize = conf.getInt(
- FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE,
- FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT);
- this.locationCache = CacheBuilder.newBuilder()
- .maximumSize(maxCacheSize)
- .build();
+ boolean mountTableCacheEnable = conf.getBoolean(
+ FEDERATION_MOUNT_TABLE_CACHE_ENABLE,
+ FEDERATION_MOUNT_TABLE_CACHE_ENABLE_DEFAULT);
+ if (mountTableCacheEnable) {
+ int maxCacheSize = conf.getInt(
+ FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE,
+ FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT);
+ this.locationCache = CacheBuilder.newBuilder()
+ .maximumSize(maxCacheSize)
+ .build();
+ } else {
+ this.locationCache = null;
+ }
registerCacheExternal();
initDefaultNameService(conf);
@@ -239,7 +248,7 @@ public class MountTableResolver
*/
private void invalidateLocationCache(final String path) {
LOG.debug("Invalidating {} from {}", path, locationCache);
- if (locationCache.size() == 0) {
+ if (locationCache == null || locationCache.size() == 0) {
return;
}
@@ -359,7 +368,9 @@ public class MountTableResolver
LOG.info("Clearing all mount location caches");
writeLock.lock();
try {
- this.locationCache.invalidateAll();
+ if (this.locationCache != null) {
+ this.locationCache.invalidateAll();
+ }
this.tree.clear();
} finally {
writeLock.unlock();
@@ -372,6 +383,9 @@ public class MountTableResolver
verifyMountTable();
readLock.lock();
try {
+ if (this.locationCache == null) {
+ return lookupLocation(path);
+ }
Callable<? extends PathLocation> meh = new Callable<PathLocation>() {
@Override
public PathLocation call() throws Exception {
@@ -603,7 +617,10 @@ public class MountTableResolver
* Get the size of the cache.
* @return Size of the cache.
*/
- protected long getCacheSize() {
- return this.locationCache.size();
+ protected long getCacheSize() throws IOException{
+ if (this.locationCache != null) {
+ return this.locationCache.size();
+ }
+ throw new IOException("localCache is null");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81847392/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
index 363db20..87df5d2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
@@ -186,6 +186,10 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic {
FEDERATION_ROUTER_PREFIX + "mount-table.max-cache-size";
/** Remove cache entries if we have more than 10k. */
public static final int FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT = 10000;
+ public static final String FEDERATION_MOUNT_TABLE_CACHE_ENABLE =
+ FEDERATION_ROUTER_PREFIX + "mount-table.cache.enable";
+ public static final boolean FEDERATION_MOUNT_TABLE_CACHE_ENABLE_DEFAULT =
+ true;
// HDFS Router-based federation admin
public static final String DFS_ROUTER_ADMIN_HANDLER_COUNT_KEY =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81847392/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
index 8806cb2..8be5b8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
@@ -394,6 +394,15 @@
</property>
<property>
+ <name>dfs.federation.router.mount-table.cache.enable</name>
+ <value>true</value>
+ <description>
+ Set to true to enable mount table cache (Path to Remote Location cache).
+ Disabling the cache is recommended when a large amount of unique paths are queried.
+ </description>
+ </property>
+
+ <property>
<name>dfs.federation.router.quota.enable</name>
<value>false</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81847392/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
index cb3b472..b19a973 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.federation.resolver;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_CACHE_ENABLE;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
import static org.junit.Assert.assertEquals;
@@ -37,6 +38,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.server.federation.router.Router;
import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
@@ -473,6 +475,35 @@ public class TestMountTableResolver {
}
@Test
+ public void testDisableLocalCache() throws IOException {
+ Configuration conf = new Configuration();
+ // Disable mount table cache
+ conf.setBoolean(FEDERATION_MOUNT_TABLE_CACHE_ENABLE, false);
+ conf.setStrings(DFS_ROUTER_DEFAULT_NAMESERVICE, "0");
+ MountTableResolver tmpMountTable = new MountTableResolver(conf);
+
+ // Root mount point
+ Map<String, String> map = getMountTableEntry("1", "/");
+ tmpMountTable.addEntry(MountTable.newInstance("/", map));
+
+ // /tmp
+ map = getMountTableEntry("2", "/tmp");
+ tmpMountTable.addEntry(MountTable.newInstance("/tmp", map));
+
+ // Check localCache is null
+ try {
+ tmpMountTable.getCacheSize();
+ fail("getCacheSize call should fail.");
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains("localCache is null", e);
+ }
+
+ // Check resolve path without cache
+ assertEquals("2->/tmp/tesfile1.txt",
+ tmpMountTable.getDestinationForPath("/tmp/tesfile1.txt").toString());
+ }
+
+ @Test
public void testCacheCleaning() throws Exception {
for (int i = 0; i < 1000; i++) {
String filename = String.format("/user/a/file-%04d.txt", i);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org