You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by yq...@apache.org on 2018/02/09 05:58:17 UTC
hadoop git commit: HDFS-13099. RBF: Use the ZooKeeper as the default
State Store. Contributed by Yiqun Lin.
Repository: hadoop
Updated Branches:
refs/heads/trunk ddec08d7c -> 543f3abbe
HDFS-13099. RBF: Use the ZooKeeper as the default State Store. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/543f3abb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/543f3abb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/543f3abb
Branch: refs/heads/trunk
Commit: 543f3abbee79d7ec70353f0cdda6397ee001324e
Parents: ddec08d
Author: Yiqun Lin <yq...@apache.org>
Authored: Fri Feb 9 13:57:42 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Fri Feb 9 13:57:42 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++--
.../src/main/resources/hdfs-default.xml | 10 ++++++++--
.../src/site/markdown/HDFSRouterFederation.md | 2 +-
.../server/federation/RouterConfigBuilder.java | 6 ++++++
.../store/FederationStateStoreTestUtils.java | 17 +++++++++++------
5 files changed, 28 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543f3abb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index e0b5b85..c0ad4ec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -34,8 +34,8 @@ import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCPerformance
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
-import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl;
import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl;
import org.apache.hadoop.http.HttpConfig;
/**
@@ -1275,7 +1275,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String FEDERATION_STORE_DRIVER_CLASS =
FEDERATION_STORE_PREFIX + "driver.class";
public static final Class<? extends StateStoreDriver>
- FEDERATION_STORE_DRIVER_CLASS_DEFAULT = StateStoreFileImpl.class;
+ FEDERATION_STORE_DRIVER_CLASS_DEFAULT = StateStoreZooKeeperImpl.class;
public static final String FEDERATION_STORE_CONNECTION_TEST_MS =
FEDERATION_STORE_PREFIX + "connection.test";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543f3abb/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 59df122..f6d232e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -5085,9 +5085,15 @@
<property>
<name>dfs.federation.router.store.driver.class</name>
- <value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl</value>
+ <value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl</value>
<description>
- Class to implement the State Store. By default it uses the local disk.
+ Class to implement the State Store. There are three implementation classes currently
+ being supported:
+ org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl,
+ org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileSystemImpl and
+ org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl.
+ These implementation classes use the local file, filesystem and ZooKeeper as a backend respectively.
+ By default it uses the ZooKeeper as the default State Store.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543f3abb/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
index 5649755..ebe94a0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
@@ -325,7 +325,7 @@ The connection to the State Store and the internal caching at the Router.
|:---- |:---- |:---- |
| dfs.federation.router.store.enable | `true` | If `true`, the Router connects to the State Store. |
| dfs.federation.router.store.serializer | `StateStoreSerializerPBImpl` | Class to serialize State Store records. |
-| dfs.federation.router.store.driver.class | `StateStoreZKImpl` | Class to implement the State Store. |
+| dfs.federation.router.store.driver.class | `StateStoreZooKeeperImpl` | Class to implement the State Store. |
| dfs.federation.router.store.connection.test | 60000 | How often to check for the connection to the State Store in milliseconds. |
| dfs.federation.router.cache.ttl | 60000 | How often to refresh the State Store caches in milliseconds. |
| dfs.federation.router.store.membership.expiration | 300000 | Expiration time in milliseconds for a membership record. |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543f3abb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
index 3659bf9..b332f1f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.hdfs.server.federation;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
/**
* Constructs a router configuration with individual features enabled/disabled.
@@ -119,6 +121,10 @@ public class RouterConfigBuilder {
}
public RouterConfigBuilder stateStore() {
+ // reset the State Store driver implementation class for testing
+ conf.setClass(DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS,
+ FederationStateStoreTestUtils.getTestDriverClass(),
+ StateStoreDriver.class);
return this.stateStore(true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543f3abb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
index dbb8f3f..def3935 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileBaseImpl;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl;
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
@@ -50,17 +51,21 @@ import org.apache.hadoop.util.Time;
*/
public final class FederationStateStoreTestUtils {
+ /** The State Store Driver implementation class for testing .*/
+ private static final Class<? extends StateStoreDriver>
+ FEDERATION_STORE_DRIVER_CLASS_FOR_TEST = StateStoreFileImpl.class;
+
private FederationStateStoreTestUtils() {
// Utility Class
}
/**
- * Get the default State Store driver implementation.
+ * Get the State Store driver implementation for testing.
*
- * @return Class of the default State Store driver implementation.
+ * @return Class of the State Store driver implementation.
*/
- public static Class<? extends StateStoreDriver> getDefaultDriver() {
- return DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS_DEFAULT;
+ public static Class<? extends StateStoreDriver> getTestDriverClass() {
+ return FEDERATION_STORE_DRIVER_CLASS_FOR_TEST;
}
/**
@@ -69,7 +74,7 @@ public final class FederationStateStoreTestUtils {
* @return State Store configuration.
*/
public static Configuration getStateStoreConfiguration() {
- Class<? extends StateStoreDriver> clazz = getDefaultDriver();
+ Class<? extends StateStoreDriver> clazz = getTestDriverClass();
return getStateStoreConfiguration(clazz);
}
@@ -146,7 +151,7 @@ public final class FederationStateStoreTestUtils {
* @throws IOException
*/
public static void deleteStateStore() throws IOException {
- Class<? extends StateStoreDriver> driverClass = getDefaultDriver();
+ Class<? extends StateStoreDriver> driverClass = getTestDriverClass();
deleteStateStore(driverClass);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org