You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2019/03/03 18:36:46 UTC
[hadoop] 07/45: HDFS-13852. RBF: The DN_REPORT_TIME_OUT and
DN_REPORT_CACHE_EXPIRE should be configured in RBFConfigKeys. Contributed
by yanghuafeng.
This is an automated email from the ASF dual-hosted git repository.
inigoiri pushed a commit to branch HDFS-13891
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit a04622148ab506e42bf1be2d367d15cf80213b14
Author: Inigo Goiri <in...@apache.org>
AuthorDate: Tue Nov 13 10:14:35 2018 -0800
HDFS-13852. RBF: The DN_REPORT_TIME_OUT and DN_REPORT_CACHE_EXPIRE should be configured in RBFConfigKeys. Contributed by yanghuafeng.
---
.../federation/metrics/FederationMetrics.java | 12 ++++++++++--
.../federation/metrics/NamenodeBeanMetrics.java | 22 ++++------------------
.../server/federation/router/RBFConfigKeys.java | 7 +++++++
.../src/main/resources/hdfs-rbf-default.xml | 17 +++++++++++++++++
.../router/TestRouterRPCClientRetries.java | 2 +-
.../server/federation/router/TestRouterRpc.java | 2 +-
6 files changed, 40 insertions(+), 22 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
index 23f62b6..6a0a46e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
@@ -47,12 +47,14 @@ import javax.management.NotCompliantMBeanException;
import javax.management.ObjectName;
import javax.management.StandardMBean;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
import org.apache.hadoop.hdfs.server.federation.router.Router;
import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer;
import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
@@ -95,7 +97,7 @@ public class FederationMetrics implements FederationMBean {
private static final String DATE_FORMAT = "yyyy/MM/dd HH:mm:ss";
/** Prevent holding the page from load too long. */
- private static final long TIME_OUT = TimeUnit.SECONDS.toMillis(1);
+ private final long timeOut;
/** Router interface. */
@@ -143,6 +145,12 @@ public class FederationMetrics implements FederationMBean {
this.routerStore = stateStore.getRegisteredRecordStore(
RouterStore.class);
}
+
+ // Initialize the cache for the DN reports
+ Configuration conf = router.getConfig();
+ this.timeOut = conf.getTimeDuration(RBFConfigKeys.DN_REPORT_TIME_OUT,
+ RBFConfigKeys.DN_REPORT_TIME_OUT_MS_DEFAULT, TimeUnit.MILLISECONDS);
+
}
/**
@@ -434,7 +442,7 @@ public class FederationMetrics implements FederationMBean {
try {
RouterRpcServer rpcServer = this.router.getRpcServer();
DatanodeInfo[] live = rpcServer.getDatanodeReport(
- DatanodeReportType.LIVE, false, TIME_OUT);
+ DatanodeReportType.LIVE, false, timeOut);
if (live.length > 0) {
float totalDfsUsed = 0;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
index 0ca5f73..64df10c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
@@ -74,21 +74,6 @@ public class NamenodeBeanMetrics
private static final Logger LOG =
LoggerFactory.getLogger(NamenodeBeanMetrics.class);
- /** Prevent holding the page from loading too long. */
- private static final String DN_REPORT_TIME_OUT =
- RBFConfigKeys.FEDERATION_ROUTER_PREFIX + "dn-report.time-out";
- /** We only wait for 1 second. */
- private static final long DN_REPORT_TIME_OUT_DEFAULT =
- TimeUnit.SECONDS.toMillis(1);
-
- /** Time to cache the DN information. */
- public static final String DN_REPORT_CACHE_EXPIRE =
- RBFConfigKeys.FEDERATION_ROUTER_PREFIX + "dn-report.cache-expire";
- /** We cache the DN information for 10 seconds by default. */
- public static final long DN_REPORT_CACHE_EXPIRE_DEFAULT =
- TimeUnit.SECONDS.toMillis(10);
-
-
/** Instance of the Router being monitored. */
private final Router router;
@@ -148,10 +133,11 @@ public class NamenodeBeanMetrics
// Initialize the cache for the DN reports
Configuration conf = router.getConfig();
this.dnReportTimeOut = conf.getTimeDuration(
- DN_REPORT_TIME_OUT, DN_REPORT_TIME_OUT_DEFAULT, TimeUnit.MILLISECONDS);
+ RBFConfigKeys.DN_REPORT_TIME_OUT,
+ RBFConfigKeys.DN_REPORT_TIME_OUT_MS_DEFAULT, TimeUnit.MILLISECONDS);
long dnCacheExpire = conf.getTimeDuration(
- DN_REPORT_CACHE_EXPIRE,
- DN_REPORT_CACHE_EXPIRE_DEFAULT, TimeUnit.MILLISECONDS);
+ RBFConfigKeys.DN_REPORT_CACHE_EXPIRE,
+ RBFConfigKeys.DN_REPORT_CACHE_EXPIRE_MS_DEFAULT, TimeUnit.MILLISECONDS);
this.dnCache = CacheBuilder.newBuilder()
.expireAfterWrite(dnCacheExpire, TimeUnit.MILLISECONDS)
.build(
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
index fa474f4..dd72e36 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
@@ -233,6 +233,13 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic {
FEDERATION_ROUTER_PREFIX + "https-bind-host";
public static final String DFS_ROUTER_HTTPS_ADDRESS_DEFAULT =
"0.0.0.0:" + DFS_ROUTER_HTTPS_PORT_DEFAULT;
+ public static final String DN_REPORT_TIME_OUT =
+ FEDERATION_ROUTER_PREFIX + "dn-report.time-out";
+ public static final long DN_REPORT_TIME_OUT_MS_DEFAULT = 1000;
+ public static final String DN_REPORT_CACHE_EXPIRE =
+ FEDERATION_ROUTER_PREFIX + "dn-report.cache-expire";
+ public static final long DN_REPORT_CACHE_EXPIRE_MS_DEFAULT =
+ TimeUnit.SECONDS.toMillis(10);
// HDFS Router-based federation quota
public static final String DFS_ROUTER_QUOTA_ENABLE =
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
index 29c3093..53bf53a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
@@ -143,6 +143,23 @@
</description>
</property>
+
+ <property>
+ <name>dfs.federation.router.dn-report.time-out</name>
+ <value>1000</value>
+ <description>
+ Time out, in milliseconds for getDatanodeReport.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.dn-report.cache-expire</name>
+ <value>10s</value>
+ <description>
+ Expiration time in seconds for datanodereport.
+ </description>
+ </property>
+
<property>
<name>dfs.federation.router.metrics.class</name>
<value>org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCPerformanceMonitor</value>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
index e5ab3ab..f84e9a0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
@@ -81,7 +81,7 @@ public class TestRouterRPCClientRetries {
.rpc()
.build();
routerConf.setTimeDuration(
- NamenodeBeanMetrics.DN_REPORT_CACHE_EXPIRE, 1, TimeUnit.SECONDS);
+ RBFConfigKeys.DN_REPORT_CACHE_EXPIRE, 1, TimeUnit.SECONDS);
// reduce IPC client connection retry times and interval time
Configuration clientConf = new Configuration(false);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
index a32cba1..204366e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
@@ -175,7 +175,7 @@ public class TestRouterRpc {
.build();
// We decrease the DN cache times to make the test faster
routerConf.setTimeDuration(
- NamenodeBeanMetrics.DN_REPORT_CACHE_EXPIRE, 1, TimeUnit.SECONDS);
+ RBFConfigKeys.DN_REPORT_CACHE_EXPIRE, 1, TimeUnit.SECONDS);
cluster.addRouterOverrides(routerConf);
cluster.startRouters();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org