You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xk...@apache.org on 2019/03/13 20:19:58 UTC
[hadoop] branch branch-3.2 updated: HDFS-14346. Add better time
precision to Configuration#getTimeDuration,
allowing return unit and default unit to be specified independently.
Contributed by Chao Sun.
This is an automated email from the ASF dual-hosted git repository.
xkrogen pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new fec7c5f HDFS-14346. Add better time precision to Configuration#getTimeDuration, allowing return unit and default unit to be specified independently. Contributed by Chao Sun.
fec7c5f is described below
commit fec7c5f3ebbaea7b290e0904570c3a485b541a22
Author: Erik Krogen <xk...@apache.org>
AuthorDate: Wed Mar 13 09:38:10 2019 -0700
HDFS-14346. Add better time precision to Configuration#getTimeDuration, allowing return unit and default unit to be specified independently. Contributed by Chao Sun.
(cherry picked from commit 66357574ae1da09ced735da36bf7d80a40c3fa1b)
---
.../java/org/apache/hadoop/conf/Configuration.java | 60 ++++++++++++++++++----
.../org/apache/hadoop/conf/TestConfiguration.java | 5 ++
.../hadoop/hdfs/client/impl/DfsClientConf.java | 2 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 4 +-
.../hdfs/server/blockmanagement/BlockManager.java | 2 +-
.../BlockPlacementPolicyDefault.java | 3 +-
.../apache/hadoop/hdfs/server/datanode/DNConf.java | 9 ++--
.../org/apache/hadoop/hdfs/server/mover/Mover.java | 4 +-
.../hdfs/server/namenode/ha/EditLogTailer.java | 11 ++--
9 files changed, 77 insertions(+), 23 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 50810ab..d3be357 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -1777,6 +1777,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* Return time duration in the given time unit. Valid units are encoded in
* properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
* (ms), seconds (s), minutes (m), hours (h), and days (d).
+ *
* @param name Property name
* @param defaultValue Value returned if no mapping exists.
* @param unit Unit to convert the stored property, if it exists.
@@ -1784,20 +1785,44 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* a number
*/
public long getTimeDuration(String name, long defaultValue, TimeUnit unit) {
+ return getTimeDuration(name, defaultValue, unit, unit);
+ }
+
+ public long getTimeDuration(String name, String defaultValue, TimeUnit unit) {
+ return getTimeDuration(name, defaultValue, unit, unit);
+ }
+
+ /**
+ * Return time duration in the given time unit. Valid units are encoded in
+ * properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ * (ms), seconds (s), minutes (m), hours (h), and days (d). If no unit is
+ * provided, the default unit is applied.
+ *
+ * @param name Property name
+ * @param defaultValue Value returned if no mapping exists.
+ * @param defaultUnit Default time unit if no valid suffix is provided.
+ * @param returnUnit The unit used for the returned value.
+ * @throws NumberFormatException If the property stripped of its unit is not
+ * a number
+ * @return time duration in given time unit
+ */
+ public long getTimeDuration(String name, long defaultValue,
+ TimeUnit defaultUnit, TimeUnit returnUnit) {
String vStr = get(name);
if (null == vStr) {
return defaultValue;
} else {
- return getTimeDurationHelper(name, vStr, unit);
+ return getTimeDurationHelper(name, vStr, defaultUnit, returnUnit);
}
}
- public long getTimeDuration(String name, String defaultValue, TimeUnit unit) {
+ public long getTimeDuration(String name, String defaultValue,
+ TimeUnit defaultUnit, TimeUnit returnUnit) {
String vStr = get(name);
if (null == vStr) {
- return getTimeDurationHelper(name, defaultValue, unit);
+ return getTimeDurationHelper(name, defaultValue, defaultUnit, returnUnit);
} else {
- return getTimeDurationHelper(name, vStr, unit);
+ return getTimeDurationHelper(name, vStr, defaultUnit, returnUnit);
}
}
@@ -1805,26 +1830,43 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* Return time duration in the given time unit. Valid units are encoded in
* properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
* (ms), seconds (s), minutes (m), hours (h), and days (d).
+ *
* @param name Property name
* @param vStr The string value with time unit suffix to be converted.
* @param unit Unit to convert the stored property, if it exists.
*/
public long getTimeDurationHelper(String name, String vStr, TimeUnit unit) {
+ return getTimeDurationHelper(name, vStr, unit, unit);
+ }
+
+ /**
+ * Return time duration in the given time unit. Valid units are encoded in
+ * properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ * (ms), seconds (s), minutes (m), hours (h), and days (d).
+ *
+ * @param name Property name
+ * @param vStr The string value with time unit suffix to be converted.
+ * @param defaultUnit Unit to convert the stored property, if it exists.
+ * @param returnUnit Unit for the returned value.
+ */
+ private long getTimeDurationHelper(String name, String vStr,
+ TimeUnit defaultUnit, TimeUnit returnUnit) {
vStr = vStr.trim();
vStr = StringUtils.toLowerCase(vStr);
ParsedTimeDuration vUnit = ParsedTimeDuration.unitFor(vStr);
if (null == vUnit) {
- logDeprecation("No unit for " + name + "(" + vStr + ") assuming " + unit);
- vUnit = ParsedTimeDuration.unitFor(unit);
+ logDeprecation("No unit for " + name + "(" + vStr + ") assuming " +
+ defaultUnit);
+ vUnit = ParsedTimeDuration.unitFor(defaultUnit);
} else {
vStr = vStr.substring(0, vStr.lastIndexOf(vUnit.suffix()));
}
long raw = Long.parseLong(vStr);
- long converted = unit.convert(raw, vUnit.unit());
- if (vUnit.unit().convert(converted, unit) < raw) {
+ long converted = returnUnit.convert(raw, vUnit.unit());
+ if (vUnit.unit().convert(converted, returnUnit) < raw) {
logDeprecation("Possible loss of precision converting " + vStr
- + vUnit.suffix() + " to " + unit + " for " + name);
+ + vUnit.suffix() + " to " + returnUnit + " for " + name);
}
return converted;
}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index 4c80d10..1cf7a20 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -1404,7 +1404,10 @@ public class TestConfiguration {
conf.setTimeDuration("test.time.a", 7L, SECONDS);
assertEquals("7s", conf.get("test.time.a"));
assertEquals(0L, conf.getTimeDuration("test.time.a", 30, MINUTES));
+ assertEquals(0L, conf.getTimeDuration("test.time.a", 30, SECONDS, MINUTES));
assertEquals(7L, conf.getTimeDuration("test.time.a", 30, SECONDS));
+ assertEquals(7L,
+ conf.getTimeDuration("test.time.a", 30, MILLISECONDS, SECONDS));
assertEquals(7000L, conf.getTimeDuration("test.time.a", 30, MILLISECONDS));
assertEquals(7000000L,
conf.getTimeDuration("test.time.a", 30, MICROSECONDS));
@@ -1421,6 +1424,8 @@ public class TestConfiguration {
assertEquals(30L, conf.getTimeDuration("test.time.X", 30, SECONDS));
conf.set("test.time.X", "30");
assertEquals(30L, conf.getTimeDuration("test.time.X", 40, SECONDS));
+ assertEquals(30000L,
+ conf.getTimeDuration("test.time.X", 40, SECONDS, MILLISECONDS));
assertEquals(10L, conf.getTimeDuration("test.time.c", "10", SECONDS));
assertEquals(30L, conf.getTimeDuration("test.time.c", "30s", SECONDS));
assertEquals(120L, conf.getTimeDuration("test.time.c", "2m", SECONDS));
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
index e63e3f5..8120998 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
@@ -244,7 +244,7 @@ public class DfsClientConf {
datanodeRestartTimeout = conf.getTimeDuration(
DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY,
DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT,
- TimeUnit.SECONDS) * 1000;
+ TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
slowIoWarningThresholdMs = conf.getLong(
DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY,
DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index a58e391..d9b50c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -676,11 +676,11 @@ public class Balancer {
final long sleeptime =
conf.getTimeDuration(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT,
- TimeUnit.SECONDS) * 2000 +
+ TimeUnit.SECONDS, TimeUnit.MILLISECONDS) * 2 +
conf.getTimeDuration(
DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT,
- TimeUnit.SECONDS) * 1000;
+ TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
LOG.info("namenodes = " + namenodes);
LOG.info("parameters = " + p);
LOG.info("included nodes = " + p.getIncludedNodes());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 8eb7670..a9915fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -519,7 +519,7 @@ public class BlockManager implements BlockStatsMXBean {
this.redundancyRecheckIntervalMs = conf.getTimeDuration(
DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT,
- TimeUnit.SECONDS) * 1000;
+ TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
this.storageInfoDefragmentInterval =
conf.getLong(
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index d396845..2424b0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -121,7 +121,8 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
this.host2datanodeMap = host2datanodeMap;
this.heartbeatInterval = conf.getTimeDuration(
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS) * 1000;
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
this.tolerateHeartbeatMultiplier = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_KEY,
DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_DEFAULT);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
index 8e5b597..2f0ef78 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
@@ -199,7 +199,8 @@ public class DNConf {
long initBRDelay = getConf().getTimeDuration(
DFS_BLOCKREPORT_INITIAL_DELAY_KEY,
- DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT, TimeUnit.SECONDS) * 1000L;
+ DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT,
+ TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
if (initBRDelay >= blockReportInterval) {
initBRDelay = 0;
DataNode.LOG.info("dfs.blockreport.initialDelay is "
@@ -209,11 +210,13 @@ public class DNConf {
initialBlockReportDelayMs = initBRDelay;
heartBeatInterval = getConf().getTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY,
- DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS) * 1000L;
+ DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS,
+ TimeUnit.MILLISECONDS);
long confLifelineIntervalMs =
getConf().getLong(DFS_DATANODE_LIFELINE_INTERVAL_SECONDS_KEY,
3 * getConf().getTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY,
- DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS)) * 1000L;
+ DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS,
+ TimeUnit.MILLISECONDS));
if (confLifelineIntervalMs <= heartBeatInterval) {
confLifelineIntervalMs = 3 * heartBeatInterval;
DataNode.LOG.warn(
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index 80df711..06c7cc5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -631,11 +631,11 @@ public class Mover {
final long sleeptime =
conf.getTimeDuration(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT,
- TimeUnit.SECONDS) * 2000 +
+ TimeUnit.SECONDS, TimeUnit.MILLISECONDS) * 2 +
conf.getTimeDuration(
DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT,
- TimeUnit.SECONDS) * 1000;
+ TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
AtomicInteger retryCount = new AtomicInteger(0);
// TODO: Need to limit the size of the pinned blocks to limit memory usage
Map<Long, Set<DatanodeInfo>> excludedPinnedBlocks = new HashMap<>();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
index a67d2eb..549ad57 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
@@ -172,7 +172,8 @@ public class EditLogTailer {
logRollPeriodMs = conf.getTimeDuration(
DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY,
- DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_DEFAULT, TimeUnit.SECONDS) * 1000;
+ DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_DEFAULT,
+ TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
List<RemoteNameNodeInfo> nns = Collections.emptyList();
if (logRollPeriodMs >= 0) {
try {
@@ -199,11 +200,13 @@ public class EditLogTailer {
sleepTimeMs = conf.getTimeDuration(
DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY,
- DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_DEFAULT, TimeUnit.SECONDS) * 1000;
+ DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_DEFAULT,
+ TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
- rollEditsTimeoutMs = conf.getInt(
+ rollEditsTimeoutMs = conf.getTimeDuration(
DFSConfigKeys.DFS_HA_TAILEDITS_ROLLEDITS_TIMEOUT_KEY,
- DFSConfigKeys.DFS_HA_TAILEDITS_ROLLEDITS_TIMEOUT_DEFAULT) * 1000;
+ DFSConfigKeys.DFS_HA_TAILEDITS_ROLLEDITS_TIMEOUT_DEFAULT,
+ TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
rollEditsRpcExecutor = Executors.newSingleThreadExecutor(
new ThreadFactoryBuilder().setDaemon(true).build());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org