You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2018/10/19 17:38:10 UTC
hadoop git commit: HDFS-9872. HDFS bytes-default configurations
should accept multiple size units. Contributed by Yiqun Lin.
Repository: hadoop
Updated Branches:
refs/heads/trunk 8b64fbab1 -> 88cce3255
HDFS-9872. HDFS bytes-default configurations should accept multiple size units. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88cce325
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88cce325
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88cce325
Branch: refs/heads/trunk
Commit: 88cce32551e6d52fd1c5a5bfd6c41499bf6ab1ab
Parents: 8b64fba
Author: Inigo Goiri <in...@apache.org>
Authored: Fri Oct 19 10:38:04 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Fri Oct 19 10:38:04 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 2 +-
.../hadoop/hdfs/server/datanode/DNConf.java | 2 +-
.../AvailableSpaceVolumeChoosingPolicy.java | 2 +-
.../fsdataset/impl/ReservedSpaceCalculator.java | 2 +-
.../hdfs/server/namenode/FSDirectory.java | 4 +--
.../hdfs/server/namenode/FSNamesystem.java | 3 +-
.../hdfs/server/namenode/ImageServlet.java | 8 ++---
.../namenode/NameNodeResourceChecker.java | 2 +-
.../hdfs/server/namenode/TransferFsImage.java | 2 +-
.../src/main/resources/hdfs-default.xml | 32 +++++++++++++++-----
10 files changed, 38 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88cce325/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index ce1083d..2badbb1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -371,7 +371,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
(conf.get(DFS_CLIENT_CACHE_DROP_BEHIND_READS) == null) ?
null : conf.getBoolean(DFS_CLIENT_CACHE_DROP_BEHIND_READS, false);
Long readahead = (conf.get(DFS_CLIENT_CACHE_READAHEAD) == null) ?
- null : conf.getLong(DFS_CLIENT_CACHE_READAHEAD, 0);
+ null : conf.getLongBytes(DFS_CLIENT_CACHE_READAHEAD, 0);
this.serverDefaultsValidityPeriod =
conf.getLong(DFS_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_KEY,
DFS_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88cce325/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
index 8e5b597..d396600 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
@@ -248,7 +248,7 @@ public class DNConf {
DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_KEY,
DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT);
- this.maxLockedMemory = getConf().getLong(
+ this.maxLockedMemory = getConf().getLongBytes(
DFS_DATANODE_MAX_LOCKED_MEMORY_KEY,
DFS_DATANODE_MAX_LOCKED_MEMORY_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88cce325/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
index 67a66fd..72ed47c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
@@ -76,7 +76,7 @@ public class AvailableSpaceVolumeChoosingPolicy<V extends FsVolumeSpi>
@Override
public void setConf(Configuration conf) {
- balancedSpaceThreshold = conf.getLong(
+ balancedSpaceThreshold = conf.getLongBytes(
DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_THRESHOLD_KEY,
DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_THRESHOLD_DEFAULT);
balancedPreferencePercent = conf.getFloat(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88cce325/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReservedSpaceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReservedSpaceCalculator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReservedSpaceCalculator.java
index 5523cfd..749e16e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReservedSpaceCalculator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReservedSpaceCalculator.java
@@ -97,7 +97,7 @@ public abstract class ReservedSpaceCalculator {
long getReservedFromConf(String key, long defaultValue) {
return conf.getLong(key + "." + StringUtils.toLowerCase(
- storageType.toString()), conf.getLong(key, defaultValue));
+ storageType.toString()), conf.getLongBytes(key, defaultValue));
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88cce325/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 2a976d2..0140912 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -291,7 +291,7 @@ public class FSDirectory implements Closeable {
DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY,
DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_DEFAULT);
LOG.info("XAttrs enabled? " + xattrsEnabled);
- this.xattrMaxSize = conf.getInt(
+ this.xattrMaxSize = (int) conf.getLongBytes(
DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY,
DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_DEFAULT);
Preconditions.checkArgument(xattrMaxSize > 0,
@@ -327,7 +327,7 @@ public class FSDirectory implements Closeable {
DFSConfigKeys.DFS_CONTENT_SUMMARY_SLEEP_MICROSEC_DEFAULT);
// filesystem limits
- this.maxComponentLength = conf.getInt(
+ this.maxComponentLength = (int) conf.getLongBytes(
DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_KEY,
DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_DEFAULT);
this.maxDirItems = conf.getInt(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88cce325/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index cc38036..eda1164 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -825,7 +825,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
this.maxFsObjects = conf.getLong(DFS_NAMENODE_MAX_OBJECTS_KEY,
DFS_NAMENODE_MAX_OBJECTS_DEFAULT);
- this.minBlockSize = conf.getLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY,
+ this.minBlockSize = conf.getLongBytes(
+ DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY,
DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_DEFAULT);
this.maxBlocksPerFile = conf.getLong(DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY,
DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88cce325/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java
index 9028b36..7250cca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java
@@ -220,9 +220,9 @@ public class ImageServlet extends HttpServlet {
* @return a data transfer throttler
*/
public static DataTransferThrottler getThrottler(Configuration conf) {
- long transferBandwidth =
- conf.getLong(DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_KEY,
- DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_DEFAULT);
+ long transferBandwidth = conf.getLongBytes(
+ DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_KEY,
+ DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_DEFAULT);
DataTransferThrottler throttler = null;
if (transferBandwidth > 0) {
throttler = new DataTransferThrottler(transferBandwidth);
@@ -233,7 +233,7 @@ public class ImageServlet extends HttpServlet {
private static DataTransferThrottler getThrottlerForBootstrapStandby(
Configuration conf) {
long transferBandwidth =
- conf.getLong(
+ conf.getLongBytes(
DFSConfigKeys.DFS_IMAGE_TRANSFER_BOOTSTRAP_STANDBY_RATE_KEY,
DFSConfigKeys.DFS_IMAGE_TRANSFER_BOOTSTRAP_STANDBY_RATE_DEFAULT);
DataTransferThrottler throttler = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88cce325/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java
index 898f57e..d0245d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java
@@ -111,7 +111,7 @@ public class NameNodeResourceChecker {
this.conf = conf;
volumes = new HashMap<String, CheckedVolume>();
- duReserved = conf.getLong(DFSConfigKeys.DFS_NAMENODE_DU_RESERVED_KEY,
+ duReserved = conf.getLongBytes(DFSConfigKeys.DFS_NAMENODE_DU_RESERVED_KEY,
DFSConfigKeys.DFS_NAMENODE_DU_RESERVED_DEFAULT);
Collection<URI> extraCheckedVolumes = Util.stringCollectionAsURIs(conf
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88cce325/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
index 14ce000..c1270c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
@@ -275,7 +275,7 @@ public class TransferFsImage {
connection.setDoOutput(true);
- int chunkSize = conf.getInt(
+ int chunkSize = (int) conf.getLongBytes(
DFSConfigKeys.DFS_IMAGE_TRANSFER_CHUNKSIZE_KEY,
DFSConfigKeys.DFS_IMAGE_TRANSFER_CHUNKSIZE_DEFAULT);
if (imageFile.length() > chunkSize) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88cce325/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index cb770f5..b894abb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -348,7 +348,8 @@
corresponding storage types ([ssd]/[disk]/[archive]/[ram_disk]) for cluster with heterogeneous storage.
For example, reserved space for RAM_DISK storage can be configured using property
'dfs.datanode.du.reserved.ram_disk'. If specific storage type reservation is not configured
- then dfs.datanode.du.reserved will be used.
+ then dfs.datanode.du.reserved will be used. Support multiple size unit suffix(case insensitive),
+ as described in dfs.blocksize.
Note: In case of using tune2fs to set reserved-blocks-percentage, or other filesystem tools,
then you can possibly run into out of disk errors because hadoop will not check those
external tool configurations.
@@ -390,7 +391,9 @@
<name>dfs.namenode.fs-limits.max-component-length</name>
<value>255</value>
<description>Defines the maximum number of bytes in UTF-8 encoding in each
- component of a path. A value of 0 will disable the check.</description>
+ component of a path. A value of 0 will disable the check. Support
+ multiple size unit suffix(case insensitive), as described in dfs.blocksize.
+ </description>
</property>
<property>
@@ -406,8 +409,9 @@
<value>1048576</value>
<description>Minimum block size in bytes, enforced by the Namenode at create
time. This prevents the accidental creation of files with tiny block
- sizes (and thus many blocks), which can degrade
- performance.</description>
+ sizes (and thus many blocks), which can degrade performance. Support multiple
+ size unit suffix(case insensitive), as described in dfs.blocksize.
+ </description>
</property>
<property>
@@ -945,7 +949,8 @@
<value>104857600</value>
<description>
The amount of space to reserve/require for a NameNode storage directory
- in bytes. The default is 100MB.
+ in bytes. The default is 100MB. Support multiple size unit
+ suffix(case insensitive), as described in dfs.blocksize.
</description>
</property>
@@ -1331,6 +1336,8 @@
A default value of 0 indicates that throttling is disabled.
The maximum bandwidth used for bootstrapping standby namenode is
configured with dfs.image.transfer-bootstrap-standby.bandwidthPerSec.
+ Support multiple size unit suffix(case insensitive), as described
+ in dfs.blocksize.
</description>
</property>
@@ -1344,6 +1351,8 @@
value should be used in most cases, to ensure timely HA operations.
The maximum bandwidth used for regular image transfers is configured
with dfs.image.transfer.bandwidthPerSec.
+ Support multiple size unit suffix(case insensitive), as described in
+ dfs.blocksize.
</description>
</property>
@@ -1354,6 +1363,8 @@
Chunksize in bytes to upload the checkpoint.
Chunked streaming is used to avoid internal buffering of contents
of image file of huge size.
+ Support multiple size unit suffix(case insensitive), as described
+ in dfs.blocksize.
</description>
</property>
@@ -2293,7 +2304,8 @@
bytes of free disk space before they are considered imbalanced. If the free
space of all the volumes are within this range of each other, the volumes
will be considered balanced and block assignments will be done on a pure
- round robin basis.
+ round robin basis. Support multiple size unit suffix(case insensitive), as
+ described in dfs.blocksize.
</description>
</property>
@@ -2366,7 +2378,8 @@
read ahead in the block file using posix_fadvise, potentially decreasing
I/O wait times. Unlike dfs.datanode.readahead.bytes, this is a client-side
setting rather than a setting for the entire datanode. If present, this
- setting will override the DataNode default.
+ setting will override the DataNode default. Support multiple size unit
+ suffix(case insensitive), as described in dfs.blocksize.
When using local reads, this setting determines how much readahead we do in
BlockReaderLocal.
@@ -2510,7 +2523,8 @@
The amount of memory in bytes to use for caching of block replicas in
memory on the datanode. The datanode's maximum locked memory soft ulimit
(RLIMIT_MEMLOCK) must be set to at least this value, else the datanode
- will abort on startup.
+ will abort on startup. Support multiple size unit suffix(case insensitive),
+ as described in dfs.blocksize.
By default, this parameter is set to 0, which disables in-memory caching.
@@ -2835,6 +2849,8 @@
The maximum combined size of the name and value of an extended attribute
in bytes. It should be larger than 0, and less than or equal to maximum
size hard limit which is 32768.
+ Support multiple size unit suffix(case insensitive), as described in
+ dfs.blocksize.
</description>
</property>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org