You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by yq...@apache.org on 2018/08/27 06:56:13 UTC
hadoop git commit: HDFS-13831. Make block increment deletion number
configurable. Contributed by Ryan Wu.
Repository: hadoop
Updated Branches:
refs/heads/trunk a4121c71c -> b9b964d25
HDFS-13831. Make block increment deletion number configurable. Contributed by Ryan Wu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9b964d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9b964d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9b964d2
Branch: refs/heads/trunk
Commit: b9b964d25335943fb15cdfcf369d123bbd7e454a
Parents: a4121c7
Author: Yiqun Lin <yq...@apache.org>
Authored: Mon Aug 27 14:55:46 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Mon Aug 27 14:55:46 2018 +0800
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +++++
.../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 11 +++++++++--
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 10 ++++++++++
.../hdfs/server/namenode/TestLargeDirectoryDelete.java | 2 +-
4 files changed, 25 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9b964d2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 5ed35b8..bd88341 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -395,6 +395,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY = "dfs.namenode.startup.delay.block.deletion.sec";
public static final long DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT = 0L;
+ /** Block deletion increment. */
+ public static final String DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY =
+ "dfs.namenode.block.deletion.increment";
+ public static final int DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT = 1000;
+
public static final String DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES =
HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES;
public static final boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9b964d2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 06bf008..6ba0e0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -428,12 +428,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
FSNamesystem.class.getName() + ".audit");
private final int maxCorruptFileBlocksReturn;
- static int BLOCK_DELETION_INCREMENT = 1000;
private final boolean isPermissionEnabled;
private final UserGroupInformation fsOwner;
private final String supergroup;
private final boolean standbyShouldCheckpoint;
private final int snapshotDiffReportLimit;
+ private final int blockDeletionIncrement;
/** Interval between each check of lease to release. */
private final long leaseRecheckIntervalMs;
@@ -909,6 +909,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
DFSConfigKeys.DFS_NAMENODE_LIST_OPENFILES_NUM_RESPONSES +
" must be a positive integer."
);
+
+ this.blockDeletionIncrement = conf.getInt(
+ DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY,
+ DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT);
+ Preconditions.checkArgument(blockDeletionIncrement > 0,
+ DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY +
+ " must be a positive integer.");
} catch(IOException e) {
LOG.error(getClass().getSimpleName() + " initialization failed.", e);
close();
@@ -3094,7 +3101,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
while (iter.hasNext()) {
writeLock();
try {
- for (int i = 0; i < BLOCK_DELETION_INCREMENT && iter.hasNext(); i++) {
+ for (int i = 0; i < blockDeletionIncrement && iter.hasNext(); i++) {
blockManager.removeBlock(iter.next());
}
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9b964d2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index f720d0d..48ff329 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -5160,4 +5160,14 @@
will throw NameNodeFormatException.
</description>
</property>
+
+ <property>
+ <name>dfs.namenode.block.deletion.increment</name>
+ <value>1000</value>
+ <description>
+ The number of block deletion increment.
+ This setting will control the block increment deletion rate to
+ ensure that other waiters on the lock can get in.
+ </description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9b964d2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java
index 0882d18..3027c79 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java
@@ -49,6 +49,7 @@ public class TestLargeDirectoryDelete {
static {
CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1);
CONF.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1);
+ CONF.setInt(DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY, 1);
}
/** create a file with a length of <code>filelen</code> */
@@ -137,7 +138,6 @@ public class TestLargeDirectoryDelete {
threads[1].start();
final long start = Time.now();
- FSNamesystem.BLOCK_DELETION_INCREMENT = 1;
mc.getFileSystem().delete(new Path("/root"), true); // recursive delete
final long end = Time.now();
threads[0].endThread();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org