You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2021/04/02 09:21:18 UTC
[hadoop] 08/09: HDFS-15351. Blocks scheduled count was wrong on
truncate. Contributed by hemanthboyina.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 887948d12715a32228a8196db4b32a0a2394a0a1
Author: Inigo Goiri <in...@apache.org>
AuthorDate: Sat Jun 13 09:35:05 2020 -0700
HDFS-15351. Blocks scheduled count was wrong on truncate. Contributed by hemanthboyina.
(cherry picked from commit 719b53a79dc169a8c52229831dcb011935a8a151)
---
.../hdfs/server/blockmanagement/BlockManager.java | 9 +++-
.../hadoop/hdfs/TestBlocksScheduledCounter.java | 56 ++++++++++++++++++++++
2 files changed, 64 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 7461d46..6fd2343 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1280,7 +1280,14 @@ public class BlockManager implements BlockStatsMXBean {
neededReconstruction.remove(lastBlock, replicas.liveReplicas(),
replicas.readOnlyReplicas(),
replicas.outOfServiceReplicas(), getExpectedRedundancyNum(lastBlock));
- pendingReconstruction.remove(lastBlock);
+ PendingBlockInfo remove = pendingReconstruction.remove(lastBlock);
+ if (remove != null) {
+ List<DatanodeStorageInfo> locations = remove.getTargets();
+ DatanodeStorageInfo[] removedBlockTargets =
+ new DatanodeStorageInfo[locations.size()];
+ locations.toArray(removedBlockTargets);
+ DatanodeStorageInfo.decrementBlocksScheduled(removedBlockTargets);
+ }
// remove this block from the list of pending blocks to be deleted.
for (DatanodeStorageInfo storage : targets) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
index 95f4e2c..95d6825 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
@@ -202,4 +202,60 @@ public class TestBlocksScheduledCounter {
}
}
+ /**
+ * Test Block Scheduled counter on truncating a file.
+ * @throws Exception
+ */
+ @Test
+ public void testBlocksScheduledCounterOnTruncate() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
+ cluster.waitActive();
+ BlockManager bm = cluster.getNamesystem().getBlockManager();
+ try {
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ // 1. stop a datanode
+ cluster.stopDataNode(0);
+
+ // 2. create a file
+ Path filePath = new Path("/tmp");
+ DFSTestUtil.createFile(dfs, filePath, 1024, (short) 3, 0L);
+
+ DatanodeManager datanodeManager =
+ cluster.getNamesystem().getBlockManager().getDatanodeManager();
+ ArrayList<DatanodeDescriptor> dnList =
+ new ArrayList<DatanodeDescriptor>();
+ datanodeManager.fetchDatanodes(dnList, dnList, false);
+
+ // 3. restart the stopped datanode
+ cluster.restartDataNode(0);
+
+ // 4. disable the heartbeats
+ for (DataNode dn : cluster.getDataNodes()) {
+ DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
+ }
+
+ cluster.getNamesystem().writeLock();
+ try {
+ BlockManagerTestUtil.computeAllPendingWork(bm);
+ BlockManagerTestUtil.updateState(bm);
+ assertEquals(1L, bm.getPendingReconstructionBlocksCount());
+ } finally {
+ cluster.getNamesystem().writeUnlock();
+ }
+
+ // 5.truncate the file whose block exists in pending reconstruction
+ dfs.truncate(filePath, 10);
+ int blocksScheduled = 0;
+ for (DatanodeDescriptor descriptor : dnList) {
+ if (descriptor.getBlocksScheduled() != 0) {
+ blocksScheduled += descriptor.getBlocksScheduled();
+ }
+ }
+ assertEquals(0, blocksScheduled);
+ } finally {
+ cluster.shutdown();
+ }
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org