You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by he...@apache.org on 2021/09/17 14:19:59 UTC
[hadoop] 06/06: HDFS-15113. Addendum: Missing IBR when NameNode
restart if open processCommand async feature. Contributed by Xiaoqiao He.
This is an automated email from the ASF dual-hosted git repository.
hexiaoqiao pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 4944dafa297e9aad93d39e861b5dc2835b7cbb5f
Author: He Xiaoqiao <he...@apache.org>
AuthorDate: Mon Mar 23 12:42:57 2020 -0700
HDFS-15113. Addendum: Missing IBR when NameNode restart if open processCommand async feature. Contributed by Xiaoqiao He.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
(cherry picked from commit af64ce2f4a72705c5b68ddfe4d29f0d208fd38e7)
---
.../hdfs/server/datanode/TestBPOfferService.java | 33 ++++++++++++++++++----
1 file changed, 27 insertions(+), 6 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
index 79ad5d1..a045653 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
@@ -286,23 +286,30 @@ public class TestBPOfferService {
public void testMissBlocksWhenReregister() throws Exception {
BPOfferService bpos = setupBPOSForNNs(mockNN1, mockNN2);
bpos.start();
+ int totalTestBlocks = 4000;
+ Thread addNewBlockThread = null;
+ final AtomicInteger count = new AtomicInteger(0);
+
try {
waitForBothActors(bpos);
waitForInitialization(bpos);
-
DataNodeFaultInjector.set(new DataNodeFaultInjector() {
public void blockUtilSendFullBlockReport() {
try {
- Thread.sleep(200);
- } catch (InterruptedException e) {
+ GenericTestUtils.waitFor(() -> {
+ if(count.get() > 2000) {
+ return true;
+ }
+ return false;
+ }, 100, 1000);
+ } catch (Exception e) {
e.printStackTrace();
}
}
});
countBlockReportItems(FAKE_BLOCK, mockNN1);
- int totalTestBlocks = 4000;
- Thread addNewBlockThread = new Thread(() -> {
+ addNewBlockThread = new Thread(() -> {
for (int i = 0; i < totalTestBlocks; i++) {
SimulatedFSDataset fsDataset = (SimulatedFSDataset) mockFSDataset;
SimulatedStorage simulatedStorage = fsDataset.getStorages().get(0);
@@ -312,6 +319,7 @@ public class TestBPOfferService {
fsDataset.createRbw(StorageType.DEFAULT, storageId, b, false);
bpos.notifyNamenodeReceivingBlock(b, storageId);
fsDataset.finalizeBlock(b, false);
+ count.addAndGet(1);
Thread.sleep(1);
} catch (Exception e) {
e.printStackTrace();
@@ -321,7 +329,13 @@ public class TestBPOfferService {
addNewBlockThread.start();
// Make sure that generate blocks for DataNode and IBR not empty now.
- Thread.sleep(200);
+ GenericTestUtils.waitFor(() -> {
+ if(count.get() > 0) {
+ return true;
+ }
+ return false;
+ }, 100, 1000);
+
// Trigger re-register using DataNode Command.
datanodeCommands[0] = new DatanodeCommand[]{RegisterCommand.REGISTER};
bpos.triggerHeartbeatForTests();
@@ -340,6 +354,7 @@ public class TestBPOfferService {
assertTrue(fullBlockReportCount == totalTestBlocks ||
incrBlockReportCount == totalTestBlocks);
} finally {
+ addNewBlockThread.join();
bpos.stop();
bpos.join();
@@ -698,12 +713,17 @@ public class TestBPOfferService {
}
}
+ /**
+ * Record blocks counts of block report and total adding blocks count of IBR
+ * which assume no deleting blocks here.
+ */
private void countBlockReportItems(final ExtendedBlock fakeBlock,
final DatanodeProtocolClientSideTranslatorPB mockNN) throws Exception {
final String fakeBlockPoolId = fakeBlock.getBlockPoolId();
final ArgumentCaptor<StorageBlockReport[]> captor =
ArgumentCaptor.forClass(StorageBlockReport[].class);
+ // Record blocks count about the last time block report.
Mockito.doAnswer((Answer<Object>) invocation -> {
Object[] arguments = invocation.getArguments();
StorageBlockReport[] list = (StorageBlockReport[])arguments[2];
@@ -716,6 +736,7 @@ public class TestBPOfferService {
Mockito.any()
);
+ // Record total adding blocks count and assume no deleting blocks here.
Mockito.doAnswer((Answer<Object>) invocation -> {
Object[] arguments = invocation.getArguments();
StorageReceivedDeletedBlocks[] list =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org