You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ay...@apache.org on 2023/02/11 20:38:16 UTC
[hadoop] branch branch-3.3 updated (9e7a9fd46d2 -> b6d732b96b0)
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a change to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
from 9e7a9fd46d2 HDFS-18324. Fix race condition in closing IPC connections. (#5371)
new c17734b7472 HDFS-15654. TestBPOfferService#testMissBlocksWhenReregister fails intermittently (#2419)
new d85c12ee69c HDFS-15674. TestBPOfferService#testMissBlocksWhenReregister fails on trunk. (#2467)
new b6d732b96b0 HDFS-15368. TestBalancerWithHANameNodes#testBalancerWithObserver failed occasionally. Contributed by Xiaoqiao He.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../balancer/TestBalancerWithHANameNodes.java | 1 +
.../hdfs/server/datanode/TestBPOfferService.java | 82 ++++++++++------------
2 files changed, 38 insertions(+), 45 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/03: HDFS-15674. TestBPOfferService#testMissBlocksWhenReregister fails on trunk. (#2467)
Posted by ay...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit d85c12ee69c19d1961ff6763fb6a653169c471ae
Author: Masatake Iwasaki <iw...@apache.org>
AuthorDate: Wed Nov 18 16:11:09 2020 +0900
HDFS-15674. TestBPOfferService#testMissBlocksWhenReregister fails on trunk. (#2467)
---
.../hdfs/server/datanode/TestBPOfferService.java | 39 ++++++++++------------
1 file changed, 18 insertions(+), 21 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
index 7542ef97b72..bf919e16fcf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
@@ -44,6 +44,8 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -55,6 +57,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockListAsLongs.BlockReportReplica;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
@@ -109,8 +112,6 @@ public class TestBPOfferService {
private long firstLeaseId = 0;
private long secondLeaseId = 0;
private long nextFullBlockReportLeaseId = 1L;
- private int fullBlockReportCount = 0;
- private int incrBlockReportCount = 0;
static {
GenericTestUtils.setLogLevel(DataNode.LOG, Level.ALL);
@@ -233,14 +234,6 @@ public class TestBPOfferService {
}
}
- private void setBlockReportCount(int count) {
- fullBlockReportCount = count;
- }
-
- private void setIncreaseBlockReportCount(int count) {
- incrBlockReportCount += count;
- }
-
/**
* Test that the BPOS can register to talk to two different NNs,
* sends block reports to both, etc.
@@ -288,6 +281,7 @@ public class TestBPOfferService {
Thread addNewBlockThread = null;
final AtomicInteger count = new AtomicInteger(0);
DataNodeFaultInjector prevDNFaultInjector = null;
+ Set<Long> blocks = new TreeSet<>();
try {
waitForBothActors(bpos);
waitForInitialization(bpos);
@@ -303,7 +297,7 @@ public class TestBPOfferService {
}
});
- countBlockReportItems(FAKE_BLOCK, mockNN1);
+ countBlockReportItems(FAKE_BLOCK, mockNN1, blocks);
addNewBlockThread = new Thread(() -> {
for (int i = 0; i < totalTestBlocks; i++) {
SimulatedFSDataset fsDataset = (SimulatedFSDataset) mockFSDataset;
@@ -334,14 +328,12 @@ public class TestBPOfferService {
addNewBlockThread = null;
// Verify FBR/IBR count is equal to generate number.
try {
- GenericTestUtils.waitFor(() ->
- (fullBlockReportCount == totalTestBlocks ||
- incrBlockReportCount == totalTestBlocks), 1000, 15000);
+ GenericTestUtils.waitFor(() -> blocks.size() == totalTestBlocks,
+ 1000, 15000);
} catch (Exception e) {
- fail(String.format("Timed out wait for IBR counts FBRCount = %d,"
- + " IBRCount = %d; expected = %d. Exception: %s",
- fullBlockReportCount, incrBlockReportCount, totalTestBlocks,
- e.getMessage()));
+ fail(String.format("Timed out waiting for blocks count. "
+ + "reported = %d, expected = %d. Exception: %s",
+ blocks.size(), totalTestBlocks, e.getMessage()));
}
} finally {
@@ -711,7 +703,8 @@ public class TestBPOfferService {
* which assume no deleting blocks here.
*/
private void countBlockReportItems(final ExtendedBlock fakeBlock,
- final DatanodeProtocolClientSideTranslatorPB mockNN) throws Exception {
+ final DatanodeProtocolClientSideTranslatorPB mockNN,
+ final Set<Long> blocks) throws Exception {
final String fakeBlockPoolId = fakeBlock.getBlockPoolId();
final ArgumentCaptor<StorageBlockReport[]> captor =
ArgumentCaptor.forClass(StorageBlockReport[].class);
@@ -720,7 +713,9 @@ public class TestBPOfferService {
Mockito.doAnswer((Answer<Object>) invocation -> {
Object[] arguments = invocation.getArguments();
StorageBlockReport[] list = (StorageBlockReport[])arguments[2];
- setBlockReportCount(list[0].getBlocks().getNumberOfBlocks());
+ for (BlockReportReplica brr : list[0].getBlocks()) {
+ blocks.add(brr.getBlockId());
+ }
return null;
}).when(mockNN).blockReport(
Mockito.any(),
@@ -734,7 +729,9 @@ public class TestBPOfferService {
Object[] arguments = invocation.getArguments();
StorageReceivedDeletedBlocks[] list =
(StorageReceivedDeletedBlocks[])arguments[2];
- setIncreaseBlockReportCount(list[0].getBlocks().length);
+ for (ReceivedDeletedBlockInfo rdbi : list[0].getBlocks()) {
+ blocks.add(rdbi.getBlock().getBlockId());
+ }
return null;
}).when(mockNN).blockReceivedAndDeleted(
Mockito.any(),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/03: HDFS-15654. TestBPOfferService#testMissBlocksWhenReregister fails intermittently (#2419)
Posted by ay...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit c17734b7472f7c92f933f2c6a7ab7e7ef5f17dea
Author: Ahmed Hussein <50...@users.noreply.github.com>
AuthorDate: Wed Oct 28 18:24:34 2020 -0500
HDFS-15654. TestBPOfferService#testMissBlocksWhenReregister fails intermittently (#2419)
---
.../hdfs/server/datanode/TestBPOfferService.java | 57 ++++++++++------------
1 file changed, 26 insertions(+), 31 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
index a3440115057..7542ef97b72 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
@@ -34,6 +34,7 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
@@ -47,6 +48,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -285,21 +287,18 @@ public class TestBPOfferService {
int totalTestBlocks = 4000;
Thread addNewBlockThread = null;
final AtomicInteger count = new AtomicInteger(0);
-
+ DataNodeFaultInjector prevDNFaultInjector = null;
try {
waitForBothActors(bpos);
waitForInitialization(bpos);
+ prevDNFaultInjector = DataNodeFaultInjector.get();
DataNodeFaultInjector.set(new DataNodeFaultInjector() {
public void blockUtilSendFullBlockReport() {
try {
- GenericTestUtils.waitFor(() -> {
- if(count.get() > 2000) {
- return true;
- }
- return false;
- }, 100, 1000);
+ GenericTestUtils.waitFor(() -> count.get() > 2000,
+ 100, 1000);
} catch (Exception e) {
- e.printStackTrace();
+ LOG.error("error DataNodeFaultInjector", e);
}
}
});
@@ -318,45 +317,41 @@ public class TestBPOfferService {
count.addAndGet(1);
Thread.sleep(1);
} catch (Exception e) {
- e.printStackTrace();
+ LOG.error("error addNewBlockThread", e);
}
}
});
addNewBlockThread.start();
// Make sure that generate blocks for DataNode and IBR not empty now.
- GenericTestUtils.waitFor(() -> {
- if(count.get() > 0) {
- return true;
- }
- return false;
- }, 100, 1000);
+ GenericTestUtils.waitFor(() -> count.get() > 0, 100, 1000);
// Trigger re-register using DataNode Command.
datanodeCommands[0] = new DatanodeCommand[]{RegisterCommand.REGISTER};
- bpos.triggerHeartbeatForTests();
+ bpos.triggerHeartbeatForTests();
+ addNewBlockThread.join();
+ addNewBlockThread = null;
+ // Verify FBR/IBR count is equal to generate number.
try {
- GenericTestUtils.waitFor(() -> {
- if(fullBlockReportCount == totalTestBlocks ||
- incrBlockReportCount == totalTestBlocks) {
- return true;
- }
- return false;
- }, 1000, 15000);
- } catch (Exception e) {}
+ GenericTestUtils.waitFor(() ->
+ (fullBlockReportCount == totalTestBlocks ||
+ incrBlockReportCount == totalTestBlocks), 1000, 15000);
+ } catch (Exception e) {
+ fail(String.format("Timed out wait for IBR counts FBRCount = %d,"
+ + " IBRCount = %d; expected = %d. Exception: %s",
+ fullBlockReportCount, incrBlockReportCount, totalTestBlocks,
+ e.getMessage()));
+ }
- // Verify FBR/IBR count is equal to generate number.
- assertTrue(fullBlockReportCount == totalTestBlocks ||
- incrBlockReportCount == totalTestBlocks);
} finally {
- addNewBlockThread.join();
+ if (addNewBlockThread != null) {
+ addNewBlockThread.interrupt();
+ }
bpos.stop();
bpos.join();
- DataNodeFaultInjector.set(new DataNodeFaultInjector() {
- public void blockUtilSendFullBlockReport() {}
- });
+ DataNodeFaultInjector.set(prevDNFaultInjector);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 03/03: HDFS-15368. TestBalancerWithHANameNodes#testBalancerWithObserver failed occasionally. Contributed by Xiaoqiao He.
Posted by ay...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit b6d732b96b01a11c416223ddaf10910938116eaf
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Thu May 28 20:47:46 2020 +0530
HDFS-15368. TestBalancerWithHANameNodes#testBalancerWithObserver failed occasionally. Contributed by Xiaoqiao He.
---
.../apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
index cdfa76a9d1f..65623be781b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
@@ -236,6 +236,7 @@ public class TestBalancerWithHANameNodes {
TestBalancer.initConf(conf);
// Avoid the same FS being reused between tests
conf.setBoolean("fs.hdfs.impl.disable.cache", true);
+ conf.setBoolean(HdfsClientConfigKeys.Failover.RANDOM_ORDER, false);
MiniQJMHACluster qjmhaCluster = null;
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org