You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2026/06/27 15:52:16 UTC
(ozone) branch master updated: HDDS-14774. Intermittent timeout in TestContainerReportHandling (part 2) (#10620)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new bf59f31a7e5 HDDS-14774. Intermittent timeout in TestContainerReportHandling (part 2) (#10620)
bf59f31a7e5 is described below
commit bf59f31a7e5fcad1acc85e1228d77c22ee7f036c
Author: Chi-Hsuan Huang <ch...@gmail.com>
AuthorDate: Sat Jun 27 23:52:10 2026 +0800
HDDS-14774. Intermittent timeout in TestContainerReportHandling (part 2) (#10620)
---
.../container/TestContainerReportHandling.java | 26 ++++++++++++----------
.../TestContainerReportHandlingWithHA.java | 26 ++++++++++++----------
.../apache/hadoop/ozone/container/TestHelper.java | 20 +++++++++++++++++
3 files changed, 48 insertions(+), 24 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandling.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandling.java
index ad092591341..00f912d8ddd 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandling.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandling.java
@@ -38,8 +38,8 @@
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
@@ -79,8 +79,8 @@ private static Stream<Arguments> delStatesAndReplication() {
* Tests that a DELETING (or DELETED) container replica gets deleted when replica bcsid <= container bcsid
* applicable to RATIS; EC ignores bcsid.
* To do this, the test first creates a key and closes its corresponding container. Then it moves that container to
- * DELETING (or DELETED) state using ContainerManager. Then it restarts a Datanode hosting that container,
- * making it send a full container report.
+ * DELETING (or DELETED) state using ContainerManager. SCM then deletes the replicas when it processes a periodic
+ * container report for the CLOSED replicas.
* Tests wait for a DELETING (or DELETED) container replica gets deleted based on the bcsid comparison.
*/
@ParameterizedTest
@@ -111,8 +111,15 @@ void testDeletingOrDeletedContainerWhenNonEmptyReplicaIsReported(
// also wait till the container is closed in SCM
waitForContainerStateInSCM(cluster.getStorageContainerManager(), containerID, HddsProtos.LifeCycleState.CLOSED);
- // move the container to DELETING
ContainerManager containerManager = cluster.getStorageContainerManager().getContainerManager();
+ // Wait until SCM sees all replicas CLOSED before moving the container to DELETING. The container state above
+ // flips to CLOSED as soon as the first replica is reported CLOSED, so a lagging replica may still be CLOSING in
+ // SCM. Deleting then races with that lagging CLOSING report, which would resurrect the container out of
+ // DELETING/DELETED and the replicas would never be deleted.
+ TestHelper.waitForReplicaState(containerManager, containerID, replicationInput.getNumDatanodes(),
+ ContainerReplicaProto.State.CLOSED);
+
+ // move the container to DELETING
assertFalse(containerManager.getContainerReplicas(containerID).isEmpty());
containerManager.updateContainerState(containerID, HddsProtos.LifeCycleEvent.DELETE);
assertEquals(HddsProtos.LifeCycleState.DELETING, containerManager.getContainer(containerID).getState());
@@ -123,14 +130,9 @@ void testDeletingOrDeletedContainerWhenNonEmptyReplicaIsReported(
assertEquals(HddsProtos.LifeCycleState.DELETED, containerManager.getContainer(containerID).getState());
}
- // restart all the DNs
- List<DatanodeDetails> dnlist = keyLocation.getPipeline().getNodes();
- for (DatanodeDetails dn: dnlist) {
- cluster.restartHddsDatanode(dn, false);
- }
-
- // Since replica state is CLOSED and container is DELETED/DELETING in SCM
- // bcsid of replica and container is same, SCM will trigger delete replica for RATIS, while EC ignores bcsid
+ // Since replica state is CLOSED and container is DELETED/DELETING in SCM, and the bcsid of replica and
+ // container is same, SCM will trigger delete replica for RATIS (EC ignores bcsid) when it processes a
+ // periodic container report for the CLOSED replicas.
// wait for all replica to be deleted
GenericTestUtils.waitFor(() -> {
try {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandlingWithHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandlingWithHA.java
index ff4455321f5..d68880ddd36 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandlingWithHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandlingWithHA.java
@@ -39,8 +39,8 @@
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
@@ -82,8 +82,8 @@ private static Stream<Arguments> delStatesAndReplication() {
* Tests that a DELETING (or DELETED) container replica gets deleted when replica bcsid <= container bcsid
* applicable to RATIS; EC ignores bcsid.
* To do this, the test first creates a key and closes its corresponding container. Then it moves that container to
- * DELETING (or DELETED) state using ContainerManager. Then it restarts Datanodes hosting that container,
- * making it send a full container report.
+ * DELETING (or DELETED) state using ContainerManager. SCM then deletes the replicas when it processes a periodic
+ * container report for the CLOSED replicas.
* Tests wait for a DELETING (or DELETED) container replica gets deleted based on the bcsid comparison.
*/
@ParameterizedTest
@@ -114,8 +114,15 @@ void testDeletingOrDeletedContainerWhenNonEmptyReplicaIsReportedWithScmHA(
waitForContainerStateInAllSCMs(cluster, containerID, HddsProtos.LifeCycleState.CLOSED);
- // move the container to DELETING
ContainerManager containerManager = cluster.getScmLeader().getContainerManager();
+ // Wait until SCM sees all replicas CLOSED before moving the container to DELETING. The container state above
+ // flips to CLOSED as soon as the first replica is reported CLOSED, so a lagging replica may still be CLOSING in
+ // SCM. Deleting then races with that lagging CLOSING report, which would resurrect the container out of
+ // DELETING/DELETED and the replicas would never be deleted.
+ TestHelper.waitForReplicaState(containerManager, containerID, replicationInput.getNumDatanodes(),
+ ContainerReplicaProto.State.CLOSED);
+
+ // move the container to DELETING
assertFalse(containerManager.getContainerReplicas(containerID).isEmpty());
containerManager.updateContainerState(containerID, HddsProtos.LifeCycleEvent.DELETE);
assertEquals(HddsProtos.LifeCycleState.DELETING, containerManager.getContainer(containerID).getState());
@@ -126,14 +133,9 @@ void testDeletingOrDeletedContainerWhenNonEmptyReplicaIsReportedWithScmHA(
assertEquals(HddsProtos.LifeCycleState.DELETED, containerManager.getContainer(containerID).getState());
}
- // restart all the DNs
- List<DatanodeDetails> dnlist = keyLocation.getPipeline().getNodes();
- for (DatanodeDetails dn: dnlist) {
- cluster.restartHddsDatanode(dn, false);
- }
-
- // Since replica state is CLOSED and container is DELETED/DELETING in SCM
- // bcsid of replica and container is same, SCM will trigger delete replica for RATIS, while EC ignores bcsid
+ // Since replica state is CLOSED and container is DELETED/DELETING in SCM, and the bcsid of replica and
+ // container is same, SCM will trigger delete replica for RATIS (EC ignores bcsid) when it processes a
+ // periodic container report for the CLOSED replicas.
// wait for all replica to be deleted
GenericTestUtils.waitFor(() -> {
try {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java
index 246a8807600..bf6dd3306d3 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java
@@ -44,6 +44,7 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.ratis.RatisHelper;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
@@ -464,6 +465,25 @@ public static void waitForReplicaCount(long containerID, int count,
200, 30000);
}
+ /**
+ * Wait until SCM reports exactly {@code count} replicas for the container and every replica is in {@code state}.
+ * Unlike {@link #waitForContainerStateInSCM}, which checks the container's aggregate state (it flips as soon as the
+ * first replica reaches the state), this requires all replicas to have settled, so a lagging replica cannot trip
+ * later report handling.
+ */
+ public static void waitForReplicaState(ContainerManager containerManager, ContainerID containerID,
+ int count, ContainerReplicaProto.State state) throws TimeoutException, InterruptedException {
+ GenericTestUtils.waitFor(() -> {
+ try {
+ Set<ContainerReplica> replicas = containerManager.getContainerReplicas(containerID);
+ return replicas.size() == count
+ && replicas.stream().allMatch(replica -> replica.getState() == state);
+ } catch (ContainerNotFoundException e) {
+ return false;
+ }
+ }, 100, 60000);
+ }
+
/** Helper to set config even if {@code value} is null, which
* {@link OzoneConfiguration#set(String, String) does not allow. */
public static void setConfig(OzoneConfiguration conf, String key, String value) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org