You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2026/10/24 12:49:12 UTC
(ozone) branch master updated: HDDS-16455. Skip stale recovering scrub when container is no longer RECOVERING (#11713)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ae71e2dd50c HDDS-16455. Skip stale recovering scrub when container is no longer RECOVERING (#11713)
ae71e2dd50c is described below
commit ae71e2dd50c27c79cca7de4f7298178e0a388630
Author: Chi-Hsuan Huang <ch...@gmail.com>
AuthorDate: Sat Oct 24 20:49:06 2026 +0800
HDDS-16455. Skip stale recovering scrub when container is no longer RECOVERING (#11713)
Co-authored-by: Claude Opus 5.5 <no...@anthropic.com>
---
.../StaleRecoveringContainerScrubbingService.java | 16 +++-
...stStaleRecoveringContainerScrubbingService.java | 98 ++++++++++++++++++++++
2 files changed, 113 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/StaleRecoveringContainerScrubbingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/StaleRecoveringContainerScrubbingService.java
index 6fd0bd59f8e..4d112c5648e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/StaleRecoveringContainerScrubbingService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/StaleRecoveringContainerScrubbingService.java
@@ -17,9 +17,12 @@
package org.apache.hadoop.ozone.container.keyvalue.statemachine.background;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.RECOVERING;
+
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State;
import org.apache.hadoop.hdds.utils.BackgroundService;
import org.apache.hadoop.hdds.utils.BackgroundTask;
import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
@@ -89,7 +92,18 @@ public BackgroundTaskResult call() throws Exception {
}
Container con = containerSet.getContainer(containerID);
if (null != con) {
- con.markContainerUnhealthy();
+ con.writeLock();
+ try {
+ // The container may have left RECOVERING after this task was scheduled.
+ State state = con.getContainerState();
+ if (state != RECOVERING) {
+ LOG.debug("Skipping stale recovering scrub for container {} - state is {}", containerID, state);
+ return new BackgroundTaskResult.EmptyTaskResult();
+ }
+ con.markContainerUnhealthy();
+ } finally {
+ con.writeUnlock();
+ }
LOG.info("Stale recovering container {} marked UNHEALTHY", containerID);
}
return new BackgroundTaskResult.EmptyTaskResult();
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java
index e2881bd6f7e..ef2d3dd7e1e 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java
@@ -18,11 +18,13 @@
package org.apache.hadoop.ozone.container.common;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSED;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSING;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.RECOVERING;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.UNHEALTHY;
import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.anyList;
import static org.mockito.Mockito.anyLong;
@@ -42,6 +44,7 @@
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.HddsConfigKeys;
@@ -62,6 +65,7 @@
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.keyvalue.statemachine.background.StaleRecoveringContainerScrubbingService;
+import org.apache.ozone.test.GenericTestUtils;
import org.apache.ozone.test.MockClock;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.io.TempDir;
@@ -237,4 +241,98 @@ public void testScrubSkippedWhenDeadlineExtendedBeforeTaskRuns(
assertEquals(RECOVERING, containerSet.getContainer(containerId).getContainerState());
assertTrue(containerSet.getRecoveringContainerMap().containsKey(containerId));
}
+
+ @ContainerTestVersionInfo.ContainerTest
+ public void testScrubSkippedWhenContainerClosedBeforeTaskRuns(
+ ContainerTestVersionInfo versionInfo) throws Exception {
+ initVersionInfo(versionInfo);
+ ContainerSet containerSet = newContainerSet(1000, testClock);
+ StaleRecoveringContainerScrubbingService srcss =
+ new StaleRecoveringContainerScrubbingService(
+ 50, TimeUnit.MILLISECONDS, 10,
+ Duration.ofSeconds(300).toMillis(),
+ containerSet);
+ List<Long> ids = createTestContainers(containerSet, 1, RECOVERING);
+ long containerId = ids.get(0);
+ Container<?> container = containerSet.getContainer(containerId);
+ testClock.fastForward(1000L);
+ BackgroundTaskQueue tasks = srcss.getTasks();
+ assertFalse(tasks.isEmpty());
+
+ container.markContainerForClose();
+ container.close();
+ while (!tasks.isEmpty()) {
+ BackgroundTask task = tasks.poll();
+ task.call();
+ }
+ assertEquals(CLOSED, container.getContainerState());
+ }
+
+ @ContainerTestVersionInfo.ContainerTest
+ public void testScrubSkippedForStaleEntryOfClosedContainer(
+ ContainerTestVersionInfo versionInfo) throws Exception {
+ initVersionInfo(versionInfo);
+ ContainerSet containerSet = newContainerSet(1000, testClock);
+ StaleRecoveringContainerScrubbingService srcss =
+ new StaleRecoveringContainerScrubbingService(
+ 50, TimeUnit.MILLISECONDS, 10,
+ Duration.ofSeconds(300).toMillis(),
+ containerSet);
+ List<Long> ids = createTestContainers(containerSet, 1, RECOVERING);
+ long containerId = ids.get(0);
+ Container<?> container = containerSet.getContainer(containerId);
+ containerSet.removeRecoveringContainer(containerId);
+ container.markContainerForClose();
+ container.close();
+ // A write that checked RECOVERING before the close re-inserts the entry afterwards.
+ containerSet.updateRecoveringContainerTimeout(containerId);
+ assertEquals(CLOSED, container.getContainerState());
+ assertTrue(containerSet.getRecoveringContainerMap().containsKey(containerId));
+
+ testClock.fastForward(1000L);
+ srcss.runPeriodicalTaskNow();
+ assertEquals(CLOSED, container.getContainerState());
+ assertFalse(containerSet.getRecoveringContainerMap().containsKey(containerId));
+ }
+
+ @ContainerTestVersionInfo.ContainerTest
+ public void testScrubDoesNotOverrideConcurrentClose(
+ ContainerTestVersionInfo versionInfo) throws Exception {
+ initVersionInfo(versionInfo);
+ ContainerSet containerSet = newContainerSet(1000, testClock);
+ StaleRecoveringContainerScrubbingService srcss =
+ new StaleRecoveringContainerScrubbingService(
+ 50, TimeUnit.MILLISECONDS, 10,
+ Duration.ofSeconds(300).toMillis(),
+ containerSet);
+ List<Long> ids = createTestContainers(containerSet, 1, RECOVERING);
+ long containerId = ids.get(0);
+ Container<?> container = containerSet.getContainer(containerId);
+ testClock.fastForward(1000L);
+ BackgroundTaskQueue tasks = srcss.getTasks();
+ assertFalse(tasks.isEmpty());
+ BackgroundTask task = tasks.poll();
+
+ // Hold the container lock like a close in progress, so the scrub task blocks on it.
+ AtomicReference<Exception> error = new AtomicReference<>();
+ Thread scrubber = new Thread(() -> {
+ try {
+ task.call();
+ } catch (Exception e) {
+ error.set(e);
+ }
+ });
+ container.writeLock();
+ try {
+ scrubber.start();
+ GenericTestUtils.waitFor(() -> scrubber.getState() == Thread.State.WAITING, 10, 10000);
+ container.markContainerForClose();
+ } finally {
+ container.writeUnlock();
+ }
+ scrubber.join(10000);
+ assertFalse(scrubber.isAlive());
+ assertNull(error.get());
+ assertEquals(CLOSING, container.getContainerState());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org