You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2026/10/24 12:49:12 UTC

(ozone) branch master updated: HDDS-16455. Skip stale recovering scrub when container is no longer RECOVERING (#11713)

This is an automated email from the ASF dual-hosted git repository.

adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git


The following commit(s) were added to refs/heads/master by this push:
     new ae71e2dd50c HDDS-16455. Skip stale recovering scrub when container is no longer RECOVERING (#11713)
ae71e2dd50c is described below

commit ae71e2dd50c27c79cca7de4f7298178e0a388630
Author: Chi-Hsuan Huang <ch...@gmail.com>
AuthorDate: Sat Oct 24 20:49:06 2026 +0800

    HDDS-16455. Skip stale recovering scrub when container is no longer RECOVERING (#11713)
    
    Co-authored-by: Claude Opus 5.5 <no...@anthropic.com>
---
 .../StaleRecoveringContainerScrubbingService.java  | 16 +++-
 ...stStaleRecoveringContainerScrubbingService.java | 98 ++++++++++++++++++++++
 2 files changed, 113 insertions(+), 1 deletion(-)

diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/StaleRecoveringContainerScrubbingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/StaleRecoveringContainerScrubbingService.java
index 6fd0bd59f8e..4d112c5648e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/StaleRecoveringContainerScrubbingService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/StaleRecoveringContainerScrubbingService.java
@@ -17,9 +17,12 @@
 
 package org.apache.hadoop.ozone.container.keyvalue.statemachine.background;
 
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.RECOVERING;
+
 import java.util.Iterator;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State;
 import org.apache.hadoop.hdds.utils.BackgroundService;
 import org.apache.hadoop.hdds.utils.BackgroundTask;
 import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
@@ -89,7 +92,18 @@ public BackgroundTaskResult call() throws Exception {
       }
       Container con = containerSet.getContainer(containerID);
       if (null != con) {
-        con.markContainerUnhealthy();
+        con.writeLock();
+        try {
+          // The container may have left RECOVERING after this task was scheduled.
+          State state = con.getContainerState();
+          if (state != RECOVERING) {
+            LOG.debug("Skipping stale recovering scrub for container {} - state is {}", containerID, state);
+            return new BackgroundTaskResult.EmptyTaskResult();
+          }
+          con.markContainerUnhealthy();
+        } finally {
+          con.writeUnlock();
+        }
         LOG.info("Stale recovering container {} marked UNHEALTHY", containerID);
       }
       return new BackgroundTaskResult.EmptyTaskResult();
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java
index e2881bd6f7e..ef2d3dd7e1e 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java
@@ -18,11 +18,13 @@
 package org.apache.hadoop.ozone.container.common;
 
 import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSED;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSING;
 import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.RECOVERING;
 import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.UNHEALTHY;
 import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.anyList;
 import static org.mockito.Mockito.anyLong;
@@ -42,6 +44,7 @@
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 import org.apache.hadoop.conf.StorageUnit;
 import org.apache.hadoop.hdds.HddsConfigKeys;
@@ -62,6 +65,7 @@
 import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
 import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
 import org.apache.hadoop.ozone.container.keyvalue.statemachine.background.StaleRecoveringContainerScrubbingService;
+import org.apache.ozone.test.GenericTestUtils;
 import org.apache.ozone.test.MockClock;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.io.TempDir;
@@ -237,4 +241,98 @@ public void testScrubSkippedWhenDeadlineExtendedBeforeTaskRuns(
     assertEquals(RECOVERING, containerSet.getContainer(containerId).getContainerState());
     assertTrue(containerSet.getRecoveringContainerMap().containsKey(containerId));
   }
+
+  @ContainerTestVersionInfo.ContainerTest
+  public void testScrubSkippedWhenContainerClosedBeforeTaskRuns(
+      ContainerTestVersionInfo versionInfo) throws Exception {
+    initVersionInfo(versionInfo);
+    ContainerSet containerSet = newContainerSet(1000, testClock);
+    StaleRecoveringContainerScrubbingService srcss =
+        new StaleRecoveringContainerScrubbingService(
+            50, TimeUnit.MILLISECONDS, 10,
+            Duration.ofSeconds(300).toMillis(),
+            containerSet);
+    List<Long> ids = createTestContainers(containerSet, 1, RECOVERING);
+    long containerId = ids.get(0);
+    Container<?> container = containerSet.getContainer(containerId);
+    testClock.fastForward(1000L);
+    BackgroundTaskQueue tasks = srcss.getTasks();
+    assertFalse(tasks.isEmpty());
+
+    container.markContainerForClose();
+    container.close();
+    while (!tasks.isEmpty()) {
+      BackgroundTask task = tasks.poll();
+      task.call();
+    }
+    assertEquals(CLOSED, container.getContainerState());
+  }
+
+  @ContainerTestVersionInfo.ContainerTest
+  public void testScrubSkippedForStaleEntryOfClosedContainer(
+      ContainerTestVersionInfo versionInfo) throws Exception {
+    initVersionInfo(versionInfo);
+    ContainerSet containerSet = newContainerSet(1000, testClock);
+    StaleRecoveringContainerScrubbingService srcss =
+        new StaleRecoveringContainerScrubbingService(
+            50, TimeUnit.MILLISECONDS, 10,
+            Duration.ofSeconds(300).toMillis(),
+            containerSet);
+    List<Long> ids = createTestContainers(containerSet, 1, RECOVERING);
+    long containerId = ids.get(0);
+    Container<?> container = containerSet.getContainer(containerId);
+    containerSet.removeRecoveringContainer(containerId);
+    container.markContainerForClose();
+    container.close();
+    // A write that checked RECOVERING before the close re-inserts the entry afterwards.
+    containerSet.updateRecoveringContainerTimeout(containerId);
+    assertEquals(CLOSED, container.getContainerState());
+    assertTrue(containerSet.getRecoveringContainerMap().containsKey(containerId));
+
+    testClock.fastForward(1000L);
+    srcss.runPeriodicalTaskNow();
+    assertEquals(CLOSED, container.getContainerState());
+    assertFalse(containerSet.getRecoveringContainerMap().containsKey(containerId));
+  }
+
+  @ContainerTestVersionInfo.ContainerTest
+  public void testScrubDoesNotOverrideConcurrentClose(
+      ContainerTestVersionInfo versionInfo) throws Exception {
+    initVersionInfo(versionInfo);
+    ContainerSet containerSet = newContainerSet(1000, testClock);
+    StaleRecoveringContainerScrubbingService srcss =
+        new StaleRecoveringContainerScrubbingService(
+            50, TimeUnit.MILLISECONDS, 10,
+            Duration.ofSeconds(300).toMillis(),
+            containerSet);
+    List<Long> ids = createTestContainers(containerSet, 1, RECOVERING);
+    long containerId = ids.get(0);
+    Container<?> container = containerSet.getContainer(containerId);
+    testClock.fastForward(1000L);
+    BackgroundTaskQueue tasks = srcss.getTasks();
+    assertFalse(tasks.isEmpty());
+    BackgroundTask task = tasks.poll();
+
+    // Hold the container lock like a close in progress, so the scrub task blocks on it.
+    AtomicReference<Exception> error = new AtomicReference<>();
+    Thread scrubber = new Thread(() -> {
+      try {
+        task.call();
+      } catch (Exception e) {
+        error.set(e);
+      }
+    });
+    container.writeLock();
+    try {
+      scrubber.start();
+      GenericTestUtils.waitFor(() -> scrubber.getState() == Thread.State.WAITING, 10, 10000);
+      container.markContainerForClose();
+    } finally {
+      container.writeUnlock();
+    }
+    scrubber.join(10000);
+    assertFalse(scrubber.isAlive());
+    assertNull(error.get());
+    assertEquals(CLOSING, container.getContainerState());
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org