[ratis] branch master updated: RATIS-1375. Handle bad storage dir due to disk failures. (#477)

[sz...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ratis.git


The following commit(s) were added to refs/heads/master by this push:
     new 363dd07  RATIS-1375. Handle bad storage dir due to disk failures. (#477)
363dd07 is described below

commit 363dd07f37742285fdfffcb7168bcbb2b21896a9
Author: Gui Hecheng <ma...@tencent.com>
AuthorDate: Wed May 12 11:29:56 2021 +0800

    RATIS-1375. Handle bad storage dir due to disk failures. (#477)
---
 .../org/apache/ratis/server/impl/ServerState.java     | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java
index 322feb5..b5557c5 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java
@@ -40,7 +40,7 @@ import org.apache.ratis.util.Timestamp;
 import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
-import java.nio.file.AccessDeniedException;
+import java.nio.channels.OverlappingFileLockException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -104,17 +104,30 @@ class ServerState implements Closeable {
     configurationManager = new ConfigurationManager(initialConf);
     LOG.info("{}: {}", getMemberId(), configurationManager);
 
+    boolean storageFound = false;
     List<File> directories = RaftServerConfigKeys.storageDir(prop);
     while (!directories.isEmpty()) {
       // use full uuid string to create a subdirectory
       File dir = chooseStorageDir(directories, group.getGroupId().getUuid().toString());
       try {
         storage = new RaftStorageImpl(dir, RaftServerConfigKeys.Log.corruptionPolicy(prop));
+        storageFound = true;
         break;
-      } catch (AccessDeniedException e) {
-        directories.remove(dir);
+      } catch (IOException e) {
+        if (e.getCause() instanceof OverlappingFileLockException) {
+          throw e;
+        }
+        LOG.warn("Failed to init RaftStorage under {} for {}: {}",
+            dir.getParent(), group.getGroupId().getUuid().toString(), e);
+        directories.removeIf(d -> d.getAbsolutePath().equals(dir.getParent()));
       }
     }
+
+    if (!storageFound) {
+      throw new IOException("No healthy directories found for RaftStorage among: " +
+          RaftServerConfigKeys.storageDir(prop));
+    }
+
     snapshotManager = new SnapshotManager(storage, id);
 
     stateMachine.initialize(server.getRaftServer(), group.getGroupId(), storage);