You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by re...@apache.org on 2023/05/10 06:28:57 UTC

[incubator-celeborn] branch main updated: [CELEBORN-573] Guarantee resource/app/worker change persistent to raft in Ha Mode. (#1477)

This is an automated email from the ASF dual-hosted git repository.

rexxiong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/main by this push:
     new fb753fd48 [CELEBORN-573] Guarantee resource/app/worker change persistent to raft in Ha Mode. (#1477)
fb753fd48 is described below

commit fb753fd48eb3e5c470e714e61e73fb4fc0bb5cba
Author: Shuang <lv...@gmail.com>
AuthorDate: Wed May 10 14:28:52 2023 +0800

    [CELEBORN-573] Guarantee resource/app/worker change persistent to raft in Ha Mode. (#1477)
---
 .../apache/celeborn/client/LifecycleManager.scala  |  7 +++-
 .../exception/CelebornRuntimeException.scala       | 24 +++++++++++
 .../deploy/master/clustermeta/ha/HAHelper.java     | 16 ++++++--
 .../master/clustermeta/ha/HAMasterMetaManager.java | 35 ++++++++++------
 .../deploy/master/clustermeta/ha/HARaftServer.java | 19 ++++-----
 .../celeborn/service/deploy/master/Master.scala    | 10 ++++-
 .../ha/RatisMasterStatusSystemSuiteJ.java          | 48 ++++++++++++++++++++++
 .../celeborn/service/deploy/worker/Worker.scala    | 12 +++++-
 8 files changed, 141 insertions(+), 30 deletions(-)

diff --git a/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala b/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala
index 3c14c2070..ee4524389 100644
--- a/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala
+++ b/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala
@@ -999,13 +999,16 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
         // clear for the shuffle
         registeredShuffle.remove(shuffleId)
         registeringShuffleRequest.remove(shuffleId)
-        unregisterShuffleTime.remove(shuffleId)
         shuffleAllocatedWorkers.remove(shuffleId)
         latestPartitionLocation.remove(shuffleId)
         commitManager.removeExpiredShuffle(shuffleId)
         changePartitionManager.removeExpiredShuffle(shuffleId)
-        requestMasterUnregisterShuffle(
+        val unregisterShuffleResponse = requestMasterUnregisterShuffle(
           UnregisterShuffle(appId, shuffleId, RssHARetryClient.genRequestId()))
+        // if unregister shuffle not success, wait next turn
+        if (StatusCode.SUCCESS == Utils.toStatusCode(unregisterShuffleResponse.getStatus)) {
+          unregisterShuffleTime.remove(shuffleId)
+        }
       }
     }
   }
diff --git a/common/src/main/scala/org/apache/celeborn/common/exception/CelebornRuntimeException.scala b/common/src/main/scala/org/apache/celeborn/common/exception/CelebornRuntimeException.scala
new file mode 100644
index 000000000..322bcbb78
--- /dev/null
+++ b/common/src/main/scala/org/apache/celeborn/common/exception/CelebornRuntimeException.scala
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.common.exception
+
+class CelebornRuntimeException(message: String, cause: Throwable)
+  extends RuntimeException(message, cause) {
+
+  def this(message: String) = this(message, null)
+}
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAHelper.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAHelper.java
index a17ce7c6e..3dc684c9b 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAHelper.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAHelper.java
@@ -25,6 +25,7 @@ import org.apache.ratis.protocol.Message;
 import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
 import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
 
+import org.apache.celeborn.common.exception.CelebornIOException;
 import org.apache.celeborn.common.haclient.MasterNotLeaderException;
 import org.apache.celeborn.common.rpc.RpcCallContext;
 import org.apache.celeborn.service.deploy.master.clustermeta.AbstractMetaManager;
@@ -39,7 +40,16 @@ public class HAHelper {
       if (ratisServer.isLeader()) {
         return true;
       }
-      if (context != null) {
+      sendFailure(context, ratisServer, null);
+      return false;
+    }
+    return true;
+  }
+
+  public static void sendFailure(
+      RpcCallContext context, HARaftServer ratisServer, Throwable cause) {
+    if (context != null) {
+      if (ratisServer != null) {
         if (ratisServer.getCachedLeaderPeerRpcEndpoint().isPresent()) {
           context.sendFailure(
               new MasterNotLeaderException(
@@ -50,10 +60,10 @@ public class HAHelper {
               new MasterNotLeaderException(
                   ratisServer.getRpcEndpoint(), MasterNotLeaderException.LEADER_NOT_PRESENTED));
         }
+      } else {
+        context.sendFailure(new CelebornIOException(cause.getMessage(), cause));
       }
-      return false;
     }
-    return true;
   }
 
   public static long getWorkerTimeoutDeadline(AbstractMetaManager masterStatusSystem) {
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java
index 9db4102f4..e1c36ec54 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java
@@ -21,11 +21,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
-import com.google.protobuf.ServiceException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.celeborn.common.CelebornConf;
+import org.apache.celeborn.common.exception.CelebornRuntimeException;
 import org.apache.celeborn.common.haclient.RssHARetryClient;
 import org.apache.celeborn.common.identity.UserIdentifier;
 import org.apache.celeborn.common.meta.AppDiskUsageMetric;
@@ -84,8 +84,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
               .setRequestId(requestId)
               .setRequestSlotsRequest(builder.build())
               .build());
-    } catch (ServiceException e) {
+    } catch (CelebornRuntimeException e) {
       LOG.error("Handle request slots for {} failed!", shuffleKey, e);
+      throw e;
     }
   }
 
@@ -112,8 +113,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
                               .collect(Collectors.toList()))
                       .build())
               .build());
-    } catch (ServiceException e) {
+    } catch (CelebornRuntimeException e) {
       LOG.error("Handle release slots for {} failed!", shuffleKey, e);
+      throw e;
     }
   }
 
@@ -129,8 +131,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
                       .setShuffleKey(shuffleKey)
                       .build())
               .build());
-    } catch (ServiceException e) {
+    } catch (CelebornRuntimeException e) {
       LOG.error("Handle unregister shuffle for {} failed!", shuffleKey, e);
+      throw e;
     }
   }
 
@@ -150,8 +153,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
                       .setFileCount(fileCount)
                       .build())
               .build());
-    } catch (ServiceException e) {
+    } catch (CelebornRuntimeException e) {
       LOG.error("Handle heart beat for {} failed!", appId, e);
+      throw e;
     }
   }
 
@@ -164,8 +168,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
               .setRequestId(requestId)
               .setAppLostRequest(ResourceProtos.AppLostRequest.newBuilder().setAppId(appId).build())
               .build());
-    } catch (ServiceException e) {
+    } catch (CelebornRuntimeException e) {
       LOG.error("Handle app lost for {} failed!", appId, e);
+      throw e;
     }
   }
 
@@ -186,8 +191,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
                       .setReplicatePort(replicatePort)
                       .build())
               .build());
-    } catch (ServiceException e) {
+    } catch (CelebornRuntimeException e) {
       LOG.error("Handle worker lost for {} failed!", host, e);
+      throw e;
     }
   }
 
@@ -208,8 +214,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
                       .setReplicatePort(replicatePort)
                       .build())
               .build());
-    } catch (ServiceException e) {
+    } catch (CelebornRuntimeException e) {
       LOG.error("Handle worker lost for {} failed!", host, e);
+      throw e;
     }
   }
 
@@ -244,8 +251,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
                       .setTime(time)
                       .build())
               .build());
-    } catch (ServiceException e) {
+    } catch (CelebornRuntimeException e) {
       LOG.error("Handle worker heartbeat for {} failed!", host, e);
+      throw e;
     }
   }
 
@@ -276,8 +284,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
                           MetaUtil.toPbUserResourceConsumption(userResourceConsumption))
                       .build())
               .build());
-    } catch (ServiceException e) {
+    } catch (CelebornRuntimeException e) {
       LOG.error("Handle worker register for {} failed!", host, e);
+      throw e;
     }
   }
 
@@ -295,8 +304,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
                       .addAllUnavailable(addrs)
                       .build())
               .build());
-    } catch (ServiceException e) {
+    } catch (CelebornRuntimeException e) {
       LOG.error("Handle report node failure for {} failed!", failedNodes, e);
+      throw e;
     }
   }
 
@@ -308,8 +318,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
               .setCmdType(Type.UpdatePartitionSize)
               .setRequestId(RssHARetryClient.genRequestId())
               .build());
-    } catch (ServiceException e) {
+    } catch (CelebornRuntimeException e) {
       LOG.error("Handle update partition size failed!", e);
+      throw e;
     }
   }
 }
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java
index 5a49a1bb1..af9d549af 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java
@@ -31,7 +31,6 @@ import scala.Tuple2;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.protobuf.InvalidProtocolBufferException;
-import com.google.protobuf.ServiceException;
 import org.apache.ratis.RaftConfigKeys;
 import org.apache.ratis.conf.RaftProperties;
 import org.apache.ratis.grpc.GrpcConfigKeys;
@@ -55,6 +54,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.celeborn.common.CelebornConf;
+import org.apache.celeborn.common.exception.CelebornRuntimeException;
 import org.apache.celeborn.common.haclient.RssHARetryClient;
 import org.apache.celeborn.common.util.ThreadUtils;
 import org.apache.celeborn.service.deploy.master.clustermeta.ResourceProtos;
@@ -199,11 +199,12 @@ public class HARaftServer {
   }
 
   public ResourceResponse submitRequest(ResourceProtos.ResourceRequest request)
-      throws ServiceException {
+      throws CelebornRuntimeException {
     String requestId = request.getRequestId();
     Tuple2<String, Long> decoded = RssHARetryClient.decodeRequestId(requestId);
     if (decoded == null) {
-      throw new ServiceException("RequestId:" + requestId + " invalid, should be: uuid#callId.");
+      throw new CelebornRuntimeException(
+          "RequestId:" + requestId + " invalid, should be: uuid#callId.");
     }
     ClientId clientId = ClientId.valueOf(UUID.fromString(decoded._1));
     long callId = decoded._2;
@@ -221,19 +222,19 @@ public class HARaftServer {
     try {
       raftClientReply = server.submitClientRequestAsync(raftClientRequest).get();
     } catch (Exception ex) {
-      throw new ServiceException(ex.getMessage(), ex);
+      throw new CelebornRuntimeException(ex.getMessage(), ex);
     }
 
     if (!raftClientReply.isSuccess()) {
       NotLeaderException notLeaderException = raftClientReply.getNotLeaderException();
       if (notLeaderException != null) {
-        throw new ServiceException("Not leader!");
+        throw new CelebornRuntimeException("Not leader!");
       }
 
       LeaderNotReadyException leaderNotReadyException =
           raftClientReply.getLeaderNotReadyException();
       if (leaderNotReadyException != null) {
-        throw new ServiceException("Not leader!");
+        throw new CelebornRuntimeException("Not leader!");
       }
 
       StateMachineException stateMachineException = raftClientReply.getStateMachineException();
@@ -261,11 +262,7 @@ public class HARaftServer {
       byte[] bytes = raftClientReply.getMessage().getContent().toByteArray();
       return ResourceResponse.newBuilder(ResourceResponse.parseFrom(bytes)).build();
     } catch (InvalidProtocolBufferException ex) {
-      if (ex.getMessage() != null) {
-        throw new ServiceException(ex.getMessage(), ex);
-      } else {
-        throw new ServiceException(ex);
-      }
+      throw new CelebornRuntimeException(ex.getMessage(), ex);
     }
   }
 
diff --git a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
index 0321a0562..5864e6574 100644
--- a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
+++ b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
@@ -26,6 +26,7 @@ import scala.collection.JavaConverters._
 import scala.util.Random
 
 import org.apache.celeborn.common.CelebornConf
+import org.apache.celeborn.common.exception.CelebornRuntimeException
 import org.apache.celeborn.common.haclient.RssHARetryClient
 import org.apache.celeborn.common.identity.UserIdentifier
 import org.apache.celeborn.common.internal.Logging
@@ -200,7 +201,14 @@ private[celeborn] class Master(
   }
 
   def executeWithLeaderChecker[T](context: RpcCallContext, f: => T): Unit =
-    if (HAHelper.checkShouldProcess(context, statusSystem)) f
+    if (HAHelper.checkShouldProcess(context, statusSystem)) {
+      try {
+        f
+      } catch {
+        case e: Exception =>
+          HAHelper.sendFailure(context, HAHelper.getRatisServer(statusSystem), e)
+      }
+    }
 
   override def receive: PartialFunction[Any, Unit] = {
     case _: PbCheckForWorkerTimeout =>
diff --git a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java
index 1a1b9ab8b..fe5f4271d 100644
--- a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java
+++ b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java
@@ -29,6 +29,7 @@ import org.junit.*;
 import org.mockito.Mockito;
 
 import org.apache.celeborn.common.CelebornConf;
+import org.apache.celeborn.common.exception.CelebornRuntimeException;
 import org.apache.celeborn.common.haclient.RssHARetryClient;
 import org.apache.celeborn.common.identity.UserIdentifier;
 import org.apache.celeborn.common.meta.DiskInfo;
@@ -58,10 +59,26 @@ public class RatisMasterStatusSystemSuiteJ {
 
   @BeforeClass
   public static void init() throws IOException, InterruptedException {
+    resetRaftServer();
+  }
+
+  public static void resetRaftServer() throws IOException, InterruptedException {
     Mockito.when(mockRpcEnv.setupEndpointRef(Mockito.any(), Mockito.any()))
         .thenReturn(mockRpcEndpoint);
     when(mockRpcEnv.setupEndpointRef(any(), any())).thenReturn(dummyRef);
 
+    if (RATISSERVER1 != null) {
+      RATISSERVER1.stop();
+    }
+
+    if (RATISSERVER2 != null) {
+      RATISSERVER2.stop();
+    }
+
+    if (RATISSERVER3 != null) {
+      RATISSERVER3.stop();
+    }
+
     STATUSSYSTEM1 = new HAMasterMetaManager(mockRpcEnv, new CelebornConf());
     STATUSSYSTEM2 = new HAMasterMetaManager(mockRpcEnv, new CelebornConf());
     STATUSSYSTEM3 = new HAMasterMetaManager(mockRpcEnv, new CelebornConf());
@@ -197,6 +214,37 @@ public class RatisMasterStatusSystemSuiteJ {
     return null;
   }
 
+  private void stopNoneLeaderRaftServer(HARaftServer... raftServers) {
+    for (HARaftServer raftServer : raftServers) {
+      if (!raftServer.isLeader()) {
+        raftServer.stop();
+      }
+    }
+  }
+
+  @Test
+  public void testRaftSystemException() throws Exception {
+    AbstractMetaManager statusSystem = pickLeaderStatusSystem();
+    Assert.assertNotNull(statusSystem);
+    try {
+      stopNoneLeaderRaftServer(RATISSERVER1, RATISSERVER2, RATISSERVER3);
+      statusSystem.handleRegisterWorker(
+          HOSTNAME1,
+          RPCPORT1,
+          PUSHPORT1,
+          FETCHPORT1,
+          REPLICATEPORT1,
+          disks1,
+          userResourceConsumption1,
+          getNewReqeustId());
+      Assert.assertTrue(false);
+    } catch (CelebornRuntimeException e) {
+      Assert.assertTrue(true);
+    } finally {
+      resetRaftServer();
+    }
+  }
+
   @Test
   public void testHandleRegisterWorker() throws InterruptedException {
     AbstractMetaManager statusSystem = pickLeaderStatusSystem();
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
index 3945376e5..c07953195 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
@@ -510,7 +510,17 @@ private[celeborn] class Worker(
           // During graceful shutdown, to avoid allocate slots in this worker,
           // add this worker to master's blacklist. When restart, register worker will
           // make master remove this worker from blacklist.
-          rssHARetryClient.send(ReportWorkerUnavailable(List(workerInfo).asJava))
+          try {
+            rssHARetryClient.askSync(
+              ReportWorkerUnavailable(List(workerInfo).asJava),
+              OneWayMessageResponse.getClass)
+          } catch {
+            case e: Throwable =>
+              logError(
+                s"Fail report to master, need wait PartitionLocation auto release: \n$partitionLocationInfo",
+                e)
+          }
+
           val interval = conf.checkSlotsFinishedInterval
           val timeout = conf.checkSlotsFinishedTimeoutMs
           var waitTimes = 0