You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by re...@apache.org on 2023/05/10 06:28:57 UTC
[incubator-celeborn] branch main updated: [CELEBORN-573] Guarantee resource/app/worker change persistent to raft in Ha Mode. (#1477)
This is an automated email from the ASF dual-hosted git repository.
rexxiong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new fb753fd48 [CELEBORN-573] Guarantee resource/app/worker change persistent to raft in Ha Mode. (#1477)
fb753fd48 is described below
commit fb753fd48eb3e5c470e714e61e73fb4fc0bb5cba
Author: Shuang <lv...@gmail.com>
AuthorDate: Wed May 10 14:28:52 2023 +0800
[CELEBORN-573] Guarantee resource/app/worker change persistent to raft in Ha Mode. (#1477)
---
.../apache/celeborn/client/LifecycleManager.scala | 7 +++-
.../exception/CelebornRuntimeException.scala | 24 +++++++++++
.../deploy/master/clustermeta/ha/HAHelper.java | 16 ++++++--
.../master/clustermeta/ha/HAMasterMetaManager.java | 35 ++++++++++------
.../deploy/master/clustermeta/ha/HARaftServer.java | 19 ++++-----
.../celeborn/service/deploy/master/Master.scala | 10 ++++-
.../ha/RatisMasterStatusSystemSuiteJ.java | 48 ++++++++++++++++++++++
.../celeborn/service/deploy/worker/Worker.scala | 12 +++++-
8 files changed, 141 insertions(+), 30 deletions(-)
diff --git a/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala b/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala
index 3c14c2070..ee4524389 100644
--- a/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala
+++ b/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala
@@ -999,13 +999,16 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
// clear for the shuffle
registeredShuffle.remove(shuffleId)
registeringShuffleRequest.remove(shuffleId)
- unregisterShuffleTime.remove(shuffleId)
shuffleAllocatedWorkers.remove(shuffleId)
latestPartitionLocation.remove(shuffleId)
commitManager.removeExpiredShuffle(shuffleId)
changePartitionManager.removeExpiredShuffle(shuffleId)
- requestMasterUnregisterShuffle(
+ val unregisterShuffleResponse = requestMasterUnregisterShuffle(
UnregisterShuffle(appId, shuffleId, RssHARetryClient.genRequestId()))
+ // if unregister shuffle not success, wait next turn
+ if (StatusCode.SUCCESS == Utils.toStatusCode(unregisterShuffleResponse.getStatus)) {
+ unregisterShuffleTime.remove(shuffleId)
+ }
}
}
}
diff --git a/common/src/main/scala/org/apache/celeborn/common/exception/CelebornRuntimeException.scala b/common/src/main/scala/org/apache/celeborn/common/exception/CelebornRuntimeException.scala
new file mode 100644
index 000000000..322bcbb78
--- /dev/null
+++ b/common/src/main/scala/org/apache/celeborn/common/exception/CelebornRuntimeException.scala
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.common.exception
+
+class CelebornRuntimeException(message: String, cause: Throwable)
+ extends RuntimeException(message, cause) {
+
+ def this(message: String) = this(message, null)
+}
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAHelper.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAHelper.java
index a17ce7c6e..3dc684c9b 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAHelper.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAHelper.java
@@ -25,6 +25,7 @@ import org.apache.ratis.protocol.Message;
import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.apache.celeborn.common.exception.CelebornIOException;
import org.apache.celeborn.common.haclient.MasterNotLeaderException;
import org.apache.celeborn.common.rpc.RpcCallContext;
import org.apache.celeborn.service.deploy.master.clustermeta.AbstractMetaManager;
@@ -39,7 +40,16 @@ public class HAHelper {
if (ratisServer.isLeader()) {
return true;
}
- if (context != null) {
+ sendFailure(context, ratisServer, null);
+ return false;
+ }
+ return true;
+ }
+
+ public static void sendFailure(
+ RpcCallContext context, HARaftServer ratisServer, Throwable cause) {
+ if (context != null) {
+ if (ratisServer != null) {
if (ratisServer.getCachedLeaderPeerRpcEndpoint().isPresent()) {
context.sendFailure(
new MasterNotLeaderException(
@@ -50,10 +60,10 @@ public class HAHelper {
new MasterNotLeaderException(
ratisServer.getRpcEndpoint(), MasterNotLeaderException.LEADER_NOT_PRESENTED));
}
+ } else {
+ context.sendFailure(new CelebornIOException(cause.getMessage(), cause));
}
- return false;
}
- return true;
}
public static long getWorkerTimeoutDeadline(AbstractMetaManager masterStatusSystem) {
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java
index 9db4102f4..e1c36ec54 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java
@@ -21,11 +21,11 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-import com.google.protobuf.ServiceException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.celeborn.common.CelebornConf;
+import org.apache.celeborn.common.exception.CelebornRuntimeException;
import org.apache.celeborn.common.haclient.RssHARetryClient;
import org.apache.celeborn.common.identity.UserIdentifier;
import org.apache.celeborn.common.meta.AppDiskUsageMetric;
@@ -84,8 +84,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
.setRequestId(requestId)
.setRequestSlotsRequest(builder.build())
.build());
- } catch (ServiceException e) {
+ } catch (CelebornRuntimeException e) {
LOG.error("Handle request slots for {} failed!", shuffleKey, e);
+ throw e;
}
}
@@ -112,8 +113,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
.collect(Collectors.toList()))
.build())
.build());
- } catch (ServiceException e) {
+ } catch (CelebornRuntimeException e) {
LOG.error("Handle release slots for {} failed!", shuffleKey, e);
+ throw e;
}
}
@@ -129,8 +131,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
.setShuffleKey(shuffleKey)
.build())
.build());
- } catch (ServiceException e) {
+ } catch (CelebornRuntimeException e) {
LOG.error("Handle unregister shuffle for {} failed!", shuffleKey, e);
+ throw e;
}
}
@@ -150,8 +153,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
.setFileCount(fileCount)
.build())
.build());
- } catch (ServiceException e) {
+ } catch (CelebornRuntimeException e) {
LOG.error("Handle heart beat for {} failed!", appId, e);
+ throw e;
}
}
@@ -164,8 +168,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
.setRequestId(requestId)
.setAppLostRequest(ResourceProtos.AppLostRequest.newBuilder().setAppId(appId).build())
.build());
- } catch (ServiceException e) {
+ } catch (CelebornRuntimeException e) {
LOG.error("Handle app lost for {} failed!", appId, e);
+ throw e;
}
}
@@ -186,8 +191,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
.setReplicatePort(replicatePort)
.build())
.build());
- } catch (ServiceException e) {
+ } catch (CelebornRuntimeException e) {
LOG.error("Handle worker lost for {} failed!", host, e);
+ throw e;
}
}
@@ -208,8 +214,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
.setReplicatePort(replicatePort)
.build())
.build());
- } catch (ServiceException e) {
+ } catch (CelebornRuntimeException e) {
LOG.error("Handle worker lost for {} failed!", host, e);
+ throw e;
}
}
@@ -244,8 +251,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
.setTime(time)
.build())
.build());
- } catch (ServiceException e) {
+ } catch (CelebornRuntimeException e) {
LOG.error("Handle worker heartbeat for {} failed!", host, e);
+ throw e;
}
}
@@ -276,8 +284,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
MetaUtil.toPbUserResourceConsumption(userResourceConsumption))
.build())
.build());
- } catch (ServiceException e) {
+ } catch (CelebornRuntimeException e) {
LOG.error("Handle worker register for {} failed!", host, e);
+ throw e;
}
}
@@ -295,8 +304,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
.addAllUnavailable(addrs)
.build())
.build());
- } catch (ServiceException e) {
+ } catch (CelebornRuntimeException e) {
LOG.error("Handle report node failure for {} failed!", failedNodes, e);
+ throw e;
}
}
@@ -308,8 +318,9 @@ public class HAMasterMetaManager extends AbstractMetaManager {
.setCmdType(Type.UpdatePartitionSize)
.setRequestId(RssHARetryClient.genRequestId())
.build());
- } catch (ServiceException e) {
+ } catch (CelebornRuntimeException e) {
LOG.error("Handle update partition size failed!", e);
+ throw e;
}
}
}
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java
index 5a49a1bb1..af9d549af 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java
@@ -31,7 +31,6 @@ import scala.Tuple2;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.InvalidProtocolBufferException;
-import com.google.protobuf.ServiceException;
import org.apache.ratis.RaftConfigKeys;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.grpc.GrpcConfigKeys;
@@ -55,6 +54,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.celeborn.common.CelebornConf;
+import org.apache.celeborn.common.exception.CelebornRuntimeException;
import org.apache.celeborn.common.haclient.RssHARetryClient;
import org.apache.celeborn.common.util.ThreadUtils;
import org.apache.celeborn.service.deploy.master.clustermeta.ResourceProtos;
@@ -199,11 +199,12 @@ public class HARaftServer {
}
public ResourceResponse submitRequest(ResourceProtos.ResourceRequest request)
- throws ServiceException {
+ throws CelebornRuntimeException {
String requestId = request.getRequestId();
Tuple2<String, Long> decoded = RssHARetryClient.decodeRequestId(requestId);
if (decoded == null) {
- throw new ServiceException("RequestId:" + requestId + " invalid, should be: uuid#callId.");
+ throw new CelebornRuntimeException(
+ "RequestId:" + requestId + " invalid, should be: uuid#callId.");
}
ClientId clientId = ClientId.valueOf(UUID.fromString(decoded._1));
long callId = decoded._2;
@@ -221,19 +222,19 @@ public class HARaftServer {
try {
raftClientReply = server.submitClientRequestAsync(raftClientRequest).get();
} catch (Exception ex) {
- throw new ServiceException(ex.getMessage(), ex);
+ throw new CelebornRuntimeException(ex.getMessage(), ex);
}
if (!raftClientReply.isSuccess()) {
NotLeaderException notLeaderException = raftClientReply.getNotLeaderException();
if (notLeaderException != null) {
- throw new ServiceException("Not leader!");
+ throw new CelebornRuntimeException("Not leader!");
}
LeaderNotReadyException leaderNotReadyException =
raftClientReply.getLeaderNotReadyException();
if (leaderNotReadyException != null) {
- throw new ServiceException("Not leader!");
+ throw new CelebornRuntimeException("Not leader!");
}
StateMachineException stateMachineException = raftClientReply.getStateMachineException();
@@ -261,11 +262,7 @@ public class HARaftServer {
byte[] bytes = raftClientReply.getMessage().getContent().toByteArray();
return ResourceResponse.newBuilder(ResourceResponse.parseFrom(bytes)).build();
} catch (InvalidProtocolBufferException ex) {
- if (ex.getMessage() != null) {
- throw new ServiceException(ex.getMessage(), ex);
- } else {
- throw new ServiceException(ex);
- }
+ throw new CelebornRuntimeException(ex.getMessage(), ex);
}
}
diff --git a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
index 0321a0562..5864e6574 100644
--- a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
+++ b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
@@ -26,6 +26,7 @@ import scala.collection.JavaConverters._
import scala.util.Random
import org.apache.celeborn.common.CelebornConf
+import org.apache.celeborn.common.exception.CelebornRuntimeException
import org.apache.celeborn.common.haclient.RssHARetryClient
import org.apache.celeborn.common.identity.UserIdentifier
import org.apache.celeborn.common.internal.Logging
@@ -200,7 +201,14 @@ private[celeborn] class Master(
}
def executeWithLeaderChecker[T](context: RpcCallContext, f: => T): Unit =
- if (HAHelper.checkShouldProcess(context, statusSystem)) f
+ if (HAHelper.checkShouldProcess(context, statusSystem)) {
+ try {
+ f
+ } catch {
+ case e: Exception =>
+ HAHelper.sendFailure(context, HAHelper.getRatisServer(statusSystem), e)
+ }
+ }
override def receive: PartialFunction[Any, Unit] = {
case _: PbCheckForWorkerTimeout =>
diff --git a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java
index 1a1b9ab8b..fe5f4271d 100644
--- a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java
+++ b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java
@@ -29,6 +29,7 @@ import org.junit.*;
import org.mockito.Mockito;
import org.apache.celeborn.common.CelebornConf;
+import org.apache.celeborn.common.exception.CelebornRuntimeException;
import org.apache.celeborn.common.haclient.RssHARetryClient;
import org.apache.celeborn.common.identity.UserIdentifier;
import org.apache.celeborn.common.meta.DiskInfo;
@@ -58,10 +59,26 @@ public class RatisMasterStatusSystemSuiteJ {
@BeforeClass
public static void init() throws IOException, InterruptedException {
+ resetRaftServer();
+ }
+
+ public static void resetRaftServer() throws IOException, InterruptedException {
Mockito.when(mockRpcEnv.setupEndpointRef(Mockito.any(), Mockito.any()))
.thenReturn(mockRpcEndpoint);
when(mockRpcEnv.setupEndpointRef(any(), any())).thenReturn(dummyRef);
+ if (RATISSERVER1 != null) {
+ RATISSERVER1.stop();
+ }
+
+ if (RATISSERVER2 != null) {
+ RATISSERVER2.stop();
+ }
+
+ if (RATISSERVER3 != null) {
+ RATISSERVER3.stop();
+ }
+
STATUSSYSTEM1 = new HAMasterMetaManager(mockRpcEnv, new CelebornConf());
STATUSSYSTEM2 = new HAMasterMetaManager(mockRpcEnv, new CelebornConf());
STATUSSYSTEM3 = new HAMasterMetaManager(mockRpcEnv, new CelebornConf());
@@ -197,6 +214,37 @@ public class RatisMasterStatusSystemSuiteJ {
return null;
}
+ private void stopNoneLeaderRaftServer(HARaftServer... raftServers) {
+ for (HARaftServer raftServer : raftServers) {
+ if (!raftServer.isLeader()) {
+ raftServer.stop();
+ }
+ }
+ }
+
+ @Test
+ public void testRaftSystemException() throws Exception {
+ AbstractMetaManager statusSystem = pickLeaderStatusSystem();
+ Assert.assertNotNull(statusSystem);
+ try {
+ stopNoneLeaderRaftServer(RATISSERVER1, RATISSERVER2, RATISSERVER3);
+ statusSystem.handleRegisterWorker(
+ HOSTNAME1,
+ RPCPORT1,
+ PUSHPORT1,
+ FETCHPORT1,
+ REPLICATEPORT1,
+ disks1,
+ userResourceConsumption1,
+ getNewReqeustId());
+ Assert.assertTrue(false);
+ } catch (CelebornRuntimeException e) {
+ Assert.assertTrue(true);
+ } finally {
+ resetRaftServer();
+ }
+ }
+
@Test
public void testHandleRegisterWorker() throws InterruptedException {
AbstractMetaManager statusSystem = pickLeaderStatusSystem();
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
index 3945376e5..c07953195 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
@@ -510,7 +510,17 @@ private[celeborn] class Worker(
// During graceful shutdown, to avoid allocate slots in this worker,
// add this worker to master's blacklist. When restart, register worker will
// make master remove this worker from blacklist.
- rssHARetryClient.send(ReportWorkerUnavailable(List(workerInfo).asJava))
+ try {
+ rssHARetryClient.askSync(
+ ReportWorkerUnavailable(List(workerInfo).asJava),
+ OneWayMessageResponse.getClass)
+ } catch {
+ case e: Throwable =>
+ logError(
+ s"Fail report to master, need wait PartitionLocation auto release: \n$partitionLocationInfo",
+ e)
+ }
+
val interval = conf.checkSlotsFinishedInterval
val timeout = conf.checkSlotsFinishedTimeoutMs
var waitTimes = 0