You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by zh...@apache.org on 2023/04/15 14:27:33 UTC
[incubator-celeborn] branch main updated: [CELEBORN-521] correct exception and unify unRetryableException (#1425)
This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new a22c6ca74 [CELEBORN-521] correct exception and unify unRetryableException (#1425)
a22c6ca74 is described below
commit a22c6ca749a5b89ae3d9892d0d202b3ae3c903f8
Author: Shuang <lv...@gmail.com>
AuthorDate: Sat Apr 15 22:27:28 2023 +0800
[CELEBORN-521] correct exception and unify unRetryableException (#1425)
---
.../plugin/flink/RemoteBufferStreamReader.java | 3 +--
.../plugin/flink/RemoteShuffleInputGate.java | 6 ++---
.../network/server/MapDataPartitionReader.java | 18 +++++++++++----
.../celeborn/common/util/ExceptionUtils.java | 14 +++++++++++
.../common/exception/FileCorruptedException.scala | 27 ++++++++++++++++++++++
.../exception/PartitionUnRetryAbleException.scala | 27 ++++++++++++++++++++++
.../service/deploy/worker/FetchHandler.scala | 23 +++++++++++-------
7 files changed, 100 insertions(+), 18 deletions(-)
diff --git a/client-flink/flink-1.14/src/main/java/org/apache/celeborn/plugin/flink/RemoteBufferStreamReader.java b/client-flink/flink-1.14/src/main/java/org/apache/celeborn/plugin/flink/RemoteBufferStreamReader.java
index 4cb98fbc1..2761e6819 100644
--- a/client-flink/flink-1.14/src/main/java/org/apache/celeborn/plugin/flink/RemoteBufferStreamReader.java
+++ b/client-flink/flink-1.14/src/main/java/org/apache/celeborn/plugin/flink/RemoteBufferStreamReader.java
@@ -23,7 +23,6 @@ import org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.celeborn.common.exception.CelebornIOException;
import org.apache.celeborn.common.network.protocol.BacklogAnnouncement;
import org.apache.celeborn.common.network.protocol.ReadAddCredit;
import org.apache.celeborn.common.network.protocol.RequestMessage;
@@ -90,7 +89,7 @@ public class RemoteBufferStreamReader extends CreditListener {
RemoteBufferStreamReader.this::requestBuffer, initialCredit, messageConsumer);
} catch (Exception e) {
logger.warn("Failed to open stream and report to flink framework. ", e);
- messageConsumer.accept(new TransportableError(0L, new CelebornIOException(e)));
+ messageConsumer.accept(new TransportableError(0L, e));
}
isOpened = true;
}
diff --git a/client-flink/flink-1.14/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleInputGate.java b/client-flink/flink-1.14/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleInputGate.java
index 6ea53f82d..d3f2574bf 100644
--- a/client-flink/flink-1.14/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleInputGate.java
+++ b/client-flink/flink-1.14/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleInputGate.java
@@ -62,7 +62,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.celeborn.common.CelebornConf;
-import org.apache.celeborn.common.exception.CelebornIOException;
+import org.apache.celeborn.common.exception.PartitionUnRetryAbleException;
import org.apache.celeborn.common.identity.UserIdentifier;
import org.apache.celeborn.plugin.flink.buffer.BufferPacker;
import org.apache.celeborn.plugin.flink.buffer.TransferBufferPool;
@@ -484,8 +484,8 @@ public class RemoteShuffleInputGate extends IndexedInputGate {
return;
}
LOG.error("Input {} gate failed ", this, throwable);
- Class<?> clazz = CelebornIOException.class;
- if (throwable.getMessage().contains(clazz.getName())) {
+ Class<?> clazz = PartitionUnRetryAbleException.class;
+ if (throwable.getMessage() != null && throwable.getMessage().contains(clazz.getName())) {
cause = new PartitionNotFoundException(rpID);
} else {
cause = throwable;
diff --git a/common/src/main/java/org/apache/celeborn/common/network/server/MapDataPartitionReader.java b/common/src/main/java/org/apache/celeborn/common/network/server/MapDataPartitionReader.java
index d030f476f..759cb64ac 100644
--- a/common/src/main/java/org/apache/celeborn/common/network/server/MapDataPartitionReader.java
+++ b/common/src/main/java/org/apache/celeborn/common/network/server/MapDataPartitionReader.java
@@ -19,6 +19,7 @@ package org.apache.celeborn.common.network.server;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
import java.nio.channels.FileChannel;
import java.util.ArrayDeque;
import java.util.Queue;
@@ -33,7 +34,7 @@ import io.netty.channel.ChannelFutureListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.celeborn.common.exception.CelebornIOException;
+import org.apache.celeborn.common.exception.FileCorruptedException;
import org.apache.celeborn.common.meta.FileInfo;
import org.apache.celeborn.common.network.protocol.BacklogAnnouncement;
import org.apache.celeborn.common.network.protocol.ReadData;
@@ -41,6 +42,7 @@ import org.apache.celeborn.common.network.protocol.TransportableError;
import org.apache.celeborn.common.network.server.memory.BufferQueue;
import org.apache.celeborn.common.network.server.memory.BufferRecycler;
import org.apache.celeborn.common.network.server.memory.RecyclableBuffer;
+import org.apache.celeborn.common.util.ExceptionUtils;
import org.apache.celeborn.common.util.Utils;
public class MapDataPartitionReader implements Comparable<MapDataPartitionReader> {
@@ -257,7 +259,7 @@ public class MapDataPartitionReader implements Comparable<MapDataPartitionReader
int bufferLength = header.getInt(12);
if (bufferLength <= 0 || bufferLength > buffer.capacity()) {
logger.error("Incorrect buffer header, buffer length: {}.", bufferLength);
- throw new RuntimeException("File " + filename + " is corrupted");
+ throw new FileCorruptedException("File " + filename + " is corrupted");
}
buffer.writeBytes(header);
readBufferIntoReadBuffer(channel, buffer, bufferLength);
@@ -294,7 +296,7 @@ public class MapDataPartitionReader implements Comparable<MapDataPartitionReader
if (dataConsumingOffset < 0
|| dataConsumingOffset + currentPartitionRemainingBytes > dataFileChannel.size()
|| currentPartitionRemainingBytes < 0) {
- throw new RuntimeException("File " + fileInfo.getFilePath() + " is corrupted");
+ throw new FileCorruptedException("File " + fileInfo.getFilePath() + " is corrupted");
}
}
}
@@ -323,7 +325,7 @@ public class MapDataPartitionReader implements Comparable<MapDataPartitionReader
// if this check fails, the partition file must be corrupted
if (currentPartitionRemainingBytes < 0) {
- throw new RuntimeException("File is corrupted");
+ throw new FileCorruptedException("File is corrupted");
} else if (currentPartitionRemainingBytes == 0) {
logger.debug(
"readBuffer end, {}, {}, {}, {}",
@@ -370,11 +372,17 @@ public class MapDataPartitionReader implements Comparable<MapDataPartitionReader
private void notifyError(Throwable throwable) {
logger.error("read error stream id {} message:{}", streamId, throwable.getMessage(), throwable);
+ if (throwable instanceof ClosedChannelException) {
+ return;
+ }
+
if (this.associatedChannel.isActive()) {
// If a stream is failed, send exceptions with the best effort, do not expect response.
// And do not close channel because multiple streams are using the very same channel.
+ // wrapIOException to PartitionUnRetryAbleException, client may choose regenerate the data.
this.associatedChannel.writeAndFlush(
- new TransportableError(streamId, new CelebornIOException(throwable)));
+ new TransportableError(
+ streamId, ExceptionUtils.wrapIOExceptionToUnRetryable(throwable, true)));
}
}
diff --git a/common/src/main/java/org/apache/celeborn/common/util/ExceptionUtils.java b/common/src/main/java/org/apache/celeborn/common/util/ExceptionUtils.java
index ea218122d..5b7b0a228 100644
--- a/common/src/main/java/org/apache/celeborn/common/util/ExceptionUtils.java
+++ b/common/src/main/java/org/apache/celeborn/common/util/ExceptionUtils.java
@@ -17,11 +17,14 @@
package org.apache.celeborn.common.util;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import org.apache.celeborn.common.exception.CelebornIOException;
+import org.apache.celeborn.common.exception.FileCorruptedException;
+import org.apache.celeborn.common.exception.PartitionUnRetryAbleException;
public class ExceptionUtils {
@@ -35,6 +38,17 @@ public class ExceptionUtils {
}
}
+ public static Throwable wrapIOExceptionToUnRetryable(
+ Throwable throwable, boolean convertAllIOException2UnRetryable) {
+ if (throwable instanceof FileNotFoundException || throwable instanceof FileCorruptedException) {
+ return new PartitionUnRetryAbleException(throwable.getMessage(), throwable);
+ } else if (throwable instanceof IOException && convertAllIOException2UnRetryable) {
+ return new PartitionUnRetryAbleException(throwable.getMessage(), throwable);
+ } else {
+ return throwable;
+ }
+ }
+
public static String stringifyException(Throwable exception) {
if (exception == null) {
return "(null)";
diff --git a/common/src/main/scala/org/apache/celeborn/common/exception/FileCorruptedException.scala b/common/src/main/scala/org/apache/celeborn/common/exception/FileCorruptedException.scala
new file mode 100644
index 000000000..ba97897d6
--- /dev/null
+++ b/common/src/main/scala/org/apache/celeborn/common/exception/FileCorruptedException.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.common.exception
+
+import java.io.IOException
+
+class FileCorruptedException(message: String, cause: Throwable)
+ extends IOException(message, cause) {
+
+ def this(message: String) = this(message, null)
+
+}
diff --git a/common/src/main/scala/org/apache/celeborn/common/exception/PartitionUnRetryAbleException.scala b/common/src/main/scala/org/apache/celeborn/common/exception/PartitionUnRetryAbleException.scala
new file mode 100644
index 000000000..042b08c45
--- /dev/null
+++ b/common/src/main/scala/org/apache/celeborn/common/exception/PartitionUnRetryAbleException.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.common.exception
+
+import java.io.IOException
+
+class PartitionUnRetryAbleException(message: String, cause: Throwable)
+ extends IOException(message, cause) {
+
+ def this(message: String) = this(message, null)
+
+}
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/FetchHandler.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/FetchHandler.scala
index 338b1d299..b30929382 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/FetchHandler.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/FetchHandler.scala
@@ -26,7 +26,6 @@ import java.util.function.Consumer
import com.google.common.base.Throwables
import io.netty.util.concurrent.{Future, GenericFutureListener}
-import org.apache.celeborn.common.exception.CelebornException
import org.apache.celeborn.common.internal.Logging
import org.apache.celeborn.common.meta.{FileInfo, FileManagedBuffers}
import org.apache.celeborn.common.network.buffer.NioManagedBuffer
@@ -36,6 +35,7 @@ import org.apache.celeborn.common.network.protocol.Message.Type
import org.apache.celeborn.common.network.server.{BaseMessageHandler, ChunkStreamManager, CreditStreamManager}
import org.apache.celeborn.common.network.util.{NettyUtils, TransportConf}
import org.apache.celeborn.common.protocol.PartitionType
+import org.apache.celeborn.common.util.ExceptionUtils
import org.apache.celeborn.service.deploy.worker.storage.{PartitionFilesSorter, StorageManager}
class FetchHandler(val conf: TransportConf) extends BaseMessageHandler with Logging {
@@ -176,10 +176,7 @@ class FetchHandler(val conf: TransportConf) extends BaseMessageHandler with Logg
case PartitionType.MAPGROUP =>
} catch {
case e: IOException =>
- client.getChannel.writeAndFlush(new RpcFailure(
- request.requestId,
- Throwables.getStackTraceAsString(
- new CelebornException("Chunk offsets meta exception", e))))
+ handleRpcIOException(client, request.requestId, e)
} finally {
// metrics end
workerSource.stopTimer(WorkerSource.OpenStreamTime, shuffleKey)
@@ -188,12 +185,22 @@ class FetchHandler(val conf: TransportConf) extends BaseMessageHandler with Logg
} catch {
case ioe: IOException =>
workerSource.stopTimer(WorkerSource.OpenStreamTime, shuffleKey)
- client.getChannel.writeAndFlush(new RpcFailure(
- request.requestId,
- Throwables.getStackTraceAsString(ioe)))
+ handleRpcIOException(client, request.requestId, ioe)
}
}
+ private def handleRpcIOException(
+ client: TransportClient,
+ requestId: Long,
+ ioe: IOException): Unit = {
+ // if open stream rpc failed, this IOException actually should be FileNotFoundException
+ // we wrapper this IOException(Other place may have other exception like FileCorruptException) unify to
+ // PartitionUnRetryableException for reader can give up this partition and choose to regenerate the partition data
+ client.getChannel.writeAndFlush(new RpcFailure(
+ requestId,
+ Throwables.getStackTraceAsString(ExceptionUtils.wrapIOExceptionToUnRetryable(ioe, false))))
+ }
+
def handleEndStreamFromClient(req: BufferStreamEnd): Unit = {
creditStreamManager.notifyStreamEndByClient(req.getStreamId)
}