You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by zh...@apache.org on 2023/04/15 14:27:33 UTC

[incubator-celeborn] branch main updated: [CELEBORN-521] correct exception and unify unRetryableException (#1425)

This is an automated email from the ASF dual-hosted git repository.

zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/main by this push:
     new a22c6ca74 [CELEBORN-521] correct exception and unify unRetryableException (#1425)
a22c6ca74 is described below

commit a22c6ca749a5b89ae3d9892d0d202b3ae3c903f8
Author: Shuang <lv...@gmail.com>
AuthorDate: Sat Apr 15 22:27:28 2023 +0800

    [CELEBORN-521] correct exception and unify unRetryableException (#1425)
---
 .../plugin/flink/RemoteBufferStreamReader.java     |  3 +--
 .../plugin/flink/RemoteShuffleInputGate.java       |  6 ++---
 .../network/server/MapDataPartitionReader.java     | 18 +++++++++++----
 .../celeborn/common/util/ExceptionUtils.java       | 14 +++++++++++
 .../common/exception/FileCorruptedException.scala  | 27 ++++++++++++++++++++++
 .../exception/PartitionUnRetryAbleException.scala  | 27 ++++++++++++++++++++++
 .../service/deploy/worker/FetchHandler.scala       | 23 +++++++++++-------
 7 files changed, 100 insertions(+), 18 deletions(-)

diff --git a/client-flink/flink-1.14/src/main/java/org/apache/celeborn/plugin/flink/RemoteBufferStreamReader.java b/client-flink/flink-1.14/src/main/java/org/apache/celeborn/plugin/flink/RemoteBufferStreamReader.java
index 4cb98fbc1..2761e6819 100644
--- a/client-flink/flink-1.14/src/main/java/org/apache/celeborn/plugin/flink/RemoteBufferStreamReader.java
+++ b/client-flink/flink-1.14/src/main/java/org/apache/celeborn/plugin/flink/RemoteBufferStreamReader.java
@@ -23,7 +23,6 @@ import org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.celeborn.common.exception.CelebornIOException;
 import org.apache.celeborn.common.network.protocol.BacklogAnnouncement;
 import org.apache.celeborn.common.network.protocol.ReadAddCredit;
 import org.apache.celeborn.common.network.protocol.RequestMessage;
@@ -90,7 +89,7 @@ public class RemoteBufferStreamReader extends CreditListener {
           RemoteBufferStreamReader.this::requestBuffer, initialCredit, messageConsumer);
     } catch (Exception e) {
       logger.warn("Failed to open stream and report to flink framework. ", e);
-      messageConsumer.accept(new TransportableError(0L, new CelebornIOException(e)));
+      messageConsumer.accept(new TransportableError(0L, e));
     }
     isOpened = true;
   }
diff --git a/client-flink/flink-1.14/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleInputGate.java b/client-flink/flink-1.14/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleInputGate.java
index 6ea53f82d..d3f2574bf 100644
--- a/client-flink/flink-1.14/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleInputGate.java
+++ b/client-flink/flink-1.14/src/main/java/org/apache/celeborn/plugin/flink/RemoteShuffleInputGate.java
@@ -62,7 +62,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.celeborn.common.CelebornConf;
-import org.apache.celeborn.common.exception.CelebornIOException;
+import org.apache.celeborn.common.exception.PartitionUnRetryAbleException;
 import org.apache.celeborn.common.identity.UserIdentifier;
 import org.apache.celeborn.plugin.flink.buffer.BufferPacker;
 import org.apache.celeborn.plugin.flink.buffer.TransferBufferPool;
@@ -484,8 +484,8 @@ public class RemoteShuffleInputGate extends IndexedInputGate {
           return;
         }
         LOG.error("Input {} gate failed ", this, throwable);
-        Class<?> clazz = CelebornIOException.class;
-        if (throwable.getMessage().contains(clazz.getName())) {
+        Class<?> clazz = PartitionUnRetryAbleException.class;
+        if (throwable.getMessage() != null && throwable.getMessage().contains(clazz.getName())) {
           cause = new PartitionNotFoundException(rpID);
         } else {
           cause = throwable;
diff --git a/common/src/main/java/org/apache/celeborn/common/network/server/MapDataPartitionReader.java b/common/src/main/java/org/apache/celeborn/common/network/server/MapDataPartitionReader.java
index d030f476f..759cb64ac 100644
--- a/common/src/main/java/org/apache/celeborn/common/network/server/MapDataPartitionReader.java
+++ b/common/src/main/java/org/apache/celeborn/common/network/server/MapDataPartitionReader.java
@@ -19,6 +19,7 @@ package org.apache.celeborn.common.network.server;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
 import java.nio.channels.FileChannel;
 import java.util.ArrayDeque;
 import java.util.Queue;
@@ -33,7 +34,7 @@ import io.netty.channel.ChannelFutureListener;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.celeborn.common.exception.CelebornIOException;
+import org.apache.celeborn.common.exception.FileCorruptedException;
 import org.apache.celeborn.common.meta.FileInfo;
 import org.apache.celeborn.common.network.protocol.BacklogAnnouncement;
 import org.apache.celeborn.common.network.protocol.ReadData;
@@ -41,6 +42,7 @@ import org.apache.celeborn.common.network.protocol.TransportableError;
 import org.apache.celeborn.common.network.server.memory.BufferQueue;
 import org.apache.celeborn.common.network.server.memory.BufferRecycler;
 import org.apache.celeborn.common.network.server.memory.RecyclableBuffer;
+import org.apache.celeborn.common.util.ExceptionUtils;
 import org.apache.celeborn.common.util.Utils;
 
 public class MapDataPartitionReader implements Comparable<MapDataPartitionReader> {
@@ -257,7 +259,7 @@ public class MapDataPartitionReader implements Comparable<MapDataPartitionReader
     int bufferLength = header.getInt(12);
     if (bufferLength <= 0 || bufferLength > buffer.capacity()) {
       logger.error("Incorrect buffer header, buffer length: {}.", bufferLength);
-      throw new RuntimeException("File " + filename + " is corrupted");
+      throw new FileCorruptedException("File " + filename + " is corrupted");
     }
     buffer.writeBytes(header);
     readBufferIntoReadBuffer(channel, buffer, bufferLength);
@@ -294,7 +296,7 @@ public class MapDataPartitionReader implements Comparable<MapDataPartitionReader
       if (dataConsumingOffset < 0
           || dataConsumingOffset + currentPartitionRemainingBytes > dataFileChannel.size()
           || currentPartitionRemainingBytes < 0) {
-        throw new RuntimeException("File " + fileInfo.getFilePath() + " is corrupted");
+        throw new FileCorruptedException("File " + fileInfo.getFilePath() + " is corrupted");
       }
     }
   }
@@ -323,7 +325,7 @@ public class MapDataPartitionReader implements Comparable<MapDataPartitionReader
 
       // if this check fails, the partition file must be corrupted
       if (currentPartitionRemainingBytes < 0) {
-        throw new RuntimeException("File is corrupted");
+        throw new FileCorruptedException("File is corrupted");
       } else if (currentPartitionRemainingBytes == 0) {
         logger.debug(
             "readBuffer end, {},  {}, {}, {}",
@@ -370,11 +372,17 @@ public class MapDataPartitionReader implements Comparable<MapDataPartitionReader
 
   private void notifyError(Throwable throwable) {
     logger.error("read error stream id {} message:{}", streamId, throwable.getMessage(), throwable);
+    if (throwable instanceof ClosedChannelException) {
+      return;
+    }
+
     if (this.associatedChannel.isActive()) {
       // If a stream is failed, send exceptions with the best effort, do not expect response.
       // And do not close channel because multiple streams are using the very same channel.
+      // wrapIOException to PartitionUnRetryAbleException, client may choose regenerate the data.
       this.associatedChannel.writeAndFlush(
-          new TransportableError(streamId, new CelebornIOException(throwable)));
+          new TransportableError(
+              streamId, ExceptionUtils.wrapIOExceptionToUnRetryable(throwable, true)));
     }
   }
 
diff --git a/common/src/main/java/org/apache/celeborn/common/util/ExceptionUtils.java b/common/src/main/java/org/apache/celeborn/common/util/ExceptionUtils.java
index ea218122d..5b7b0a228 100644
--- a/common/src/main/java/org/apache/celeborn/common/util/ExceptionUtils.java
+++ b/common/src/main/java/org/apache/celeborn/common/util/ExceptionUtils.java
@@ -17,11 +17,14 @@
 
 package org.apache.celeborn.common.util;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.io.StringWriter;
 
 import org.apache.celeborn.common.exception.CelebornIOException;
+import org.apache.celeborn.common.exception.FileCorruptedException;
+import org.apache.celeborn.common.exception.PartitionUnRetryAbleException;
 
 public class ExceptionUtils {
 
@@ -35,6 +38,17 @@ public class ExceptionUtils {
     }
   }
 
+  public static Throwable wrapIOExceptionToUnRetryable(
+      Throwable throwable, boolean convertAllIOException2UnRetryable) {
+    if (throwable instanceof FileNotFoundException || throwable instanceof FileCorruptedException) {
+      return new PartitionUnRetryAbleException(throwable.getMessage(), throwable);
+    } else if (throwable instanceof IOException && convertAllIOException2UnRetryable) {
+      return new PartitionUnRetryAbleException(throwable.getMessage(), throwable);
+    } else {
+      return throwable;
+    }
+  }
+
   public static String stringifyException(Throwable exception) {
     if (exception == null) {
       return "(null)";
diff --git a/common/src/main/scala/org/apache/celeborn/common/exception/FileCorruptedException.scala b/common/src/main/scala/org/apache/celeborn/common/exception/FileCorruptedException.scala
new file mode 100644
index 000000000..ba97897d6
--- /dev/null
+++ b/common/src/main/scala/org/apache/celeborn/common/exception/FileCorruptedException.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.common.exception
+
+import java.io.IOException
+
+class FileCorruptedException(message: String, cause: Throwable)
+  extends IOException(message, cause) {
+
+  def this(message: String) = this(message, null)
+
+}
diff --git a/common/src/main/scala/org/apache/celeborn/common/exception/PartitionUnRetryAbleException.scala b/common/src/main/scala/org/apache/celeborn/common/exception/PartitionUnRetryAbleException.scala
new file mode 100644
index 000000000..042b08c45
--- /dev/null
+++ b/common/src/main/scala/org/apache/celeborn/common/exception/PartitionUnRetryAbleException.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.common.exception
+
+import java.io.IOException
+
+class PartitionUnRetryAbleException(message: String, cause: Throwable)
+  extends IOException(message, cause) {
+
+  def this(message: String) = this(message, null)
+
+}
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/FetchHandler.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/FetchHandler.scala
index 338b1d299..b30929382 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/FetchHandler.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/FetchHandler.scala
@@ -26,7 +26,6 @@ import java.util.function.Consumer
 import com.google.common.base.Throwables
 import io.netty.util.concurrent.{Future, GenericFutureListener}
 
-import org.apache.celeborn.common.exception.CelebornException
 import org.apache.celeborn.common.internal.Logging
 import org.apache.celeborn.common.meta.{FileInfo, FileManagedBuffers}
 import org.apache.celeborn.common.network.buffer.NioManagedBuffer
@@ -36,6 +35,7 @@ import org.apache.celeborn.common.network.protocol.Message.Type
 import org.apache.celeborn.common.network.server.{BaseMessageHandler, ChunkStreamManager, CreditStreamManager}
 import org.apache.celeborn.common.network.util.{NettyUtils, TransportConf}
 import org.apache.celeborn.common.protocol.PartitionType
+import org.apache.celeborn.common.util.ExceptionUtils
 import org.apache.celeborn.service.deploy.worker.storage.{PartitionFilesSorter, StorageManager}
 
 class FetchHandler(val conf: TransportConf) extends BaseMessageHandler with Logging {
@@ -176,10 +176,7 @@ class FetchHandler(val conf: TransportConf) extends BaseMessageHandler with Logg
         case PartitionType.MAPGROUP =>
       } catch {
         case e: IOException =>
-          client.getChannel.writeAndFlush(new RpcFailure(
-            request.requestId,
-            Throwables.getStackTraceAsString(
-              new CelebornException("Chunk offsets meta exception", e))))
+          handleRpcIOException(client, request.requestId, e)
       } finally {
         // metrics end
         workerSource.stopTimer(WorkerSource.OpenStreamTime, shuffleKey)
@@ -188,12 +185,22 @@ class FetchHandler(val conf: TransportConf) extends BaseMessageHandler with Logg
     } catch {
       case ioe: IOException =>
         workerSource.stopTimer(WorkerSource.OpenStreamTime, shuffleKey)
-        client.getChannel.writeAndFlush(new RpcFailure(
-          request.requestId,
-          Throwables.getStackTraceAsString(ioe)))
+        handleRpcIOException(client, request.requestId, ioe)
     }
   }
 
+  private def handleRpcIOException(
+      client: TransportClient,
+      requestId: Long,
+      ioe: IOException): Unit = {
+    // if open stream rpc failed, this IOException actually should be FileNotFoundException
+    // we wrapper this IOException(Other place may have other exception like FileCorruptException) unify to
+    // PartitionUnRetryableException for reader can give up this partition and choose to regenerate the partition data
+    client.getChannel.writeAndFlush(new RpcFailure(
+      requestId,
+      Throwables.getStackTraceAsString(ExceptionUtils.wrapIOExceptionToUnRetryable(ioe, false))))
+  }
+
   def handleEndStreamFromClient(req: BufferStreamEnd): Unit = {
     creditStreamManager.notifyStreamEndByClient(req.getStreamId)
   }