(uniffle) branch master updated: [#2560] improvement(client): Fast fail on hadoop reader initialization failure (#2551)

[zu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zuston pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/uniffle.git


The following commit(s) were added to refs/heads/master by this push:
     new be3a1ffc0 [#2560] improvement(client): Fast fail on hadoop reader initialization failure (#2551)
be3a1ffc0 is described below

commit be3a1ffc027fb6e587367905d21e5224d5839b16
Author: Junfan Zhang <zu...@apache.org>
AuthorDate: Tue Jul 29 10:28:01 2025 +0800

    [#2560] improvement(client): Fast fail on hadoop reader initialization failure (#2551)
    
    ### What changes were proposed in this pull request?
    
    Fast fail on hadoop reader initialization failure
    
    ### Why are the changes needed?
    
    When the hadoop reader initialization failed, this exception will be ignored. When the final unexcepted blockIds throws, it’s hard to find out the root cause.
    
    And so, we should fast fail in this case.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Existing unit tests.
---
 .../apache/spark/shuffle/reader/RssShuffleDataIteratorTest.java   | 3 ++-
 .../org/apache/uniffle/client/impl/ShuffleReadClientImplTest.java | 8 +++++++-
 .../uniffle/storage/handler/impl/HadoopClientReadHandler.java     | 1 +
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/client-spark/common/src/test/java/org/apache/spark/shuffle/reader/RssShuffleDataIteratorTest.java b/client-spark/common/src/test/java/org/apache/spark/shuffle/reader/RssShuffleDataIteratorTest.java
index 5550f67c0..0aff0706c 100644
--- a/client-spark/common/src/test/java/org/apache/spark/shuffle/reader/RssShuffleDataIteratorTest.java
+++ b/client-spark/common/src/test/java/org/apache/spark/shuffle/reader/RssShuffleDataIteratorTest.java
@@ -229,7 +229,8 @@ public class RssShuffleDataIteratorTest extends AbstractRssReaderTest {
       }
       fail(EXPECTED_EXCEPTION_MESSAGE);
     } catch (Exception e) {
-      assertTrue(e.getMessage().startsWith("Blocks read inconsistent: expected"));
+      // the underlying hdfs files have been deleted, so that the reader will initialize failed
+      assertTrue(e.getMessage().contains("don't exist or is not a file"));
     }
   }
 
diff --git a/client/src/test/java/org/apache/uniffle/client/impl/ShuffleReadClientImplTest.java b/client/src/test/java/org/apache/uniffle/client/impl/ShuffleReadClientImplTest.java
index 396657f7a..e8538754f 100644
--- a/client/src/test/java/org/apache/uniffle/client/impl/ShuffleReadClientImplTest.java
+++ b/client/src/test/java/org/apache/uniffle/client/impl/ShuffleReadClientImplTest.java
@@ -230,7 +230,13 @@ public class ShuffleReadClientImplTest extends HadoopTestBase {
     // data file is deleted after readClient checkExpectedBlockIds
     fs.delete(dataFile, true);
 
-    assertNull(readClient.readShuffleBlockData());
+    try {
+      readClient.readShuffleBlockData();
+      fail("Hdfs file reader should fail due to non-existence");
+    } catch (Exception e) {
+      // ignore
+    }
+
     try {
       fs.listStatus(dataFile);
       fail("Index file should be deleted");
diff --git a/storage/src/main/java/org/apache/uniffle/storage/handler/impl/HadoopClientReadHandler.java b/storage/src/main/java/org/apache/uniffle/storage/handler/impl/HadoopClientReadHandler.java
index 161740786..169d1c3fe 100644
--- a/storage/src/main/java/org/apache/uniffle/storage/handler/impl/HadoopClientReadHandler.java
+++ b/storage/src/main/java/org/apache/uniffle/storage/handler/impl/HadoopClientReadHandler.java
@@ -190,6 +190,7 @@ public class HadoopClientReadHandler extends AbstractClientReadHandler {
           readHandlers.add(handler);
         } catch (Exception e) {
           LOG.warn("Can't create ShuffleReaderHandler for " + filePrefix, e);
+          throw new RssException(e);
         }
       }
       Collections.shuffle(readHandlers);