You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uniffle.apache.org by zu...@apache.org on 2026/03/27 09:40:00 UTC
(uniffle) branch master updated: fix(spark): potential hang with skipped segments on overlapping decompression (#2745)
This is an automated email from the ASF dual-hosted git repository.
zuston pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new ae417e632 fix(spark): potential hang with skipped segments on overlapping decompression (#2745)
ae417e632 is described below
commit ae417e63228dc46157c35c660fc5598650f6a618
Author: Junfan Zhang <zu...@apache.org>
AuthorDate: Fri Mar 27 17:39:55 2026 +0800
fix(spark): potential hang with skipped segments on overlapping decompression (#2745)
### What changes were proposed in this pull request?
fix the PR #2735
### Why are the changes needed?
fix potential hang
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Unit tests
---
.../uniffle/client/impl/DecompressionWorker.java | 18 ++++++++++
.../uniffle/client/impl/ShuffleReadClientImpl.java | 12 +++++--
.../client/impl/DecompressionWorkerTest.java | 5 ++-
.../client/impl/ShuffleReadClientImplTest.java | 42 ++++++++++++++++++++++
4 files changed, 74 insertions(+), 3 deletions(-)
diff --git a/client/src/main/java/org/apache/uniffle/client/impl/DecompressionWorker.java b/client/src/main/java/org/apache/uniffle/client/impl/DecompressionWorker.java
index fbb81528d..a98f9553e 100644
--- a/client/src/main/java/org/apache/uniffle/client/impl/DecompressionWorker.java
+++ b/client/src/main/java/org/apache/uniffle/client/impl/DecompressionWorker.java
@@ -154,10 +154,28 @@ public class DecompressionWorker {
}
public DecompressedShuffleBlock get(int batchIndex, int segmentIndex) {
+ // guardedly safe to remove the previous batches if exist since the upstream will fetch the
+ // segments in order
+ for (int i = 0; i < batchIndex; i++) {
+ ConcurrentHashMap<Integer, DecompressedShuffleBlock> prevBlocks = tasks.remove(i);
+ if (prevBlocks != null) {
+ segmentPermits.ifPresent(x -> x.release(prevBlocks.values().size()));
+ }
+ }
+
ConcurrentHashMap<Integer, DecompressedShuffleBlock> blocks = tasks.get(batchIndex);
if (blocks == null) {
return null;
}
+
+ // guardedly safe to remove the previous segments if exist since the upstream will fetch the
+ // segments in order
+ for (int i = 0; i < segmentIndex; i++) {
+ if (blocks.remove(i) != null) {
+ segmentPermits.ifPresent(x -> x.release());
+ }
+ }
+
DecompressedShuffleBlock block = blocks.remove(segmentIndex);
// simplify the memory statistic logic here, just decrease the memory used when the block is
// fetched, this is effective due to the upstream will use single-thread to get and release the
diff --git a/client/src/main/java/org/apache/uniffle/client/impl/ShuffleReadClientImpl.java b/client/src/main/java/org/apache/uniffle/client/impl/ShuffleReadClientImpl.java
index 3bddd3dc8..1690a49ec 100644
--- a/client/src/main/java/org/apache/uniffle/client/impl/ShuffleReadClientImpl.java
+++ b/client/src/main/java/org/apache/uniffle/client/impl/ShuffleReadClientImpl.java
@@ -314,8 +314,16 @@ public class ShuffleReadClientImpl implements ShuffleReadClient {
// mark block as processed
processedBlockIds.add(bs.getBlockId());
pendingBlockIds.removeLong(bs.getBlockId());
- // update the segment index to skip the unnecessary block in overlapping decompression mode
- segmentIndex += 1;
+
+ // update the segment index to skip the unnecessary block in overlapping decompression mode.
+ // In overlapping decompression mode, decompression tasks for the whole batch have already
+ // been submitted. If we skip a segment without removing the corresponding handler, the
+ // backpressure permits may never be released, which can block subsequent decompression.
+ if (decompressionWorker != null) {
+ decompressionWorker.get(batchIndex - 1, segmentIndex++);
+ } else {
+ segmentIndex += 1;
+ }
}
if (bs != null) {
diff --git a/client/src/test/java/org/apache/uniffle/client/impl/DecompressionWorkerTest.java b/client/src/test/java/org/apache/uniffle/client/impl/DecompressionWorkerTest.java
index 92b33b218..73036389c 100644
--- a/client/src/test/java/org/apache/uniffle/client/impl/DecompressionWorkerTest.java
+++ b/client/src/test/java/org/apache/uniffle/client/impl/DecompressionWorkerTest.java
@@ -35,6 +35,7 @@ import org.apache.uniffle.common.config.RssConf;
import static org.apache.uniffle.common.config.RssClientConf.COMPRESSION_TYPE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
public class DecompressionWorkerTest {
@@ -66,7 +67,9 @@ public class DecompressionWorkerTest {
}
Thread.sleep(10);
worker.get(0, maxSegments).getByteBuffer();
- assertEquals(1024 * maxSegments, worker.getPeekMemoryUsed());
+ // Peak memory is a runtime metric and may include one additional segment due to thread timing.
+ assertTrue(worker.getPeekMemoryUsed() <= 1024L * (maxSegments + 1));
+ assertTrue(worker.getPeekMemoryUsed() >= 1024L * maxSegments);
assertEquals(maxSegments, worker.getAvailablePermits());
}
diff --git a/client/src/test/java/org/apache/uniffle/client/impl/ShuffleReadClientImplTest.java b/client/src/test/java/org/apache/uniffle/client/impl/ShuffleReadClientImplTest.java
index 18b73a478..a37f0193b 100644
--- a/client/src/test/java/org/apache/uniffle/client/impl/ShuffleReadClientImplTest.java
+++ b/client/src/test/java/org/apache/uniffle/client/impl/ShuffleReadClientImplTest.java
@@ -52,6 +52,7 @@ import org.apache.uniffle.storage.HadoopTestBase;
import org.apache.uniffle.storage.handler.impl.HadoopShuffleWriteHandler;
import org.apache.uniffle.storage.util.StorageType;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -769,6 +770,47 @@ public class ShuffleReadClientImplTest extends HadoopTestBase {
readClient.close();
}
+ @ParameterizedTest
+ @MethodSource("clientBuilderProvider")
+ public void readTestSkipBlocksWithBackpressureDoesNotHang(
+ Supplier<ShuffleClientFactory.ReadClientBuilder> builderSupplier) throws Exception {
+ // This test is meaningful only when overlapping decompression is enabled.
+ // For non-overlapping mode, it should still pass and act as a regression guard.
+ String basePath = uniq(HDFS_URI + "clientReadTestSkipBlocksWithBackpressureDoesNotHang");
+ HadoopShuffleWriteHandler writeHandler =
+ new HadoopShuffleWriteHandler("appId", 0, 1, 1, basePath, ssi1.getId(), conf);
+
+ Map<Long, byte[]> expectedData = Maps.newHashMap();
+ Roaring64NavigableMap blockIdBitmap = Roaring64NavigableMap.bitmapOf();
+
+ // Write skipped blocks first to increase the chance of exhausting permits if permits are not
+ // released when skipping.
+ writeTestData(writeHandler, 20, 30, 1, 2, Maps.newHashMap(), blockIdBitmap);
+ writeTestData(writeHandler, 5, 30, 1, 0, expectedData, blockIdBitmap);
+
+ RssConf rssConf = new RssConf();
+ // Provide required base configs to avoid reader treating this as "prod mode" with empty values.
+ rssConf.set(RssClientConf.RSS_STORAGE_TYPE, StorageType.HDFS.name());
+ rssConf.setInteger(RssClientConf.RSS_READ_OVERLAPPING_DECOMPRESSION_FETCH_SECONDS_THRESHOLD, 1);
+ rssConf.setInteger(RssClientConf.RSS_READ_OVERLAPPING_DECOMPRESSION_MAX_CONCURRENT_SEGMENTS, 1);
+
+ // Expect only taskAttemptId=0 blocks; taskAttemptId=2 blocks will be skipped by the reader.
+ Roaring64NavigableMap taskIdBitmap = Roaring64NavigableMap.bitmapOf(0);
+ ShuffleReadClientImpl readClient =
+ builderSupplier
+ .get()
+ .partitionId(1)
+ .basePath(basePath)
+ .blockIdBitmap(blockIdBitmap)
+ .taskIdBitmap(taskIdBitmap)
+ .rssConf(rssConf)
+ .build();
+
+ assertDoesNotThrow(() -> TestUtils.validateResult(readClient, expectedData));
+ readClient.checkProcessedBlockIds();
+ readClient.close();
+ }
+
private void writeTestData(
HadoopShuffleWriteHandler writeHandler,
int num,