You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by zh...@apache.org on 2023/05/17 09:46:16 UTC
[incubator-celeborn] branch branch-0.2 updated: [CELEBORN-459] Remove chunkTracker from FileManagedBuffers to avoid conflict with stream reuse (#1372) (#1493)
This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch branch-0.2
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/branch-0.2 by this push:
new 54c529747 [CELEBORN-459] Remove chunkTracker from FileManagedBuffers to avoid conflict with stream reuse (#1372) (#1493)
54c529747 is described below
commit 54c52974717586195f7bd7cddf257cc29c68461c
Author: Keyong Zhou <zh...@apache.org>
AuthorDate: Wed May 17 17:46:11 2023 +0800
[CELEBORN-459] Remove chunkTracker from FileManagedBuffers to avoid conflict with stream reuse (#1372) (#1493)
---
.../celeborn/common/meta/FileManagedBuffers.java | 27 ----------------------
.../common/network/server/ChunkStreamManager.java | 16 -------------
2 files changed, 43 deletions(-)
diff --git a/common/src/main/java/org/apache/celeborn/common/meta/FileManagedBuffers.java b/common/src/main/java/org/apache/celeborn/common/meta/FileManagedBuffers.java
index f36b270bc..72cb3ee59 100644
--- a/common/src/main/java/org/apache/celeborn/common/meta/FileManagedBuffers.java
+++ b/common/src/main/java/org/apache/celeborn/common/meta/FileManagedBuffers.java
@@ -18,7 +18,6 @@
package org.apache.celeborn.common.meta;
import java.io.File;
-import java.util.BitSet;
import java.util.List;
import org.apache.celeborn.common.network.buffer.FileSegmentManagedBuffer;
@@ -30,11 +29,8 @@ public class FileManagedBuffers {
private final long[] offsets;
private final int numChunks;
- private final BitSet chunkTracker;
private final TransportConf conf;
- private volatile boolean fullyRead = false;
-
public FileManagedBuffers(FileInfo fileInfo, TransportConf conf) {
file = fileInfo.getFile();
numChunks = fileInfo.numChunks();
@@ -48,8 +44,6 @@ public class FileManagedBuffers {
offsets = new long[1];
offsets[0] = 0;
}
- chunkTracker = new BitSet(numChunks);
- chunkTracker.clear();
this.conf = conf;
}
@@ -57,33 +51,12 @@ public class FileManagedBuffers {
return numChunks;
}
- public boolean hasAlreadyRead(int chunkIndex) {
- synchronized (chunkTracker) {
- return chunkTracker.get(chunkIndex);
- }
- }
-
public ManagedBuffer chunk(int chunkIndex, int offset, int len) {
- synchronized (chunkTracker) {
- chunkTracker.set(chunkIndex, true);
- }
// offset of the beginning of the chunk in the file
final long chunkOffset = offsets[chunkIndex];
final long chunkLength = offsets[chunkIndex + 1] - chunkOffset;
assert offset < chunkLength;
long length = Math.min(chunkLength - offset, len);
- if (len + offset >= chunkLength) {
- synchronized (chunkTracker) {
- chunkTracker.set(chunkIndex);
- }
- if (chunkTracker.cardinality() == numChunks) {
- fullyRead = true;
- }
- }
return new FileSegmentManagedBuffer(conf, file, chunkOffset + offset, length);
}
-
- public boolean isFullyRead() {
- return fullyRead;
- }
}
diff --git a/common/src/main/java/org/apache/celeborn/common/network/server/ChunkStreamManager.java b/common/src/main/java/org/apache/celeborn/common/network/server/ChunkStreamManager.java
index bc7162aff..ff9cd2317 100644
--- a/common/src/main/java/org/apache/celeborn/common/network/server/ChunkStreamManager.java
+++ b/common/src/main/java/org/apache/celeborn/common/network/server/ChunkStreamManager.java
@@ -86,24 +86,8 @@ public class ChunkStreamManager {
}
FileManagedBuffers buffers = state.buffers;
- if (buffers.hasAlreadyRead(chunkIndex)) {
- throw new IllegalStateException(
- String.format("Chunk %s for stream %s has already been read.", chunkIndex, streamId));
- }
ManagedBuffer nextChunk = buffers.chunk(chunkIndex, offset, len);
- if (state.buffers.isFullyRead()) {
- // Normally, when all chunks are returned to the client, the stream should be removed here.
- // But if there is a switch on the client side, it will not go here at this time, so we need
- // to remove the stream when the shuffle is expired, and release the unused buffer.
- logger.trace("Removing stream id {}", streamId);
- streams.remove(streamId);
- Set<Long> streamIds = shuffleStreamIds.get(state.shuffleKey);
- if (streamIds != null) {
- streamIds.remove(streamId);
- }
- }
-
return nextChunk;
}