You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by zh...@apache.org on 2023/05/17 09:46:16 UTC

[incubator-celeborn] branch branch-0.2 updated: [CELEBORN-459] Remove chunkTracker from FileManagedBuffers to avoid conflict with stream reuse (#1372) (#1493)

This is an automated email from the ASF dual-hosted git repository.

zhouky pushed a commit to branch branch-0.2
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/branch-0.2 by this push:
     new 54c529747 [CELEBORN-459] Remove chunkTracker from FileManagedBuffers to avoid conflict with stream reuse (#1372) (#1493)
54c529747 is described below

commit 54c52974717586195f7bd7cddf257cc29c68461c
Author: Keyong Zhou <zh...@apache.org>
AuthorDate: Wed May 17 17:46:11 2023 +0800

    [CELEBORN-459] Remove chunkTracker from FileManagedBuffers to avoid conflict with stream reuse (#1372) (#1493)
---
 .../celeborn/common/meta/FileManagedBuffers.java   | 27 ----------------------
 .../common/network/server/ChunkStreamManager.java  | 16 -------------
 2 files changed, 43 deletions(-)

diff --git a/common/src/main/java/org/apache/celeborn/common/meta/FileManagedBuffers.java b/common/src/main/java/org/apache/celeborn/common/meta/FileManagedBuffers.java
index f36b270bc..72cb3ee59 100644
--- a/common/src/main/java/org/apache/celeborn/common/meta/FileManagedBuffers.java
+++ b/common/src/main/java/org/apache/celeborn/common/meta/FileManagedBuffers.java
@@ -18,7 +18,6 @@
 package org.apache.celeborn.common.meta;
 
 import java.io.File;
-import java.util.BitSet;
 import java.util.List;
 
 import org.apache.celeborn.common.network.buffer.FileSegmentManagedBuffer;
@@ -30,11 +29,8 @@ public class FileManagedBuffers {
   private final long[] offsets;
   private final int numChunks;
 
-  private final BitSet chunkTracker;
   private final TransportConf conf;
 
-  private volatile boolean fullyRead = false;
-
   public FileManagedBuffers(FileInfo fileInfo, TransportConf conf) {
     file = fileInfo.getFile();
     numChunks = fileInfo.numChunks();
@@ -48,8 +44,6 @@ public class FileManagedBuffers {
       offsets = new long[1];
       offsets[0] = 0;
     }
-    chunkTracker = new BitSet(numChunks);
-    chunkTracker.clear();
     this.conf = conf;
   }
 
@@ -57,33 +51,12 @@ public class FileManagedBuffers {
     return numChunks;
   }
 
-  public boolean hasAlreadyRead(int chunkIndex) {
-    synchronized (chunkTracker) {
-      return chunkTracker.get(chunkIndex);
-    }
-  }
-
   public ManagedBuffer chunk(int chunkIndex, int offset, int len) {
-    synchronized (chunkTracker) {
-      chunkTracker.set(chunkIndex, true);
-    }
     // offset of the beginning of the chunk in the file
     final long chunkOffset = offsets[chunkIndex];
     final long chunkLength = offsets[chunkIndex + 1] - chunkOffset;
     assert offset < chunkLength;
     long length = Math.min(chunkLength - offset, len);
-    if (len + offset >= chunkLength) {
-      synchronized (chunkTracker) {
-        chunkTracker.set(chunkIndex);
-      }
-      if (chunkTracker.cardinality() == numChunks) {
-        fullyRead = true;
-      }
-    }
     return new FileSegmentManagedBuffer(conf, file, chunkOffset + offset, length);
   }
-
-  public boolean isFullyRead() {
-    return fullyRead;
-  }
 }
diff --git a/common/src/main/java/org/apache/celeborn/common/network/server/ChunkStreamManager.java b/common/src/main/java/org/apache/celeborn/common/network/server/ChunkStreamManager.java
index bc7162aff..ff9cd2317 100644
--- a/common/src/main/java/org/apache/celeborn/common/network/server/ChunkStreamManager.java
+++ b/common/src/main/java/org/apache/celeborn/common/network/server/ChunkStreamManager.java
@@ -86,24 +86,8 @@ public class ChunkStreamManager {
     }
 
     FileManagedBuffers buffers = state.buffers;
-    if (buffers.hasAlreadyRead(chunkIndex)) {
-      throw new IllegalStateException(
-          String.format("Chunk %s for stream %s has already been read.", chunkIndex, streamId));
-    }
     ManagedBuffer nextChunk = buffers.chunk(chunkIndex, offset, len);
 
-    if (state.buffers.isFullyRead()) {
-      // Normally, when all chunks are returned to the client, the stream should be removed here.
-      // But if there is a switch on the client side, it will not go here at this time, so we need
-      // to remove the stream when the shuffle is expired, and release the unused buffer.
-      logger.trace("Removing stream id {}", streamId);
-      streams.remove(streamId);
-      Set<Long> streamIds = shuffleStreamIds.get(state.shuffleKey);
-      if (streamIds != null) {
-        streamIds.remove(streamId);
-      }
-    }
-
     return nextChunk;
   }