[GitHub] [incubator-celeborn] RexXiong commented on a diff in pull request #1457: [CELEBORN-552] Add HeartBeat between the client and worker to keep alive

["RexXiong (via GitHub)" <gi...@apache.org>]

RexXiong commented on code in PR #1457:
URL: https://github.com/apache/incubator-celeborn/pull/1457#discussion_r1184476200


##########
tests/flink-it/src/test/scala/org/apache/celeborn/tests/flink/WordCountTest.scala:
##########
@@ -22,21 +22,32 @@ import org.apache.flink.configuration.{ConfigConstants, Configuration, Execution
 import org.apache.flink.runtime.jobgraph.JobType
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
 import org.apache.flink.streaming.api.graph.GlobalStreamExchangeMode
-import org.scalatest.BeforeAndAfterAll
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
 import org.scalatest.funsuite.AnyFunSuite
 
 import org.apache.celeborn.common.internal.Logging
-import org.apache.celeborn.service.deploy.MiniClusterFeature;
+import org.apache.celeborn.plugin.flink.readclient.FlinkShuffleClientImpl
+import org.apache.celeborn.service.deploy.MiniClusterFeature
+import org.apache.celeborn.service.deploy.worker.memory.MemoryManager;
 
 class WordCountTest extends AnyFunSuite with Logging with MiniClusterFeature
-  with BeforeAndAfterAll {
+  with BeforeAndAfterAll with BeforeAndAfterEach {
+
+  override def beforeEach(): Unit = {
+    FlinkShuffleClientImpl.reset()
+  }
+
+  override def afterEach(): Unit = {
+    System.gc()

Review Comment:
   why we need add this？



##########
worker/src/main/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManager.java:
##########
@@ -150,7 +154,9 @@ private MemoryManager(CelebornConf conf) {
     pausePushDataThreshold = (long) (maxDirectorMemory * pausePushDataRatio);
     pauseReplicateThreshold = (long) (maxDirectorMemory * pauseReplicateRatio);
     resumeThreshold = (long) (maxDirectorMemory * resumeRatio);
+
     readBufferThreshold = (long) (maxDirectorMemory * readBufferRatio);
+    logger.info("readbufferthreshold:{}", readBufferThreshold);

Review Comment:
   ditto



##########
worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala:
##########
@@ -532,6 +546,8 @@ private[celeborn] class Worker(
       }
     }),
     WORKER_SHUTDOWN_PRIORITY)
+
+  def getPushFetchServerPort: (Int, Int) = (pushPort, fetchPort)

Review Comment:
   we can add annotation @VisibleForTesting for this



##########
worker/src/main/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManager.java:
##########
@@ -218,6 +224,7 @@ private MemoryManager(CelebornConf conf) {
 
     if (readBufferThreshold > 0) {
       // if read buffer threshold is zero means that there will be no map data partitions
+      logger.info("start ReadBufferDispatcher:");

Review Comment:
   Better not introduce other changes in this pr



##########
tests/flink-it/src/test/scala/org/apache/celeborn/tests/flink/WordCountTest.scala:
##########
@@ -22,21 +22,32 @@ import org.apache.flink.configuration.{ConfigConstants, Configuration, Execution
 import org.apache.flink.runtime.jobgraph.JobType
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
 import org.apache.flink.streaming.api.graph.GlobalStreamExchangeMode
-import org.scalatest.BeforeAndAfterAll
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
 import org.scalatest.funsuite.AnyFunSuite
 
 import org.apache.celeborn.common.internal.Logging
-import org.apache.celeborn.service.deploy.MiniClusterFeature;
+import org.apache.celeborn.plugin.flink.readclient.FlinkShuffleClientImpl
+import org.apache.celeborn.service.deploy.MiniClusterFeature
+import org.apache.celeborn.service.deploy.worker.memory.MemoryManager;
 
 class WordCountTest extends AnyFunSuite with Logging with MiniClusterFeature
-  with BeforeAndAfterAll {
+  with BeforeAndAfterAll with BeforeAndAfterEach {
+
+  override def beforeEach(): Unit = {
+    FlinkShuffleClientImpl.reset()
+  }
+
+  override def afterEach(): Unit = {
+    System.gc()
+  }
 
   override def beforeAll(): Unit = {
     logInfo("test initialized , setup rss mini cluster")
     val masterConf = Map(
       "celeborn.master.host" -> "localhost",
       "celeborn.master.port" -> "9097")
     val workerConf = Map("celeborn.master.endpoints" -> "localhost:9097")
+    MemoryManager.reset()

Review Comment:
   ditto



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@celeborn.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org