You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by et...@apache.org on 2023/02/13 07:56:04 UTC
[incubator-celeborn] branch CELEBORN-287 created (now 3db7de28e)
This is an automated email from the ASF dual-hosted git repository.
ethanfeng pushed a change to branch CELEBORN-287
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
at 3db7de28e [CELEBORN-287] Add snapshot about memory manager.
This branch includes the following new commits:
new 3db7de28e [CELEBORN-287] Add snapshot about memory manager.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-celeborn] 01/01: [CELEBORN-287] Add snapshot about memory manager.
Posted by et...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ethanfeng pushed a commit to branch CELEBORN-287
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
commit 3db7de28ebf16e8b83cc297f441916b49523964a
Author: Ethan Feng <et...@apache.org>
AuthorDate: Mon Feb 13 15:55:53 2023 +0800
[CELEBORN-287] Add snapshot about memory manager.
---
.../rss/common/network/server/MemoryTracker.java | 48 ++++++++++++++--------
.../aliyun/emr/rss/common/util/ThreadUtils.scala | 12 ++++--
2 files changed, 40 insertions(+), 20 deletions(-)
diff --git a/common/src/main/java/com/aliyun/emr/rss/common/network/server/MemoryTracker.java b/common/src/main/java/com/aliyun/emr/rss/common/network/server/MemoryTracker.java
index 81d89805e..4309681d9 100644
--- a/common/src/main/java/com/aliyun/emr/rss/common/network/server/MemoryTracker.java
+++ b/common/src/main/java/com/aliyun/emr/rss/common/network/server/MemoryTracker.java
@@ -27,7 +27,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.LongAdder;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.aliyun.emr.rss.common.util.ThreadUtils;
import io.netty.util.internal.PlatformDependent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,15 +45,14 @@ public class MemoryTracker {
private final long maxSortMemory;
private final List<MemoryTrackerListener> memoryTrackerListeners = new ArrayList<>();
- private final ScheduledExecutorService checkService = Executors
- .newSingleThreadScheduledExecutor(new ThreadFactoryBuilder()
- .setNameFormat("MemoryTracker-check-thread").build());
- private final ScheduledExecutorService reportService = Executors
- .newSingleThreadScheduledExecutor(new ThreadFactoryBuilder()
- .setNameFormat("MemoryTracker-report-thread").build());
- private final ExecutorService actionService = Executors
- .newSingleThreadScheduledExecutor(new ThreadFactoryBuilder()
- .setNameFormat("MemoryTracker-action-thread").build());
+ private final ScheduledExecutorService checkService =
+ ThreadUtils.newDaemonSingleThreadScheduledExecutor("MemoryTracker-check-thread");
+ private final ScheduledExecutorService reportService =
+ ThreadUtils.newDaemonSingleThreadScheduledExecutor("MemoryTracker-report-thread");
+ private final ExecutorService actionService =
+ ThreadUtils.newDaemonSingleThreadExecutorWithUncaughtExceptionHandler(
+ "MemoryTracker-action-thread", (t, e) ->
+ logger.error("memory-manager-actor thread {} has exception {}", t, e));
private AtomicLong nettyMemoryCounter = null;
private final AtomicLong sortMemoryCounter = new AtomicLong(0);
@@ -61,7 +60,7 @@ public class MemoryTracker {
private final LongAdder pausePushDataCounter = new LongAdder();
private final LongAdder pausePushDataAndReplicateCounter = new LongAdder();
private MemoryTrackerStat memoryTrackerStat = MemoryTrackerStat.resumeAll;
- private boolean underPressure;
+ private volatile boolean underPressure;
private int trimCount = 0;
public static MemoryTracker initialize(
@@ -115,8 +114,10 @@ public class MemoryTracker {
MemoryTrackerStat lastAction = memoryTrackerStat;
memoryTrackerStat = currentMemoryAction();
if (lastAction != memoryTrackerStat) {
+ logger.info("MemoryTrackerStat changed {} -> {}", lastAction, memoryTrackerStat);
if (memoryTrackerStat == MemoryTrackerStat.pausePushDataAndResumeReplicate) {
pausePushDataCounter.increment();
+ logger.info("Submit trigger pausePushDataAndResumeReplicate action");
actionService.submit(() -> {
logger.info("Trigger pausePushDataAndResumeReplicate action");
memoryTrackerListeners.forEach(memoryTrackerListener ->
@@ -127,6 +128,7 @@ public class MemoryTracker {
});
} else if (memoryTrackerStat == MemoryTrackerStat.pausePushDataAndReplicate) {
pausePushDataAndReplicateCounter.increment();
+ logger.info("Submit trigger pausePushDataAndReplicate action");
actionService.submit(() -> {
logger.info("Trigger pausePushDataAndReplicate action");
memoryTrackerListeners.forEach(memoryTrackerListener ->
@@ -136,6 +138,7 @@ public class MemoryTracker {
memoryTrackerListeners.forEach(MemoryTrackerListener::onTrim);
});
} else {
+ logger.info("Submit trigger resume action");
actionService.submit(() -> {
logger.info("Trigger resume action");
memoryTrackerListeners.forEach(memoryTrackerListener ->
@@ -200,26 +203,37 @@ public class MemoryTracker {
long memoryUsage = nettyMemoryCounter.get() + sortMemoryCounter.get();
boolean pausePushData = memoryUsage > pausePushDataThreshold;
boolean pauseReplication = memoryUsage > pauseReplicateThreshold;
+ MemoryTrackerStat statToReture = MemoryTrackerStat.resumeAll;
+ boolean resume = false;
if (pausePushData) {
underPressure = true;
if (pauseReplication) {
- return MemoryTrackerStat.pausePushDataAndReplicate;
+ statToReture= MemoryTrackerStat.pausePushDataAndReplicate;
} else {
- return MemoryTrackerStat.pausePushDataAndResumeReplicate;
+ statToReture= MemoryTrackerStat.pausePushDataAndResumeReplicate;
}
} else {
- boolean resume = memoryUsage < resumeThreshold;
+ resume = memoryUsage < resumeThreshold;
if (resume) {
underPressure = false;
- return MemoryTrackerStat.resumeAll;
+ statToReture= MemoryTrackerStat.resumeAll;
} else {
if (underPressure) {
- return MemoryTrackerStat.pausePushDataAndResumeReplicate;
+ statToReture= MemoryTrackerStat.pausePushDataAndResumeReplicate;
} else {
- return MemoryTrackerStat.resumeAll;
+ statToReture= MemoryTrackerStat.resumeAll;
}
}
}
+ logger.debug(
+ "memory action result stack {},{},{},{},{},{}",
+ memoryUsage,
+ pausePushData,
+ pauseReplication,
+ resume,
+ statToReture,
+ underPressure);
+ return statToReture;
}
public interface MemoryTrackerListener {
diff --git a/common/src/main/scala/com/aliyun/emr/rss/common/util/ThreadUtils.scala b/common/src/main/scala/com/aliyun/emr/rss/common/util/ThreadUtils.scala
index 5ff35a870..441efe980 100644
--- a/common/src/main/scala/com/aliyun/emr/rss/common/util/ThreadUtils.scala
+++ b/common/src/main/scala/com/aliyun/emr/rss/common/util/ThreadUtils.scala
@@ -18,7 +18,6 @@
package com.aliyun.emr.rss.common.util
import java.util.concurrent._
-
import scala.collection.TraversableLike
import scala.collection.generic.CanBuildFrom
import scala.concurrent.{Awaitable, ExecutionContext, ExecutionContextExecutor, Future}
@@ -26,11 +25,11 @@ import scala.concurrent.duration.{Duration, FiniteDuration}
import scala.concurrent.forkjoin.{ForkJoinPool => SForkJoinPool, ForkJoinWorkerThread => SForkJoinWorkerThread}
import scala.language.higherKinds
import scala.util.control.NonFatal
-
import com.google.common.util.concurrent.{MoreExecutors, ThreadFactoryBuilder}
-
import com.aliyun.emr.rss.common.exception.RssException
+import java.lang.Thread.UncaughtExceptionHandler
+
object ThreadUtils {
private val sameThreadExecutionContext =
@@ -106,6 +105,13 @@ object ThreadUtils {
executor
}
+ def newDaemonSingleThreadExecutorWithUncaughtExceptionHandler(
+ threadName: String,handler: UncaughtExceptionHandler): ExecutorService = {
+ val threadFactory = new ThreadFactoryBuilder().setDaemon(true).setNameFormat(
+ threadName).setUncaughtExceptionHandler(handler).build()
+ Executors.newSingleThreadExecutor(threadFactory)
+ }
+
/**
* Wrapper over ScheduledThreadPoolExecutor.
*/