You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2022/10/26 18:11:37 UTC
[ozone] branch master updated: HDDS-7349. Flaky integration test have memory leak for RatisDropwizardExports (#3858)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b5ecea61de HDDS-7349. Flaky integration test have memory leak for RatisDropwizardExports (#3858)
b5ecea61de is described below
commit b5ecea61defb727d2ed5eaaaf54c732dc52f6a23
Author: Sumit Agrawal <su...@gmail.com>
AuthorDate: Wed Oct 26 23:41:31 2022 +0530
HDDS-7349. Flaky integration test have memory leak for RatisDropwizardExports (#3858)
---
.../org/apache/hadoop/ozone/HddsDatanodeService.java | 2 +-
.../hdds/server/http/RatisDropwizardExports.java | 19 ++++++++++++++-----
.../hdds/scm/server/StorageContainerManager.java | 11 +++++++++--
.../java/org/apache/hadoop/ozone/om/OzoneManager.java | 2 +-
4 files changed, 25 insertions(+), 9 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index d36a178550..92dfbf237a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -213,7 +213,7 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
serviceRuntimeInfo.setStartTime();
RatisDropwizardExports.
- registerRatisMetricReporters(ratisMetricsMap);
+ registerRatisMetricReporters(ratisMetricsMap, () -> isStopped.get());
OzoneConfiguration.activate();
HddsServerUtil.initializeMetrics(conf, "HddsDatanode");
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/RatisDropwizardExports.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/RatisDropwizardExports.java
index 76e65a1d72..50d840a7ad 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/RatisDropwizardExports.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/RatisDropwizardExports.java
@@ -22,6 +22,7 @@ import io.prometheus.client.Collector;
import io.prometheus.client.CollectorRegistry;
import io.prometheus.client.dropwizard.DropwizardExports;
import io.prometheus.client.dropwizard.samplebuilder.DefaultSampleBuilder;
+import java.util.function.BooleanSupplier;
import org.apache.ratis.metrics.MetricRegistries;
import org.apache.ratis.metrics.MetricsReporting;
import org.apache.ratis.metrics.RatisMetricRegistry;
@@ -43,14 +44,15 @@ public class RatisDropwizardExports extends DropwizardExports {
}
public static void registerRatisMetricReporters(
- Map<String, RatisDropwizardExports> ratisMetricsMap) {
+ Map<String, RatisDropwizardExports> ratisMetricsMap,
+ BooleanSupplier checkStopped) {
//All the Ratis metrics (registered from now) will be published via JMX and
//via the prometheus exporter (used by the /prom servlet
MetricRegistries.global()
.addReporterRegistration(MetricsReporting.jmxReporter(),
MetricsReporting.stopJmxReporter());
MetricRegistries.global().addReporterRegistration(
- r1 -> registerDropwizard(r1, ratisMetricsMap),
+ r1 -> registerDropwizard(r1, ratisMetricsMap, checkStopped),
r2 -> deregisterDropwizard(r2, ratisMetricsMap));
}
@@ -69,12 +71,19 @@ public class RatisDropwizardExports extends DropwizardExports {
}
private static void registerDropwizard(RatisMetricRegistry registry,
- Map<String, RatisDropwizardExports> ratisMetricsMap) {
+ Map<String, RatisDropwizardExports> ratisMetricsMap,
+ BooleanSupplier checkStopped) {
+ if (checkStopped.getAsBoolean()) {
+ return;
+ }
+
RatisDropwizardExports rde = new RatisDropwizardExports(
registry.getDropWizardMetricRegistry());
- CollectorRegistry.defaultRegistry.register(rde);
String name = registry.getMetricRegistryInfo().getName();
- ratisMetricsMap.putIfAbsent(name, rde);
+ if (null == ratisMetricsMap.putIfAbsent(name, rde)) {
+ // new rde is added for the name, so need register
+ CollectorRegistry.defaultRegistry.register(rde);
+ }
}
private static void deregisterDropwizard(RatisMetricRegistry registry,
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 09844681ab..0b49a243a1 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -28,6 +28,7 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.cache.RemovalListener;
import com.google.protobuf.BlockingService;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsConfigKeys;
@@ -290,7 +291,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
// Used to keep track of pending replication and pending deletes for
// container replicas.
private ContainerReplicaPendingOps containerReplicaPendingOps;
-
+ private final AtomicBoolean isStopped = new AtomicBoolean(false);
+
/**
* Creates a new StorageContainerManager. Configuration will be
* updated with information on the actual listening addresses used
@@ -584,7 +586,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
clusterMap = new NetworkTopologyImpl(conf);
}
// This needs to be done before initializing Ratis.
- RatisDropwizardExports.registerRatisMetricReporters(ratisMetricsMap);
+ RatisDropwizardExports.registerRatisMetricReporters(ratisMetricsMap,
+ () -> isStopped.get());
if (configurator.getSCMHAManager() != null) {
scmHAManager = configurator.getSCMHAManager();
} else {
@@ -1522,6 +1525,10 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
*/
@Override
public void stop() {
+ if (isStopped.getAndSet(true)) {
+ LOG.info("Storage Container Manager is not running.");
+ return;
+ }
try {
if (containerBalancer.isBalancerRunning()) {
LOG.info("Stopping Container Balancer service.");
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 24a81d6779..7dcb71fd66 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -1967,7 +1967,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
if (omRatisServer == null) {
// This needs to be done before initializing Ratis.
RatisDropwizardExports.
- registerRatisMetricReporters(ratisMetricsMap);
+ registerRatisMetricReporters(ratisMetricsMap, () -> isStopped());
omRatisServer = OzoneManagerRatisServer.newOMRatisServer(
configuration, this, omNodeDetails, peerNodesMap,
secConfig, certClient, shouldBootstrap);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org