You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ms...@apache.org on 2019/05/17 03:47:55 UTC
[hadoop] branch HDDS-1550 created (now a8a1727)
This is an automated email from the ASF dual-hosted git repository.
msingh pushed a change to branch HDDS-1550
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
at a8a1727 HDDS-1550. MiniOzoneChaosCluster is not shutting down all the threads during shutdown. Contributed by Mukul Kumar Singh.
This branch includes the following new commits:
new a8a1727 HDDS-1550. MiniOzoneChaosCluster is not shutting down all the threads during shutdown. Contributed by Mukul Kumar Singh.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/01: HDDS-1550. MiniOzoneChaosCluster is not shutting
down all the threads during shutdown. Contributed by Mukul Kumar Singh.
Posted by ms...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
msingh pushed a commit to branch HDDS-1550
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit a8a1727fbdf1c94fdbf4c57b098b0ce013c88fee
Author: Mukul Kumar Singh <ms...@apache.org>
AuthorDate: Fri May 17 09:17:27 2019 +0530
HDDS-1550. MiniOzoneChaosCluster is not shutting down all the threads during shutdown. Contributed by Mukul Kumar Singh.
---
.../ozone/container/common/interfaces/Handler.java | 5 +++++
.../ozone/container/common/report/ReportManager.java | 10 ++++++++++
.../common/transport/server/XceiverServerGrpc.java | 6 ++++++
.../ozone/container/keyvalue/KeyValueHandler.java | 5 +++++
.../ozone/container/ozoneimpl/OzoneContainer.java | 1 +
.../apache/hadoop/hdds/scm/node/SCMNodeManager.java | 1 +
.../hdds/scm/server/StorageContainerManager.java | 8 +++++++-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 2 +-
.../org/apache/hadoop/ozone/MiniOzoneClusterImpl.java | 19 +++++++++++--------
9 files changed, 47 insertions(+), 10 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
index a3bb34b..621dffb 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
@@ -121,6 +121,11 @@ public abstract class Handler {
throws IOException;
/**
+ * Stop the Handler.
+ */
+ public abstract void stop();
+
+ /**
* Marks the container for closing. Moves the container to CLOSING state.
*
* @param container container to update
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportManager.java
index 8097cd6..ef592a6 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportManager.java
@@ -23,10 +23,13 @@ import com.google.protobuf.GeneratedMessage;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
/**
* ReportManager is responsible for managing all the {@link ReportPublisher}
@@ -34,6 +37,8 @@ import java.util.concurrent.ScheduledExecutorService;
* which should be used for scheduling the reports.
*/
public final class ReportManager {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ReportManager.class);
private final StateContext context;
private final List<ReportPublisher> publishers;
@@ -71,6 +76,11 @@ public final class ReportManager {
*/
public void shutdown() {
executorService.shutdown();
+ try {
+ executorService.awaitTermination(1, TimeUnit.DAYS);
+ } catch (Exception e) {
+ LOG.error("failed to shutdown Report Manager", e);
+ }
}
/**
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index 6fe8fd4..3987cee 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -57,6 +57,7 @@ import java.net.SocketAddress;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
/**
* Creates a Grpc server endpoint that acts as the communication layer for
@@ -172,6 +173,11 @@ public final class XceiverServerGrpc extends XceiverServer {
public void stop() {
if (isStarted) {
server.shutdown();
+ try {
+ server.awaitTermination(1, TimeUnit.DAYS);
+ } catch (Exception e) {
+ LOG.error("failed to shutdown XceiverServerGrpc", e);
+ }
isStarted = false;
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 531fb02..0355973e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -159,6 +159,11 @@ public class KeyValueHandler extends Handler {
}
@Override
+ public void stop() {
+ blockDeletingService.shutdown();
+ }
+
+ @Override
public ContainerCommandResponseProto handle(
ContainerCommandRequestProto request, Container container,
DispatcherContext dispatcherContext) {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index f34334d..e04dd01 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -200,6 +200,7 @@ public class OzoneContainer {
stopContainerScrub();
writeChannel.stop();
readChannel.stop();
+ this.handlers.values().forEach(Handler::stop);
hddsDispatcher.shutdown();
volumeSet.shutdown();
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index 2ab7295..da45ced 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -184,6 +184,7 @@ public class SCMNodeManager implements NodeManager {
public void close() throws IOException {
unregisterMXBean();
metrics.unRegister();
+ nodeStateManager.close();
}
/**
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 8c4a514..a5e67e8 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -83,6 +83,7 @@ import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -205,6 +206,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
private final OzoneConfiguration configuration;
private final SafeModeHandler safeModeHandler;
private SCMContainerMetrics scmContainerMetrics;
+ private MetricsSystem ms;
/**
* Creates a new StorageContainerManager. Configuration will be
@@ -898,7 +900,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
buildRpcServerStartMessage(
"StorageContainerLocationProtocol RPC server",
getClientRpcAddress()));
- DefaultMetricsSystem.initialize("StorageContainerManager");
+ ms = DefaultMetricsSystem.initialize("StorageContainerManager");
commandWatcherLeaseManager.start();
getClientProtocolServer().start();
@@ -993,6 +995,10 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
metrics.unRegister();
}
+ if (ms != null) {
+ ms.stop();
+ }
+
unregisterMXBean();
if (scmContainerMetrics != null) {
scmContainerMetrics.unRegister();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
index 059af5a..855a97c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
@@ -203,7 +203,7 @@ public class MiniOzoneChaosCluster extends MiniOzoneClusterImpl {
1, TimeUnit.SECONDS);
conf.setTimeDuration(HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL, 1,
TimeUnit.SECONDS);
- conf.setInt(OzoneConfigKeys.OZONE_CONTAINER_CACHE_SIZE, 8);
+ conf.setInt(OzoneConfigKeys.OZONE_CONTAINER_CACHE_SIZE, 2);
}
@Override
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 9fbdad7..992a665 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -340,19 +340,20 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
ozoneManager.join();
}
+ if (!hddsDatanodes.isEmpty()) {
+ LOG.info("Shutting the HddsDatanodes");
+ hddsDatanodes.parallelStream()
+ .forEach(dn -> {
+ dn.stop();
+ dn.join();
+ });
+ }
+
if (scm != null) {
LOG.info("Stopping the StorageContainerManager");
scm.stop();
scm.join();
}
-
- if (!hddsDatanodes.isEmpty()) {
- LOG.info("Shutting the HddsDatanodes");
- for (HddsDatanodeService hddsDatanode : hddsDatanodes) {
- hddsDatanode.stop();
- hddsDatanode.join();
- }
- }
}
/**
@@ -568,6 +569,8 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "127.0.0.1:0");
conf.set(ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY, "127.0.0.1:0");
conf.setInt(ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY, numOfScmHandlers);
+ conf.set(HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT,
+ "3s");
configureSCMheartbeat();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org