You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aj...@apache.org on 2019/03/05 19:49:20 UTC
[hadoop] branch ozone-0.4 updated: HDDS-1193. Refactor
ContainerChillModeRule and DatanodeChillMode rule. (#534)
This is an automated email from the ASF dual-hosted git repository.
ajay pushed a commit to branch ozone-0.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/ozone-0.4 by this push:
new 4cf50e3 HDDS-1193. Refactor ContainerChillModeRule and DatanodeChillMode rule. (#534)
4cf50e3 is described below
commit 4cf50e3972c3062c53a4bff50da085181c36cd72
Author: Bharat Viswanadham <bh...@apache.org>
AuthorDate: Tue Mar 5 11:46:36 2019 -0800
HDDS-1193. Refactor ContainerChillModeRule and DatanodeChillMode rule. (#534)
* HDDS-1193. Refactor ContainerChillModeRule and DatanodeChillMode rule.
(cherry picked from commit 313e8b9f1344da376de2b0e899e78e384f37ff35)
---
.../hdds/scm/chillmode/ContainerChillModeRule.java | 43 +++++++++++++++-------
.../hdds/scm/chillmode/DataNodeChillModeRule.java | 41 +++++++++++++++------
.../hdds/scm/chillmode/SCMChillModeManager.java | 30 ++++++---------
.../hdds/scm/server/StorageContainerManager.java | 2 -
.../scm/chillmode/TestSCMChillModeManager.java | 14 ++-----
5 files changed, 75 insertions(+), 55 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java
index 9578532..17dd496 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java
@@ -29,12 +29,15 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer.NodeRegistrationContainerReport;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
/**
* Class defining Chill mode exit criteria for Containers.
*/
public class ContainerChillModeRule implements
- ChillModeExitRule<NodeRegistrationContainerReport> {
+ ChillModeExitRule<NodeRegistrationContainerReport>,
+ EventHandler<NodeRegistrationContainerReport> {
// Required cutoff % for containers with at least 1 reported replica.
private double chillModeCutoff;
@@ -68,9 +71,6 @@ public class ContainerChillModeRule implements
@Override
public boolean validate() {
- if (maxContainer == 0) {
- return true;
- }
return getCurrentContainerThreshold() >= chillModeCutoff;
}
@@ -84,10 +84,6 @@ public class ContainerChillModeRule implements
@Override
public void process(NodeRegistrationContainerReport reportsProto) {
- if (maxContainer == 0) {
- // No container to check.
- return;
- }
reportsProto.getReport().getReportsList().forEach(c -> {
if (containerMap.containsKey(c.getContainerID())) {
@@ -96,12 +92,33 @@ public class ContainerChillModeRule implements
}
}
});
- if(chillModeManager.getInChillMode()) {
- SCMChillModeManager.getLogger().info(
- "SCM in chill mode. {} % containers have at least one"
- + " reported replica.",
- (containerWithMinReplicas.get() / maxContainer) * 100);
+ }
+
+ @Override
+ public void onMessage(NodeRegistrationContainerReport
+ nodeRegistrationContainerReport, EventPublisher publisher) {
+
+ // TODO: when we have remove handlers, we can remove getInChillmode check
+
+ if (chillModeManager.getInChillMode()) {
+ if (validate()) {
+ return;
+ }
+
+ process(nodeRegistrationContainerReport);
+ if (chillModeManager.getInChillMode()) {
+ SCMChillModeManager.getLogger().info(
+ "SCM in chill mode. {} % containers have at least one"
+ + " reported replica.",
+ (containerWithMinReplicas.get() / maxContainer) * 100);
+ }
+
+ if (validate()) {
+ chillModeManager.validateChillModeExitRules(publisher);
+ }
+
}
+
}
@Override
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java
index 3950505..be99962 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java
@@ -25,13 +25,16 @@ import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer.NodeRegistrationContainerReport;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
/**
* Class defining Chill mode exit criteria according to number of DataNodes
* registered with SCM.
*/
public class DataNodeChillModeRule implements
- ChillModeExitRule<NodeRegistrationContainerReport> {
+ ChillModeExitRule<NodeRegistrationContainerReport>,
+ EventHandler<NodeRegistrationContainerReport> {
// Min DataNodes required to exit chill mode.
private int requiredDns;
@@ -62,17 +65,33 @@ public class DataNodeChillModeRule implements
@Override
public void process(NodeRegistrationContainerReport reportsProto) {
- if (requiredDns == 0) {
- // No dn check required.
- return;
- }
- if(chillModeManager.getInChillMode()) {
- registeredDnSet.add(reportsProto.getDatanodeDetails().getUuid());
- registeredDns = registeredDnSet.size();
- SCMChillModeManager.getLogger().info(
- "SCM in chill mode. {} DataNodes registered, {} required.",
- registeredDns, requiredDns);
+ registeredDnSet.add(reportsProto.getDatanodeDetails().getUuid());
+ registeredDns = registeredDnSet.size();
+
+ }
+
+ @Override
+ public void onMessage(NodeRegistrationContainerReport
+ nodeRegistrationContainerReport, EventPublisher publisher) {
+ // TODO: when we have remove handlers, we can remove getInChillmode check
+
+ if (chillModeManager.getInChillMode()) {
+ if (validate()) {
+ return;
+ }
+
+ process(nodeRegistrationContainerReport);
+
+ if (chillModeManager.getInChillMode()) {
+ SCMChillModeManager.getLogger().info(
+ "SCM in chill mode. {} DataNodes registered, {} required.",
+ registeredDns, requiredDns);
+ }
+
+ if (validate()) {
+ chillModeManager.validateChillModeExitRules(publisher);
+ }
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
index 1d97a57..8ced6d5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
@@ -28,9 +28,6 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.pipeline.RatisPipelineUtils;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer
- .NodeRegistrationContainerReport;
-import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.slf4j.Logger;
@@ -48,8 +45,7 @@ import org.slf4j.LoggerFactory;
* for reported containers and validates if cutoff threshold for
* containers is meet.
*/
-public class SCMChillModeManager implements
- EventHandler<NodeRegistrationContainerReport> {
+public class SCMChillModeManager {
private static final Logger LOG =
LoggerFactory.getLogger(SCMChillModeManager.class);
@@ -78,9 +74,16 @@ public class SCMChillModeManager implements
HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED,
HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED_DEFAULT);
if (isChillModeEnabled) {
- exitRules.put(CONT_EXIT_RULE,
- new ContainerChillModeRule(config, allContainers, this));
- exitRules.put(DN_EXIT_RULE, new DataNodeChillModeRule(config, this));
+ ContainerChillModeRule containerChillModeRule =
+ new ContainerChillModeRule(config, allContainers, this);
+ DataNodeChillModeRule dataNodeChillModeRule =
+ new DataNodeChillModeRule(config, this);
+ exitRules.put(CONT_EXIT_RULE, containerChillModeRule);
+ exitRules.put(DN_EXIT_RULE, dataNodeChillModeRule);
+ eventPublisher.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ containerChillModeRule);
+ eventPublisher.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ dataNodeChillModeRule);
if (conf.getBoolean(
HddsConfigKeys.HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK,
@@ -146,17 +149,6 @@ public class SCMChillModeManager implements
.scheduleFixedIntervalPipelineCreator(pipelineManager, config);
}
- @Override
- public void onMessage(
- NodeRegistrationContainerReport nodeRegistrationContainerReport,
- EventPublisher publisher) {
- if (getInChillMode()) {
- exitRules.get(CONT_EXIT_RULE).process(nodeRegistrationContainerReport);
- exitRules.get(DN_EXIT_RULE).process(nodeRegistrationContainerReport);
- validateChillModeExitRules(publisher);
- }
- }
-
public boolean getInChillMode() {
if (!isChillModeEnabled) {
return false;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index fc93235..c9446a5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -364,8 +364,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
replicationStatus.getChillModeStatusListener());
eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS,
(BlockManagerImpl) scmBlockManager);
- eventQueue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
- scmChillModeManager);
registerMXBean();
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
index faf8fee..957fe70 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
@@ -90,8 +90,7 @@ public class TestSCMChillModeManager {
}
scmChillModeManager = new SCMChillModeManager(
config, containers, null, queue);
- queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
- scmChillModeManager);
+
assertTrue(scmChillModeManager.getInChillMode());
queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
HddsTestUtils.createNodeRegistrationContainerReport(containers));
@@ -111,8 +110,7 @@ public class TestSCMChillModeManager {
}
scmChillModeManager = new SCMChillModeManager(
config, containers, null, queue);
- queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
- scmChillModeManager);
+
assertTrue(scmChillModeManager.getInChillMode());
testContainerThreshold(containers.subList(0, 25), 0.25);
@@ -167,8 +165,7 @@ public class TestSCMChillModeManager {
scmChillModeManager = new SCMChillModeManager(
config, containers, null, queue);
- queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
- scmChillModeManager);
+
assertTrue(scmChillModeManager.getInChillMode());
// When 10 CLOSED containers are reported by DNs, the computed container
@@ -192,8 +189,7 @@ public class TestSCMChillModeManager {
conf.setInt(HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE, numOfDns);
scmChillModeManager = new SCMChillModeManager(
conf, containers, null, queue);
- queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
- scmChillModeManager);
+
// Assert SCM is in Chill mode.
assertTrue(scmChillModeManager.getInChillMode());
@@ -256,8 +252,6 @@ public class TestSCMChillModeManager {
scmChillModeManager = new SCMChillModeManager(
config, containers, pipelineManager, queue);
- queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
- scmChillModeManager);
queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
HddsTestUtils.createNodeRegistrationContainerReport(containers));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org