You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by av...@apache.org on 2020/09/02 16:54:30 UTC
[hadoop-ozone] 11/33: HDDS-3654. Let backgroundCreator create
pipeline for the support replication factors alternately (#984)
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch HDDS-3698-upgrade
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 3c7975dd6d84dd1cb3196ee61994ca6f29196986
Author: maobaolong <30...@qq.com>
AuthorDate: Tue Aug 25 20:58:37 2020 +0800
HDDS-3654. Let backgroundCreator create pipeline for the support replication factors alternately (#984)
---
.../scm/pipeline/BackgroundPipelineCreator.java | 33 ++++++++++++++--------
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 5 +++-
2 files changed, 26 insertions(+), 12 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
index f7f1d52..591acbc 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdds.scm.pipeline;
+import org.apache.commons.collections.iterators.LoopingIterator;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -26,6 +27,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -109,13 +112,15 @@ class BackgroundPipelineCreator {
ScmConfigKeys.OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE,
ScmConfigKeys.OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE_DEFAULT);
+ List<HddsProtos.ReplicationFactor> list =
+ new ArrayList<>();
for (HddsProtos.ReplicationFactor factor : HddsProtos.ReplicationFactor
.values()) {
if (skipCreation(factor, type, autoCreateFactorOne)) {
// Skip this iteration for creating pipeline
continue;
}
-
+ list.add(factor);
if (!pipelineManager.getSafeModeStatus()) {
try {
pipelineManager.scrubPipeline(type, factor);
@@ -123,21 +128,27 @@ class BackgroundPipelineCreator {
LOG.error("Error while scrubbing pipelines {}", e);
}
}
+ }
- while (true) {
- try {
- if (scheduler.isClosed()) {
- break;
- }
- pipelineManager.createPipeline(type, factor);
- } catch (IOException ioe) {
- break;
- } catch (Throwable t) {
- LOG.error("Error while creating pipelines", t);
+ LoopingIterator it = new LoopingIterator(list);
+ while (it.hasNext()) {
+ HddsProtos.ReplicationFactor factor =
+ (HddsProtos.ReplicationFactor) it.next();
+
+ try {
+ if (scheduler.isClosed()) {
break;
}
+ pipelineManager.createPipeline(type, factor);
+ } catch (IOException ioe) {
+ it.remove();
+ } catch (Throwable t) {
+ LOG.error("Error while creating pipelines", t);
+ it.remove();
}
}
+
isPipelineCreatorRunning.set(false);
+ LOG.debug("BackgroundPipelineCreator createPipelines finished.");
}
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
index 6a6d328..f05be76 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdds.scm.node;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.RATIS;
import java.io.File;
@@ -162,7 +163,9 @@ public class TestDeadNodeHandler {
LambdaTestUtils.await(120000, 1000,
() -> {
pipelineManager.triggerPipelineCreation();
- return pipelineManager.getPipelines(RATIS, THREE).size() == 3;
+ System.out.println(pipelineManager.getPipelines(RATIS, THREE).size());
+ System.out.println(pipelineManager.getPipelines(RATIS, ONE).size());
+ return pipelineManager.getPipelines(RATIS, THREE).size() > 3;
});
TestUtils.openAllRatisPipelines(pipelineManager);
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org