You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sa...@apache.org on 2022/05/10 02:46:58 UTC
[ozone] branch master updated: HDDS-6602. Incorrect logic in PipelineProvider.pickNodesNotUsed (#3318)
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e90a44de2f HDDS-6602. Incorrect logic in PipelineProvider.pickNodesNotUsed (#3318)
e90a44de2f is described below
commit e90a44de2f79a30dd6678101a1f607f5c68c5a84
Author: Symious <yi...@foxmail.com>
AuthorDate: Tue May 10 10:46:53 2022 +0800
HDDS-6602. Incorrect logic in PipelineProvider.pickNodesNotUsed (#3318)
---
.../apache/hadoop/hdds/scm/pipeline/PipelineProvider.java | 13 ++++++++++---
.../hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java | 4 ++--
2 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java
index 063ca76d39..01d3e71150 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java
@@ -86,13 +86,14 @@ public abstract class PipelineProvider<REPLICATION_CONFIG
List<DatanodeDetails> pickNodesNotUsed(REPLICATION_CONFIG replicationConfig,
long metadataSizeRequired, long dataSizeRequired) throws SCMException {
- List<DatanodeDetails> healthyDNs = pickNodesNotUsed(replicationConfig);
+ int nodesRequired = replicationConfig.getRequiredNodes();
+ List<DatanodeDetails> healthyDNs = pickAllNodesNotUsed(replicationConfig);
List<DatanodeDetails> healthyDNsWithSpace = healthyDNs.stream()
.filter(dn -> SCMCommonPlacementPolicy
.hasEnoughSpace(dn, metadataSizeRequired, dataSizeRequired))
+ .limit(nodesRequired)
.collect(Collectors.toList());
- int nodesRequired = replicationConfig.getRequiredNodes();
if (healthyDNsWithSpace.size() < nodesRequired) {
String msg = String.format("Unable to find enough nodes that meet the " +
"space requirement of %d bytes for metadata and %d bytes for " +
@@ -109,6 +110,13 @@ public abstract class PipelineProvider<REPLICATION_CONFIG
List<DatanodeDetails> pickNodesNotUsed(REPLICATION_CONFIG replicationConfig)
throws SCMException {
+ return pickAllNodesNotUsed(replicationConfig).stream()
+ .limit(replicationConfig.getRequiredNodes())
+ .collect(Collectors.toList());
+ }
+
+ List<DatanodeDetails> pickAllNodesNotUsed(
+ REPLICATION_CONFIG replicationConfig) throws SCMException {
Set<DatanodeDetails> dnsUsed = new HashSet<>();
stateManager.getPipelines(replicationConfig).stream().filter(
p -> p.getPipelineState().equals(Pipeline.PipelineState.OPEN) ||
@@ -121,7 +129,6 @@ public abstract class PipelineProvider<REPLICATION_CONFIG
.getNodes(NodeStatus.inServiceHealthy())
.parallelStream()
.filter(dn -> !dnsUsed.contains(dn))
- .limit(replicationConfig.getRequiredNodes())
.collect(Collectors.toList());
if (dns.size() < replicationConfig.getRequiredNodes()) {
String e = String
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
index c0ce8d4f81..904ccf0cd8 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
@@ -323,7 +323,7 @@ public class TestRatisPipelineProvider {
// Use large enough container or metadata sizes that no node will have
// enough space to hold one.
OzoneConfiguration largeContainerConf = new OzoneConfiguration();
- largeContainerConf.set(OZONE_SCM_CONTAINER_SIZE, "100TB");
+ largeContainerConf.set(OZONE_SCM_CONTAINER_SIZE, "300TB");
init(1, largeContainerConf);
for (ReplicationFactor factor: ReplicationFactor.values()) {
if (factor == ReplicationFactor.ZERO) {
@@ -339,7 +339,7 @@ public class TestRatisPipelineProvider {
}
OzoneConfiguration largeMetadataConf = new OzoneConfiguration();
- largeMetadataConf.set(OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN, "100TB");
+ largeMetadataConf.set(OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN, "300TB");
init(1, largeMetadataConf);
for (ReplicationFactor factor: ReplicationFactor.values()) {
if (factor == ReplicationFactor.ZERO) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org