You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sa...@apache.org on 2021/07/16 02:17:10 UTC
[ozone] branch master updated: HDDS-5337. Apply container space
check to Ratis factor one pipelines (#2354)
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e99a0eb HDDS-5337. Apply container space check to Ratis factor one pipelines (#2354)
e99a0eb is described below
commit e99a0ebc0ea216d50cd8d0fe4b4f07758102789d
Author: Ethan Rose <33...@users.noreply.github.com>
AuthorDate: Thu Jul 15 19:16:56 2021 -0700
HDDS-5337. Apply container space check to Ratis factor one pipelines (#2354)
---
.../apache/hadoop/hdds/scm/PlacementPolicy.java | 7 ++-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 73 +++++++++++++---------
.../hdds/scm/container/ReplicationManager.java | 2 +-
.../algorithms/SCMContainerPlacementCapacity.java | 8 ++-
.../algorithms/SCMContainerPlacementRackAware.java | 55 +++++++++-------
.../algorithms/SCMContainerPlacementRandom.java | 8 ++-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 49 ++++-----------
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 31 ++++++++-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 17 ++++-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 2 +-
.../hdds/scm/container/TestReplicationManager.java | 2 +-
.../algorithms/TestContainerPlacementFactory.java | 4 +-
.../TestSCMContainerPlacementCapacity.java | 2 +-
.../TestSCMContainerPlacementRackAware.java | 28 ++++-----
.../TestSCMContainerPlacementRandom.java | 2 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 71 +++++++++------------
.../scm/pipeline/TestRatisPipelineProvider.java | 44 ++++++++++++-
.../placement/TestContainerPlacement.java | 5 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 3 +-
19 files changed, 244 insertions(+), 169 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java
index 46671df..4985164 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java
@@ -35,13 +35,14 @@ public interface PlacementPolicy {
* @param excludedNodes - list of nodes to be excluded.
* @param favoredNodes - list of nodes preferred.
* @param nodesRequired - number of datanodes required.
- * @param sizeRequired - size required for the container or block.
+ * @param dataSizeRequired - size required for the container.
+ * @param metadataSizeRequired - size required for Ratis metadata.
* @return list of datanodes chosen.
* @throws IOException
*/
List<DatanodeDetails> chooseDatanodes(List<DatanodeDetails> excludedNodes,
- List<DatanodeDetails> favoredNodes, int nodesRequired, long sizeRequired)
- throws IOException;
+ List<DatanodeDetails> favoredNodes, int nodesRequired,
+ long metadataSizeRequired, long dataSizeRequired) throws IOException;
/**
* Given a list of datanode and the number of replicas required, return
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
index 7376098..bec0db3 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
@@ -24,7 +24,6 @@ import java.util.stream.Collectors;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.MetadataStorageReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
@@ -39,9 +38,6 @@ import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN_DEFAULT;
-
/**
* This policy implements a set of invariants which are common
* for all basic placement policies, acts as the repository of helper
@@ -120,14 +116,16 @@ public abstract class SCMCommonPlacementPolicy implements PlacementPolicy {
* @param excludedNodes - datanodes with existing replicas
* @param favoredNodes - list of nodes preferred.
* @param nodesRequired - number of datanodes required.
- * @param sizeRequired - size required for the container or block.
+ * @param dataSizeRequired - size required for the container.
+ * @param metadataSizeRequired - size required for Ratis metadata.
* @return list of datanodes chosen.
* @throws SCMException SCM exception.
*/
@Override
public List<DatanodeDetails> chooseDatanodes(
List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes,
- int nodesRequired, final long sizeRequired) throws SCMException {
+ int nodesRequired, long metadataSizeRequired, long dataSizeRequired)
+ throws SCMException {
List<DatanodeDetails> healthyNodes =
nodeManager.getNodes(NodeStatus.inServiceHealthy());
if (excludedNodes != null) {
@@ -149,19 +147,30 @@ public abstract class SCMCommonPlacementPolicy implements PlacementPolicy {
throw new SCMException(msg,
SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
}
- List<DatanodeDetails> healthyList = healthyNodes.stream().filter(d ->
- hasEnoughSpace(d, sizeRequired)).collect(Collectors.toList());
- if (healthyList.size() < nodesRequired) {
- msg = String.format("Unable to find enough nodes that meet the space " +
- "requirement of %d bytes in healthy node set." +
- " Nodes required: %d Found: %d",
- sizeRequired, nodesRequired, healthyList.size());
+ return filterNodesWithSpace(healthyNodes, nodesRequired,
+ metadataSizeRequired, dataSizeRequired);
+ }
+
+ public List<DatanodeDetails> filterNodesWithSpace(List<DatanodeDetails> nodes,
+ int nodesRequired, long metadataSizeRequired, long dataSizeRequired)
+ throws SCMException {
+ List<DatanodeDetails> nodesWithSpace = nodes.stream().filter(d ->
+ hasEnoughSpace(d, metadataSizeRequired, dataSizeRequired))
+ .collect(Collectors.toList());
+
+ if (nodesWithSpace.size() < nodesRequired) {
+ String msg = String.format("Unable to find enough nodes that meet the " +
+ "space requirement of %d bytes for metadata and %d bytes for " +
+ "data in healthy node set. Required %d. Found %d.",
+ metadataSizeRequired, dataSizeRequired, nodesRequired,
+ nodesWithSpace.size());
LOG.error(msg);
throw new SCMException(msg,
SCMException.ResultCodes.FAILED_TO_FIND_NODES_WITH_SPACE);
}
- return healthyList;
+
+ return nodesWithSpace;
}
/**
@@ -170,36 +179,40 @@ public abstract class SCMCommonPlacementPolicy implements PlacementPolicy {
* @param datanodeDetails DatanodeDetails
* @return true if we have enough space.
*/
- public boolean hasEnoughSpace(DatanodeDetails datanodeDetails,
- long sizeRequired) {
+ public static boolean hasEnoughSpace(DatanodeDetails datanodeDetails,
+ long metadataSizeRequired, long dataSizeRequired) {
Preconditions.checkArgument(datanodeDetails instanceof DatanodeInfo);
- long metaSizeRequired = (long) conf.getStorageSize(
- OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN,
- OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN_DEFAULT,
- StorageUnit.BYTES);
-
boolean enoughForData = false;
boolean enoughForMeta = false;
DatanodeInfo datanodeInfo = (DatanodeInfo) datanodeDetails;
- for (StorageReportProto reportProto : datanodeInfo.getStorageReports()) {
- if (reportProto.getRemaining() > sizeRequired) {
- enoughForData = true;
- break;
+
+ if (dataSizeRequired > 0) {
+ for (StorageReportProto reportProto : datanodeInfo.getStorageReports()) {
+ if (reportProto.getRemaining() > dataSizeRequired) {
+ enoughForData = true;
+ break;
+ }
}
+ } else {
+ enoughForData = true;
}
if (!enoughForData) {
return false;
}
- for (MetadataStorageReportProto reportProto
- : datanodeInfo.getMetadataStorageReports()) {
- if (reportProto.getRemaining() > metaSizeRequired) {
- enoughForMeta = true;
- break;
+ if (metadataSizeRequired > 0) {
+ for (MetadataStorageReportProto reportProto
+ : datanodeInfo.getMetadataStorageReports()) {
+ if (reportProto.getRemaining() > metadataSizeRequired) {
+ enoughForMeta = true;
+ break;
+ }
}
+ } else {
+ enoughForMeta = true;
}
return enoughForData && enoughForMeta;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
index 2310848..d1218a8 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
@@ -755,7 +755,7 @@ public class ReplicationManager implements MetricsSource, SCMService {
excludeList.addAll(replicationInFlight);
final List<DatanodeDetails> selectedDatanodes = containerPlacement
.chooseDatanodes(excludeList, null, replicasNeeded,
- container.getUsedBytes());
+ 0, container.getUsedBytes());
if (repDelta > 0) {
LOG.info("Container {} is under replicated. Expected replica count" +
" is {}, but found {}.", id, replicationFactor,
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java
index 47806ed..afb9e1b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java
@@ -92,16 +92,18 @@ public final class SCMContainerPlacementCapacity
* @param excludedNodes - list of the datanodes to exclude.
* @param favoredNodes - list of nodes preferred.
* @param nodesRequired - number of datanodes required.
- * @param sizeRequired - size required for the container or block.
+ * @param dataSizeRequired - size required for the container.
+ * @param metadataSizeRequired - size required for Ratis metadata.
* @return List of datanodes.
* @throws SCMException SCMException
*/
@Override
public List<DatanodeDetails> chooseDatanodes(
List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes,
- final int nodesRequired, final long sizeRequired) throws SCMException {
+ final int nodesRequired,
+ long metadataSizeRequired, long dataSizeRequired) throws SCMException {
List<DatanodeDetails> healthyNodes = super.chooseDatanodes(excludedNodes,
- favoredNodes, nodesRequired, sizeRequired);
+ favoredNodes, nodesRequired, metadataSizeRequired, dataSizeRequired);
if (healthyNodes.size() == nodesRequired) {
return healthyNodes;
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java
index d3eb136..b5c59b2 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java
@@ -91,14 +91,16 @@ public final class SCMContainerPlacementRackAware
* depends on whether the nodes meets the allocator's
* requirement.
* @param nodesRequired - number of datanodes required.
- * @param sizeRequired - size required for the container or block.
+ * @param dataSizeRequired - size required for the container.
+ * @param metadataSizeRequired - size required for Ratis metadata.
* @return List of datanodes.
* @throws SCMException SCMException
*/
@Override
public List<DatanodeDetails> chooseDatanodes(
List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes,
- int nodesRequired, final long sizeRequired) throws SCMException {
+ int nodesRequired, long metadataSizeRequired, long dataSizeRequired)
+ throws SCMException {
Preconditions.checkArgument(nodesRequired > 0);
metrics.incrDatanodeRequestCount(nodesRequired);
int datanodeCount = networkTopology.getNumOfLeafNode(NetConstants.ROOT);
@@ -131,7 +133,8 @@ public final class SCMContainerPlacementRackAware
firstNode = favoredNode;
favorIndex++;
} else {
- firstNode = chooseNode(null, null, sizeRequired);
+ firstNode = chooseNode(null, null, metadataSizeRequired,
+ dataSizeRequired);
}
chosenNodes.add(firstNode);
nodesRequired--;
@@ -148,7 +151,8 @@ public final class SCMContainerPlacementRackAware
secondNode = favoredNode;
favorIndex++;
} else {
- secondNode = chooseNode(chosenNodes, firstNode, sizeRequired);
+ secondNode = chooseNode(chosenNodes, firstNode, metadataSizeRequired,
+ dataSizeRequired);
}
chosenNodes.add(secondNode);
nodesRequired--;
@@ -157,8 +161,8 @@ public final class SCMContainerPlacementRackAware
}
// choose remaining datanodes on different rack as first and second
- return chooseNodes(null, chosenNodes, mutableFavoredNodes, favorIndex,
- nodesRequired, sizeRequired);
+ return chooseNodes(null, chosenNodes, mutableFavoredNodes,
+ favorIndex, nodesRequired, metadataSizeRequired, dataSizeRequired);
} else {
List<Node> mutableExcludedNodes = new ArrayList<>();
mutableExcludedNodes.addAll(excludedNodes);
@@ -176,7 +180,7 @@ public final class SCMContainerPlacementRackAware
favorIndex++;
} else {
firstNode = chooseNode(mutableExcludedNodes, excludedNodes.get(0),
- sizeRequired);
+ metadataSizeRequired, dataSizeRequired);
}
chosenNodes.add(firstNode);
nodesRequired--;
@@ -185,7 +189,7 @@ public final class SCMContainerPlacementRackAware
}
// choose remaining nodes on different racks
return chooseNodes(null, chosenNodes, mutableFavoredNodes, favorIndex,
- nodesRequired, sizeRequired);
+ nodesRequired, metadataSizeRequired, dataSizeRequired);
}
// case 2: two or more excluded nodes, if these two nodes are
// in the same rack, then choose nodes on different racks, otherwise,
@@ -197,7 +201,8 @@ public final class SCMContainerPlacementRackAware
excludedNodes.get(i), excludedNodes.get(j))) {
// choose remaining nodes on different racks
return chooseNodes(mutableExcludedNodes, chosenNodes,
- mutableFavoredNodes, favorIndex, nodesRequired, sizeRequired);
+ mutableFavoredNodes, favorIndex, nodesRequired,
+ metadataSizeRequired, dataSizeRequired);
}
}
}
@@ -211,7 +216,8 @@ public final class SCMContainerPlacementRackAware
favorIndex++;
} else {
secondNode =
- chooseNode(chosenNodes, mutableExcludedNodes.get(0), sizeRequired);
+ chooseNode(chosenNodes, mutableExcludedNodes.get(0),
+ metadataSizeRequired, dataSizeRequired);
}
chosenNodes.add(secondNode);
mutableExcludedNodes.add(secondNode);
@@ -221,7 +227,7 @@ public final class SCMContainerPlacementRackAware
}
// choose remaining nodes on different racks
return chooseNodes(mutableExcludedNodes, chosenNodes, mutableFavoredNodes,
- favorIndex, nodesRequired, sizeRequired);
+ favorIndex, nodesRequired, metadataSizeRequired, dataSizeRequired);
}
}
@@ -239,12 +245,13 @@ public final class SCMContainerPlacementRackAware
* @param excludedNodes - list of the datanodes to excluded. Can be null.
* @param affinityNode - the chosen nodes should be on the same rack as
* affinityNode. Can be null.
- * @param sizeRequired - size required for the container or block.
+ * @param dataSizeRequired - size required for the container.
+ * @param metadataSizeRequired - size required for Ratis metadata.
* @return List of chosen datanodes.
* @throws SCMException SCMException
*/
private Node chooseNode(List<Node> excludedNodes, Node affinityNode,
- long sizeRequired) throws SCMException {
+ long metadataSizeRequired, long dataSizeRequired) throws SCMException {
int ancestorGen = RACK_LEVEL;
int maxRetry = MAX_RETRY;
List<String> excludedNodesForCapacity = null;
@@ -285,9 +292,11 @@ public final class SCMContainerPlacementRackAware
datanodeDetails);
} else {
if (datanodeInfo.getNodeStatus().isNodeWritable() &&
- (super.hasEnoughSpace(datanodeInfo, sizeRequired))) {
- LOG.debug("Datanode {} is chosen. Required storage size is {} bytes",
- node, sizeRequired);
+ (hasEnoughSpace(datanodeInfo, metadataSizeRequired,
+ dataSizeRequired))) {
+ LOG.debug("Datanode {} is chosen. Required metadata size is {} and " +
+ "required data size is {}",
+ node.toString(), metadataSizeRequired, dataSizeRequired);
metrics.incrDatanodeChooseSuccessCount();
if (isFallbacked) {
metrics.incrDatanodeChooseFallbackCount();
@@ -300,7 +309,8 @@ public final class SCMContainerPlacementRackAware
if (maxRetry == 0) {
// avoid the infinite loop
String errMsg = "No satisfied datanode to meet the space constrains. "
- + " sizeRequired: " + sizeRequired;
+ + "metadatadata size required: " + metadataSizeRequired +
+ " data size required: " + dataSizeRequired;
LOG.info(errMsg);
throw new SCMException(errMsg, null);
}
@@ -323,16 +333,16 @@ public final class SCMContainerPlacementRackAware
* are chosen depends on whether they meet the constrains.
* Can be null.
* @param favorIndex - the node index of favoredNodes which is not chosen yet.
- * @param sizeRequired - size required for the container or block.
* @param nodesRequired - number of datanodes required.
- * @param sizeRequired - size required for the container or block.
+ * @param dataSizeRequired - size required for the container.
+ * @param metadataSizeRequired - size required for Ratis metadata.
* @return List of chosen datanodes.
* @throws SCMException SCMException
*/
private List<DatanodeDetails> chooseNodes(List<Node> excludedNodes,
List<Node> chosenNodes, List<DatanodeDetails> favoredNodes,
- int favorIndex, int nodesRequired, long sizeRequired)
- throws SCMException {
+ int favorIndex, int nodesRequired, long metadataSizeRequired,
+ long dataSizeRequired) throws SCMException {
Preconditions.checkArgument(chosenNodes != null);
List<Node> excludedNodeList = excludedNodes != null ?
excludedNodes : chosenNodes;
@@ -346,7 +356,8 @@ public final class SCMContainerPlacementRackAware
chosenNode = favoredNode;
favorIndex++;
} else {
- chosenNode = chooseNode(excludedNodeList, null, sizeRequired);
+ chosenNode = chooseNode(excludedNodeList, null,
+ metadataSizeRequired, dataSizeRequired);
}
excludedNodeList.add(chosenNode);
if (excludedNodeList != chosenNodes) {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java
index b30f767..27650d1 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java
@@ -64,17 +64,19 @@ public final class SCMContainerPlacementRandom extends SCMCommonPlacementPolicy
* @param excludedNodes - list of the datanodes to exclude.
* @param favoredNodes - list of nodes preferred.
* @param nodesRequired - number of datanodes required.
- * @param sizeRequired - size required for the container or block.
+ * @param dataSizeRequired - size required for the container.
+ * @param metadataSizeRequired - size required for Ratis metadata.
* @return List of Datanodes.
* @throws SCMException SCMException
*/
@Override
public List<DatanodeDetails> chooseDatanodes(
List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes,
- final int nodesRequired, final long sizeRequired) throws SCMException {
+ final int nodesRequired,
+ long metadataSizeRequired, long dataSizeRequired) throws SCMException {
List<DatanodeDetails> healthyNodes =
super.chooseDatanodes(excludedNodes, favoredNodes, nodesRequired,
- sizeRequired);
+ metadataSizeRequired, dataSizeRequired);
if (healthyNodes.size() == nodesRequired) {
return healthyNodes;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
index b9aba12..d330df6 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
@@ -22,7 +22,6 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -41,9 +40,6 @@ import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN_DEFAULT;
-
/**
* Pipeline placement policy that choose datanodes based on load balancing
* and network topology to supply pipeline creation.
@@ -126,7 +122,8 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
/**
* Filter out viable nodes based on
* 1. nodes that are healthy
- * 2. nodes that are not too heavily engaged in other pipelines
+ * 2. nodes that have enough space
+ * 3. nodes that are not too heavily engaged in other pipelines
* The results are sorted based on pipeline count of each node.
*
* @param excludedNodes - excluded nodes
@@ -135,11 +132,14 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
* @throws SCMException when viable nodes are not enough in numbers
*/
List<DatanodeDetails> filterViableNodes(
- List<DatanodeDetails> excludedNodes, int nodesRequired)
+ List<DatanodeDetails> excludedNodes, int nodesRequired,
+ long metadataSizeRequired, long dataSizeRequired)
throws SCMException {
// get nodes in HEALTHY state
List<DatanodeDetails> healthyNodes =
nodeManager.getNodes(NodeStatus.inServiceHealthy());
+ healthyNodes = filterNodesWithSpace(healthyNodes, nodesRequired,
+ metadataSizeRequired, dataSizeRequired);
boolean multipleRacks = multipleRacksAvailable(healthyNodes);
if (excludedNodes != null) {
healthyNodes.removeAll(excludedNodes);
@@ -156,36 +156,11 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
}
- long sizeRequired = (long) conf.getStorageSize(
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT,
- StorageUnit.BYTES);
-
- long metaSizeRequired = (long) conf.getStorageSize(
- OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN,
- OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN_DEFAULT,
- StorageUnit.BYTES);
-
- // filter nodes that don't even have space for one container
- List<DatanodeDetails> canHoldList = healthyNodes.stream().filter(d ->
- hasEnoughSpace(d, sizeRequired)).collect(Collectors.toList());
-
- if (canHoldList.size() < nodesRequired) {
- msg = String.format("Pipeline creation failed due to no sufficient" +
- " healthy datanodes with enough space for container data and " +
- "metadata. Required %d. Found %d. Container data required %d, " +
- "metadata required %d.",
- nodesRequired, canHoldList.size(), sizeRequired, metaSizeRequired);
- LOG.warn(msg);
- throw new SCMException(msg,
- SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
- }
-
// filter nodes that meet the size and pipeline engagement criteria.
// Pipeline placement doesn't take node space left into account.
// Sort the DNs by pipeline load.
// TODO check if sorting could cause performance issue: HDDS-3466.
- List<DatanodeDetails> healthyList = canHoldList.stream()
+ List<DatanodeDetails> healthyList = healthyNodes.stream()
.map(d ->
new DnWithPipelines(d, currentPipelineCount(d, nodesRequired)))
.filter(d ->
@@ -247,18 +222,20 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
* @param excludedNodes - excluded nodes
* @param favoredNodes - list of nodes preferred.
* @param nodesRequired - number of datanodes required.
- * @param sizeRequired - size required for the container or block.
+ * @param dataSizeRequired - size required for the container.
+ * @param metadataSizeRequired - size required for Ratis metadata.
* @return a list of chosen datanodeDetails
* @throws SCMException when chosen nodes are not enough in numbers
*/
@Override
public List<DatanodeDetails> chooseDatanodes(
List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes,
- int nodesRequired, final long sizeRequired) throws SCMException {
+ int nodesRequired, long metadataSizeRequired, long dataSizeRequired)
+ throws SCMException {
// Get a list of viable nodes based on criteria
// and make sure excludedNodes are excluded from list.
- List<DatanodeDetails> healthyNodes =
- filterViableNodes(excludedNodes, nodesRequired);
+ List<DatanodeDetails> healthyNodes = filterViableNodes(excludedNodes,
+ nodesRequired, metadataSizeRequired, dataSizeRequired);
// Randomly picks nodes when all nodes are equal or factor is ONE.
// This happens when network topology is absent or
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java
index 50bc0e1..92b30b6 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java
@@ -26,16 +26,20 @@ import java.util.stream.Collectors;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.SCMCommonPlacementPolicy;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.node.NodeStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Interface for creating pipelines.
*/
public abstract class PipelineProvider<REPLICATION_CONFIG
extends ReplicationConfig> {
-
+ private static final Logger LOG =
+ LoggerFactory.getLogger(PipelineProvider.class);
private final NodeManager nodeManager;
private final StateManager stateManager;
@@ -70,7 +74,30 @@ public abstract class PipelineProvider<REPLICATION_CONFIG
protected abstract void shutdown();
- List<DatanodeDetails> pickNodesNeverUsed(REPLICATION_CONFIG replicationConfig)
+ List<DatanodeDetails> pickNodesNotUsed(REPLICATION_CONFIG replicationConfig,
+ long metadataSizeRequired, long dataSizeRequired) throws SCMException {
+ List<DatanodeDetails> healthyDNs = pickNodesNotUsed(replicationConfig);
+ List<DatanodeDetails> healthyDNsWithSpace = healthyDNs.stream()
+ .filter(dn -> SCMCommonPlacementPolicy
+ .hasEnoughSpace(dn, metadataSizeRequired, dataSizeRequired))
+ .collect(Collectors.toList());
+
+ int nodesRequired = replicationConfig.getRequiredNodes();
+ if (healthyDNsWithSpace.size() < nodesRequired) {
+ String msg = String.format("Unable to find enough nodes that meet the " +
+ "space requirement of %d bytes for metadata and %d bytes for " +
+ "data in healthy node set. Nodes required: %d Found: %d",
+ metadataSizeRequired, dataSizeRequired, nodesRequired,
+ healthyDNsWithSpace.size());
+ LOG.error(msg);
+ throw new SCMException(msg,
+ SCMException.ResultCodes.FAILED_TO_FIND_NODES_WITH_SPACE);
+ }
+
+ return healthyDNsWithSpace;
+ }
+
+ List<DatanodeDetails> pickNodesNotUsed(REPLICATION_CONFIG replicationConfig)
throws SCMException {
Set<DatanodeDetails> dnsUsed = new HashSet<>();
stateManager.getPipelines(replicationConfig).stream().filter(
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
index 485ea83..9cc3954 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -59,6 +60,8 @@ public class RatisPipelineProvider
private int maxPipelinePerDatanode;
private final LeaderChoosePolicy leaderChoosePolicy;
private final SCMContext scmContext;
+ private final long containerSizeBytes;
+ private final long minRatisVolumeSizeBytes;
@VisibleForTesting
public RatisPipelineProvider(NodeManager nodeManager,
@@ -78,6 +81,14 @@ public class RatisPipelineProvider
String dnLimit = conf.get(ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT);
this.maxPipelinePerDatanode = dnLimit == null ? 0 :
Integer.parseInt(dnLimit);
+ this.containerSizeBytes = (long) this.conf.getStorageSize(
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT,
+ StorageUnit.BYTES);
+ this.minRatisVolumeSizeBytes = (long) this.conf.getStorageSize(
+ ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN,
+ ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN_DEFAULT,
+ StorageUnit.BYTES);
try {
leaderChoosePolicy = LeaderChoosePolicyFactory
.getPolicy(conf, nodeManager, stateManager);
@@ -135,11 +146,13 @@ public class RatisPipelineProvider
replicationConfig.getReplicationFactor();
switch (factor) {
case ONE:
- dns = pickNodesNeverUsed(replicationConfig);
+ dns = pickNodesNotUsed(replicationConfig, minRatisVolumeSizeBytes,
+ containerSizeBytes);
break;
case THREE:
dns = placementPolicy.chooseDatanodes(null,
- null, factor.getNumber(), 0);
+ null, factor.getNumber(), minRatisVolumeSizeBytes,
+ containerSizeBytes);
break;
default:
throw new IllegalStateException("Unknown factor: " + factor.name());
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java
index 98b700e..147f773 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java
@@ -41,7 +41,7 @@ public class SimplePipelineProvider
@Override
public Pipeline create(StandaloneReplicationConfig replicationConfig)
throws IOException {
- List<DatanodeDetails> dns = pickNodesNeverUsed(replicationConfig);
+ List<DatanodeDetails> dns = pickNodesNotUsed(replicationConfig);
if (dns.size() < replicationConfig.getRequiredNodes()) {
String e = String
.format("Cannot create pipeline of factor %d using %d nodes.",
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
index 3bdc6ab..5401d0a 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
@@ -130,7 +130,7 @@ public class TestReplicationManager {
Mockito.when(containerPlacementPolicy.chooseDatanodes(
Mockito.any(),
Mockito.any(),
- Mockito.anyInt(), Mockito.anyLong()))
+ Mockito.anyInt(), Mockito.anyLong(), Mockito.anyLong()))
.thenAnswer(invocation -> {
int count = (int) invocation.getArguments()[2];
return IntStream.range(0, count)
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
index c37da63..1f0853d 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
@@ -148,7 +148,7 @@ public class TestContainerPlacementFactory {
int nodeNum = 3;
List<DatanodeDetails> datanodeDetails =
- policy.chooseDatanodes(null, null, nodeNum, 15);
+ policy.chooseDatanodes(null, null, nodeNum, 15, 15);
Assert.assertEquals(nodeNum, datanodeDetails.size());
Assert.assertTrue(cluster.isSameParent(datanodeDetails.get(0),
datanodeDetails.get(1)));
@@ -172,7 +172,7 @@ public class TestContainerPlacementFactory {
@Override
public List<DatanodeDetails> chooseDatanodes(
List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes,
- int nodesRequired, long sizeRequired) {
+ int nodesRequired, long metadataSizeRequired, long dataSizeRequired) {
return null;
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java
index 411e59f..ae30402 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java
@@ -127,7 +127,7 @@ public class TestSCMContainerPlacementCapacity {
//when
List<DatanodeDetails> datanodeDetails = scmContainerPlacementRandom
- .chooseDatanodes(existingNodes, null, 1, 15);
+ .chooseDatanodes(existingNodes, null, 1, 15, 15);
//then
Assert.assertEquals(1, datanodeDetails.size());
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java
index 87e77ff..376752b 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java
@@ -199,19 +199,19 @@ public class TestSCMContainerPlacementRackAware {
// 1 replica
int nodeNum = 1;
List<DatanodeDetails> datanodeDetails =
- policy.chooseDatanodes(null, null, nodeNum, 15);
+ policy.chooseDatanodes(null, null, nodeNum, 0, 15);
Assert.assertEquals(nodeNum, datanodeDetails.size());
// 2 replicas
nodeNum = 2;
- datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 15);
+ datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 0, 15);
Assert.assertEquals(nodeNum, datanodeDetails.size());
Assert.assertTrue(cluster.isSameParent(datanodeDetails.get(0),
datanodeDetails.get(1)) || (datanodeCount % NODE_PER_RACK == 1));
// 3 replicas
nodeNum = 3;
- datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 15);
+ datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 0, 15);
Assert.assertEquals(nodeNum, datanodeDetails.size());
// requires at least 2 racks for following statement
assumeTrue(datanodeCount > NODE_PER_RACK &&
@@ -225,7 +225,7 @@ public class TestSCMContainerPlacementRackAware {
// 4 replicas
nodeNum = 4;
- datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 15);
+ datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 0, 15);
Assert.assertEquals(nodeNum, datanodeDetails.size());
// requires at least 2 racks and enough datanodes for following statement
assumeTrue(datanodeCount > NODE_PER_RACK + 1);
@@ -248,7 +248,7 @@ public class TestSCMContainerPlacementRackAware {
excludedNodes.add(datanodes.get(0));
excludedNodes.add(datanodes.get(1));
List<DatanodeDetails> datanodeDetails = policy.chooseDatanodes(
- excludedNodes, null, nodeNum, 15);
+ excludedNodes, null, nodeNum, 0, 15);
Assert.assertEquals(nodeNum, datanodeDetails.size());
Assert.assertFalse(cluster.isSameParent(datanodeDetails.get(0),
excludedNodes.get(0)));
@@ -260,7 +260,7 @@ public class TestSCMContainerPlacementRackAware {
excludedNodes.clear();
excludedNodes.add(datanodes.get(0));
datanodeDetails = policy.chooseDatanodes(
- excludedNodes, null, nodeNum, 15);
+ excludedNodes, null, nodeNum, 0, 15);
Assert.assertEquals(nodeNum, datanodeDetails.size());
Assert.assertTrue(cluster.isSameParent(
datanodeDetails.get(0), excludedNodes.get(0)) ||
@@ -272,7 +272,7 @@ public class TestSCMContainerPlacementRackAware {
excludedNodes.add(datanodes.get(0));
excludedNodes.add(datanodes.get(5));
datanodeDetails = policy.chooseDatanodes(
- excludedNodes, null, nodeNum, 15);
+ excludedNodes, null, nodeNum, 0, 15);
Assert.assertEquals(nodeNum, datanodeDetails.size());
Assert.assertTrue(cluster.isSameParent(
datanodeDetails.get(0), excludedNodes.get(0)) ||
@@ -288,7 +288,7 @@ public class TestSCMContainerPlacementRackAware {
(datanodeCount % NODE_PER_RACK > 1));
int nodeNum = 5;
List<DatanodeDetails> datanodeDetails =
- policy.chooseDatanodes(null, null, nodeNum, 15);
+ policy.chooseDatanodes(null, null, nodeNum, 0, 15);
Assert.assertEquals(nodeNum, datanodeDetails.size());
Assert.assertTrue(cluster.isSameParent(datanodeDetails.get(0),
datanodeDetails.get(1)));
@@ -321,7 +321,7 @@ public class TestSCMContainerPlacementRackAware {
// 5 replicas. there are only 3 racks. policy prohibit fallback should fail.
int nodeNum = 5;
try {
- policyNoFallback.chooseDatanodes(null, null, nodeNum, 15);
+ policyNoFallback.chooseDatanodes(null, null, nodeNum, 0, 15);
fail("Fallback prohibited, this call should fail");
} catch (Exception e) {
assertEquals("SCMException", e.getClass().getSimpleName());
@@ -350,7 +350,7 @@ public class TestSCMContainerPlacementRackAware {
// no excludedNodes, only favoredNodes
favoredNodes.add(datanodes.get(0));
List<DatanodeDetails> datanodeDetails = policy.chooseDatanodes(
- excludedNodes, favoredNodes, nodeNum, 15);
+ excludedNodes, favoredNodes, nodeNum, 0, 15);
Assert.assertEquals(nodeNum, datanodeDetails.size());
Assert.assertEquals(datanodeDetails.get(0).getNetworkFullPath(),
favoredNodes.get(0).getNetworkFullPath());
@@ -362,7 +362,7 @@ public class TestSCMContainerPlacementRackAware {
excludedNodes.add(datanodes.get(0));
favoredNodes.add(datanodes.get(2));
datanodeDetails = policy.chooseDatanodes(
- excludedNodes, favoredNodes, nodeNum, 15);
+ excludedNodes, favoredNodes, nodeNum, 0, 15);
Assert.assertEquals(nodeNum, datanodeDetails.size());
Assert.assertEquals(datanodeDetails.get(0).getNetworkFullPath(),
favoredNodes.get(0).getNetworkFullPath());
@@ -374,7 +374,7 @@ public class TestSCMContainerPlacementRackAware {
excludedNodes.add(datanodes.get(0));
favoredNodes.add(datanodes.get(0));
datanodeDetails = policy.chooseDatanodes(
- excludedNodes, favoredNodes, nodeNum, 15);
+ excludedNodes, favoredNodes, nodeNum, 0, 15);
Assert.assertEquals(nodeNum, datanodeDetails.size());
Assert.assertFalse(datanodeDetails.get(0).getNetworkFullPath()
.equals(favoredNodes.get(0).getNetworkFullPath()));
@@ -386,7 +386,7 @@ public class TestSCMContainerPlacementRackAware {
try {
// request storage space larger than node capability
- policy.chooseDatanodes(null, null, nodeNum, STORAGE_CAPACITY + 15);
+ policy.chooseDatanodes(null, null, nodeNum, STORAGE_CAPACITY + 0, 15);
fail("Storage requested exceeds capacity, this call should fail");
} catch (Exception e) {
assertTrue(e.getClass().getSimpleName().equals("SCMException"));
@@ -449,7 +449,7 @@ public class TestSCMContainerPlacementRackAware {
new SCMContainerPlacementRackAware(nodeManager, conf, clusterMap, true,
metrics);
List<DatanodeDetails> datanodeDetails =
- newPolicy.chooseDatanodes(null, null, nodeNum, 15);
+ newPolicy.chooseDatanodes(null, null, nodeNum, 0, 15);
Assert.assertEquals(nodeNum, datanodeDetails.size());
Assert.assertTrue(cluster.isSameParent(datanodeDetails.get(0),
datanodeDetails.get(1)));
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java
index 0515874..ce6d5d1 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java
@@ -100,7 +100,7 @@ public class TestSCMContainerPlacementRandom {
for (int i = 0; i < 100; i++) {
//when
List<DatanodeDetails> datanodeDetails = scmContainerPlacementRandom
- .chooseDatanodes(existingNodes, null, 1, 15);
+ .chooseDatanodes(existingNodes, null, 1, 15, 15);
//then
Assert.assertEquals(1, datanodeDetails.size());
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
index 88261f0..9b216a9 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.hdds.scm.net.NodeSchema;
import org.apache.hadoop.hdds.scm.net.NodeSchemaManager;
import org.apache.hadoop.hdds.scm.node.states.Node2PipelineMap;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
@@ -60,7 +61,6 @@ import static junit.framework.TestCase.assertEquals;
import static junit.framework.TestCase.assertTrue;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE;
import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA;
import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA;
import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA;
@@ -168,7 +168,7 @@ public class TestPipelinePlacementPolicy {
List<DatanodeDetails> results = localPlacementPolicy.chooseDatanodes(
new ArrayList<>(datanodes.size()),
new ArrayList<>(datanodes.size()),
- nodesRequired, 0);
+ nodesRequired, 0, 0);
Assert.assertEquals(nodesRequired, results.size());
// 3 nodes should be on different racks.
@@ -181,10 +181,7 @@ public class TestPipelinePlacementPolicy {
}
@Test
- public void testChooseNodeNotEnoughSpace() throws SCMException {
- // A huge container size
- conf.set(OZONE_SCM_CONTAINER_SIZE, "10TB");
-
+ public void testChooseNodeNotEnoughSpace() {
// There is only one node on 3 racks altogether.
List<DatanodeDetails> datanodes = new ArrayList<>();
for (Node node : SINGLE_NODE_RACK) {
@@ -198,39 +195,29 @@ public class TestPipelinePlacementPolicy {
localNodeManager, new PipelineStateManager(), conf);
int nodesRequired = HddsProtos.ReplicationFactor.THREE.getNumber();
- thrownExp.expect(SCMException.class);
- thrownExp.expectMessage("healthy datanodes with enough space");
- localPlacementPolicy.chooseDatanodes(new ArrayList<>(datanodes.size()),
- new ArrayList<>(datanodes.size()), nodesRequired, 0);
- }
-
- @Test
- public void testChooseNodeNotEnoughMetadataSpace() throws SCMException {
- // a huge free space min configured
- conf.setStorageSize(OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN,
- 500, StorageUnit.TB);
- // a small container size
- conf.set(OZONE_SCM_CONTAINER_SIZE, "100MB");
-
- // There is only one node on 3 racks altogether.
- List<DatanodeDetails> datanodes = new ArrayList<>();
- for (Node node : SINGLE_NODE_RACK) {
- DatanodeDetails datanode = overwriteLocationInNode(
- MockDatanodeDetails.randomDatanodeDetails(), node);
- datanodes.add(datanode);
+ String expectedMessageSubstring = "Unable to find enough nodes that meet " +
+ "the space requirement";
+ try {
+ // A huge container size
+ localPlacementPolicy.chooseDatanodes(new ArrayList<>(datanodes.size()),
+ new ArrayList<>(datanodes.size()), nodesRequired,
+ 0, 10 * OzoneConsts.TB);
+ Assert.fail("SCMException should have been thrown.");
+ } catch(SCMException ex) {
+ Assert.assertTrue(ex.getMessage().contains(expectedMessageSubstring));
}
- MockNodeManager localNodeManager = new MockNodeManager(initTopology(),
- datanodes, false, datanodes.size());
- PipelinePlacementPolicy localPlacementPolicy = new PipelinePlacementPolicy(
- localNodeManager, new PipelineStateManager(), conf);
- int nodesRequired = HddsProtos.ReplicationFactor.THREE.getNumber();
- thrownExp.expect(SCMException.class);
- thrownExp.expectMessage("healthy datanodes with enough space");
- localPlacementPolicy.chooseDatanodes(new ArrayList<>(datanodes.size()),
- new ArrayList<>(datanodes.size()), nodesRequired, 0);
+ try {
+ // a huge free space min configured
+ localPlacementPolicy.chooseDatanodes(new ArrayList<>(datanodes.size()),
+ new ArrayList<>(datanodes.size()), nodesRequired, 10 * OzoneConsts.TB,
+ 0);
+ Assert.fail("SCMException should have been thrown.");
+ } catch(SCMException ex) {
+ Assert.assertTrue(ex.getMessage().contains(expectedMessageSubstring));
+ }
}
-
+
@Test
public void testPickLowestLoadAnchor() throws IOException{
List<DatanodeDetails> healthyNodes = nodeManager
@@ -241,7 +228,7 @@ public class TestPipelinePlacementPolicy {
for (int i = 0; i < maxPipelineCount; i++) {
try {
List<DatanodeDetails> nodes = placementPolicy.chooseDatanodes(null,
- null, HddsProtos.ReplicationFactor.THREE.getNumber(), 0);
+ null, HddsProtos.ReplicationFactor.THREE.getNumber(), 0, 0);
Pipeline pipeline = Pipeline.newBuilder()
.setId(PipelineID.randomId())
@@ -398,7 +385,7 @@ public class TestPipelinePlacementPolicy {
List<DatanodeDetails> pickedNodes1 = placementPolicy.chooseDatanodes(
new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT),
new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT),
- nodesRequired, 0);
+ nodesRequired, 0, 0);
// modify node to pipeline mapping.
insertHeavyNodesIntoNodeManager(healthyNodes, minorityHeavy);
// NODES should be sufficient.
@@ -415,7 +402,7 @@ public class TestPipelinePlacementPolicy {
pickedNodes2 = placementPolicy.chooseDatanodes(
new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT),
new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT),
- nodesRequired, 0);
+ nodesRequired, 0, 0);
} catch (SCMException e) {
Assert.assertFalse(thrown);
thrown = true;
@@ -508,7 +495,7 @@ public class TestPipelinePlacementPolicy {
// As there is only 1 rack alive, the 3 DNs on /rack2 should be returned
List<DatanodeDetails> pickedDns = placementPolicy.chooseDatanodes(
- new ArrayList<>(), new ArrayList<>(), nodesRequired, 0);
+ new ArrayList<>(), new ArrayList<>(), nodesRequired, 0, 0);
assertEquals(3, pickedDns.size());
assertTrue(pickedDns.contains(dns.get(1)));
@@ -533,7 +520,7 @@ public class TestPipelinePlacementPolicy {
int nodesRequired = HddsProtos.ReplicationFactor.THREE.getNumber();
placementPolicy.chooseDatanodes(
- new ArrayList<>(), new ArrayList<>(), nodesRequired, 0);
+ new ArrayList<>(), new ArrayList<>(), nodesRequired, 0, 0);
}
@Test
@@ -552,7 +539,7 @@ public class TestPipelinePlacementPolicy {
List<DatanodeDetails> excluded = new ArrayList<>();
excluded.add(dns.get(0));
placementPolicy.chooseDatanodes(
- excluded, new ArrayList<>(), nodesRequired, 0);
+ excluded, new ArrayList<>(), nodesRequired, 0, 0);
}
private List<DatanodeDetails> setupSkewedRacks() {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
index 8554c1f..7eb8977 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.MockNodeManager;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.NodeStatus;
import org.junit.Assert;
import org.junit.Assume;
@@ -40,6 +41,8 @@ import java.util.Set;
import java.util.stream.Collectors;
import static org.apache.commons.collections.CollectionUtils.intersection;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
@@ -55,12 +58,15 @@ public class TestRatisPipelineProvider {
private MockNodeManager nodeManager;
private RatisPipelineProvider provider;
private PipelineStateManager stateManager;
- private OzoneConfiguration conf;
public void init(int maxPipelinePerNode) throws Exception {
+ init(maxPipelinePerNode, new OzoneConfiguration());
+ }
+
+ public void init(int maxPipelinePerNode, OzoneConfiguration conf)
+ throws Exception {
nodeManager = new MockNodeManager(true, 10);
nodeManager.setNumPipelinePerDatanode(maxPipelinePerNode);
- conf = new OzoneConfiguration();
conf.setInt(ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT,
maxPipelinePerNode);
stateManager = new PipelineStateManager();
@@ -219,6 +225,40 @@ public class TestRatisPipelineProvider {
nodes.stream().anyMatch(membersOfClosedPipelines::contains));
}
+ @Test
+ public void testCreatePipelinesWhenNotEnoughSpace() throws Exception {
+ String expectedErrorSubstring = "Unable to find enough" +
+ " nodes that meet the space requirement";
+
+ // Use large enough container or metadata sizes that no node will have
+ // enough space to hold one.
+ OzoneConfiguration largeContainerConf = new OzoneConfiguration();
+ largeContainerConf.set(OZONE_SCM_CONTAINER_SIZE, "100TB");
+ init(1, largeContainerConf);
+ for (ReplicationFactor factor: ReplicationFactor.values()) {
+ try {
+ provider.create(new RatisReplicationConfig(factor));
+ Assert.fail("Expected SCMException for large container size with " +
+ "replication factor " + factor.toString());
+ } catch(SCMException ex) {
+ Assert.assertTrue(ex.getMessage().contains(expectedErrorSubstring));
+ }
+ }
+
+ OzoneConfiguration largeMetadataConf = new OzoneConfiguration();
+ largeMetadataConf.set(OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN, "100TB");
+ init(1, largeMetadataConf);
+ for (ReplicationFactor factor: ReplicationFactor.values()) {
+ try {
+ provider.create(new RatisReplicationConfig(factor));
+ Assert.fail("Expected SCMException for large metadata size with " +
+ "replication factor " + factor.toString());
+ } catch(SCMException ex) {
+ Assert.assertTrue(ex.getMessage().contains(expectedErrorSubstring));
+ }
+ }
+ }
+
private void addPipeline(
List<DatanodeDetails> dns,
Pipeline.PipelineState open, ReplicationConfig replicationConfig)
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java
index 1656376..5ec7db6 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java
@@ -86,14 +86,15 @@ public class TestContainerPlacement {
for (int x = 0; x < opsCount; x++) {
long containerSize = random.nextInt(100) * OzoneConsts.GB;
+ long metadataSize = random.nextInt(100) * OzoneConsts.GB;
List<DatanodeDetails> nodesCapacity =
capacityPlacer.chooseDatanodes(new ArrayList<>(), null, nodesRequired,
- containerSize);
+ metadataSize, containerSize);
assertEquals(nodesRequired, nodesCapacity.size());
List<DatanodeDetails> nodesRandom =
randomPlacer.chooseDatanodes(nodesCapacity, null, nodesRequired,
- containerSize);
+ metadataSize, containerSize);
// One fifth of all calls are delete
if (x % 5 == 0) {
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java
index 1b1214e..352c58c 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java
@@ -342,7 +342,8 @@ public class TestContainerHealthTask extends AbstractReconSqlDBTest {
@Override
public List<DatanodeDetails> chooseDatanodes(
List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes,
- int nodesRequired, long sizeRequired) throws IOException {
+ int nodesRequired, long metadataSizeRequired, long dataSizeRequired)
+ throws IOException {
return null;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org