You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2023/01/11 10:34:17 UTC
[ozone] branch master updated: HDDS-6447. Refine SCM handling of unhealthy container replicas. (#3920)
This is an automated email from the ASF dual-hosted git repository.
siddhant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 7229b35958 HDDS-6447. Refine SCM handling of unhealthy container replicas. (#3920)
7229b35958 is described below
commit 7229b35958c0fc7c753c02a1789333a03ec699b3
Author: Ethan Rose <33...@users.noreply.github.com>
AuthorDate: Wed Jan 11 02:34:11 2023 -0800
HDDS-6447. Refine SCM handling of unhealthy container replicas. (#3920)
---
.../hdds/scm/container/ContainerReportHandler.java | 6 +-
.../replication/ContainerReplicaCount.java | 4 +-
.../replication/ECContainerReplicaCount.java | 13 +-
.../LegacyRatisContainerReplicaCount.java | 51 +
.../replication/LegacyReplicationManager.java | 802 +++--
.../replication/RatisContainerReplicaCount.java | 67 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 3 +-
.../replication/TestLegacyReplicationManager.java | 3586 +++++++++++---------
8 files changed, 2640 insertions(+), 1892 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
index 3d4d783212..2ff50aab75 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
@@ -209,8 +209,10 @@ public class ContainerReportHandler extends AbstractContainerReportHandler
}
/**
- * Processes the ContainerReport, unknown container reported
- * that will be deleted by SCM.
+ * Processes the ContainerReport.
+ * Any unknown container reported by DN and not present in SCM
+ * containerSet will either be logged as an error or deleted based on
+ * unknownContainerHandleAction.
*
* @param datanodeDetails Datanode from which this report was received
* @param container ContainerInfo representing the container
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaCount.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaCount.java
index 06fed7438f..9c47ac244e 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaCount.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaCount.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
-import java.util.Set;
+import java.util.List;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
@@ -33,7 +33,7 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalSt
public interface ContainerReplicaCount {
ContainerInfo getContainer();
- Set<ContainerReplica> getReplicas();
+ List<ContainerReplica> getReplicas();
boolean isSufficientlyReplicated();
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECContainerReplicaCount.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECContainerReplicaCount.java
index 78c41c0f93..15108131d7 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECContainerReplicaCount.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECContainerReplicaCount.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerReplica;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -76,14 +77,20 @@ public class ECContainerReplicaCount implements ContainerReplicaCount {
private final Map<Integer, Integer> healthyIndexes = new HashMap<>();
private final Map<Integer, Integer> decommissionIndexes = new HashMap<>();
private final Map<Integer, Integer> maintenanceIndexes = new HashMap<>();
- private final Set<ContainerReplica> replicas;
+ private final List<ContainerReplica> replicas;
public ECContainerReplicaCount(ContainerInfo containerInfo,
Set<ContainerReplica> replicas,
List<ContainerReplicaOp> replicaPendingOps,
int remainingMaintenanceRedundancy) {
this.containerInfo = containerInfo;
- this.replicas = replicas;
+ // Iterate replicas in deterministic order to avoid potential data loss
+ // on delete.
+ // See https://issues.apache.org/jira/browse/HDDS-4589.
+ // N.B., sort replicas by (containerID, datanodeDetails).
+ this.replicas = replicas.stream()
+ .sorted(Comparator.comparingLong(ContainerReplica::hashCode))
+ .collect(Collectors.toList());
this.repConfig = (ECReplicationConfig)containerInfo.getReplicationConfig();
this.pendingAdd = new ArrayList<>();
this.pendingDelete = new ArrayList<>();
@@ -173,7 +180,7 @@ public class ECContainerReplicaCount implements ContainerReplicaCount {
}
@Override
- public Set<ContainerReplica> getReplicas() {
+ public List<ContainerReplica> getReplicas() {
return replicas;
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyRatisContainerReplicaCount.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyRatisContainerReplicaCount.java
new file mode 100644
index 0000000000..a2139707ca
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyRatisContainerReplicaCount.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+
+import java.util.Set;
+
+/**
+ * When HDDS-6447 was done to improve the LegacyReplicationManager, work on
+ * the new replication manager had already started. When this class was added,
+ * the LegacyReplicationManager needed separate handling for healthy and
+ * unhealthy container replicas, but the new replication manager did not yet
+ * have this functionality. This class is used by the
+ * LegacyReplicationManager to allow {@link RatisContainerReplicaCount} to
+ * function for both use cases. When the new replication manager is finished
+ * and LegacyReplicationManager is removed, this class should be deleted and
+ * all necessary functionality consolidated to
+ * {@link RatisContainerReplicaCount}
+ */
+public class LegacyRatisContainerReplicaCount extends
+ RatisContainerReplicaCount {
+ public LegacyRatisContainerReplicaCount(ContainerInfo container,
+ Set<ContainerReplica> replicas,
+ int inFlightAdd,
+ int inFlightDelete, int replicationFactor,
+ int minHealthyForMaintenance) {
+ super(container, replicas, inFlightAdd, inFlightDelete, replicationFactor,
+ minHealthyForMaintenance);
+ }
+
+ @Override
+ protected int healthyReplicaCountAdapter() {
+ return 0;
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyReplicationManager.java
index 7e86455357..80a61ee5db 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyReplicationManager.java
@@ -72,11 +72,11 @@ import java.io.IOException;
import java.lang.reflect.Proxy;
import java.time.Clock;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
-import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -93,6 +93,7 @@ import java.util.stream.Collectors;
import static org.apache.hadoop.hdds.conf.ConfigTag.OZONE;
import static org.apache.hadoop.hdds.conf.ConfigTag.SCM;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
import static org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.RequestType.MOVE;
/**
@@ -522,24 +523,32 @@ public class LegacyReplicationManager {
*/
boolean sufficientlyReplicated = replicaSet.isSufficientlyReplicated();
boolean placementSatisfied = placementStatus.isPolicySatisfied();
+ ContainerID containerID = container.containerID();
+ if (!placementStatus.isPolicySatisfied()) {
+ report.incrementAndSample(HealthState.MIS_REPLICATED, containerID);
+ }
+ if (!replicaSet.isHealthy()) {
+ report.incrementAndSample(HealthState.UNHEALTHY, containerID);
+ }
if (!sufficientlyReplicated || !placementSatisfied) {
- if (!sufficientlyReplicated) {
- report.incrementAndSample(
- HealthState.UNDER_REPLICATED, container.containerID());
+ // Replicate container if needed.
+ if (!inflightReplication.isFull() || !inflightDeletion.isFull()) {
if (replicaSet.isUnrecoverable()) {
- report.incrementAndSample(HealthState.MISSING,
- container.containerID());
+ // There are no healthy or unhealthy replicas.
+ report.incrementAndSample(HealthState.MISSING, containerID);
+ report.incrementAndSample(HealthState.UNDER_REPLICATED,
+ containerID);
+ } else {
+ if (replicaSet.getHealthyReplicaCount() == 0 &&
+ replicaSet.getUnhealthyReplicaCount() != 0) {
+ handleAllReplicasUnhealthy(container, replicaSet,
+ placementStatus, report);
+ } else {
+ handleUnderReplicatedHealthy(container,
+ replicaSet, placementStatus, report);
+ }
}
}
- if (!placementSatisfied) {
- report.incrementAndSample(HealthState.MIS_REPLICATED,
- container.containerID());
-
- }
- if (!inflightReplication.isFull() || !inflightDeletion.isFull()) {
- handleUnderReplicatedContainer(container,
- replicaSet, placementStatus);
- }
return;
}
@@ -548,9 +557,7 @@ public class LegacyReplicationManager {
* action.
*/
if (replicaSet.isOverReplicated()) {
- report.incrementAndSample(HealthState.OVER_REPLICATED,
- container.containerID());
- handleOverReplicatedContainer(container, replicaSet);
+ handleOverReplicatedHealthy(container, replicaSet, report);
return;
}
@@ -560,9 +567,7 @@ public class LegacyReplicationManager {
are not in the same state as the container itself.
*/
if (!replicaSet.isHealthy()) {
- report.incrementAndSample(HealthState.UNHEALTHY,
- container.containerID());
- handleUnstableContainer(container, replicas);
+ handleOverReplicatedExcessUnhealthy(container, replicaSet, report);
}
}
} catch (ContainerNotFoundException ex) {
@@ -997,7 +1002,7 @@ public class LegacyReplicationManager {
*/
private RatisContainerReplicaCount getContainerReplicaCount(
ContainerInfo container, Set<ContainerReplica> replica) {
- return new RatisContainerReplicaCount(
+ return new LegacyRatisContainerReplicaCount(
container,
replica,
getInflightAdd(container.containerID()),
@@ -1126,180 +1131,209 @@ public class LegacyReplicationManager {
* @param replicaSet An instance of ContainerReplicaCount, containing the
* current replica count and inflight adds and deletes
*/
- private void handleUnderReplicatedContainer(final ContainerInfo container,
+ private void handleUnderReplicatedHealthy(final ContainerInfo container,
final RatisContainerReplicaCount replicaSet,
- final ContainerPlacementStatus placementStatus) {
+ final ContainerPlacementStatus placementStatus,
+ ReplicationManagerReport report) {
LOG.debug("Handling under-replicated container: {}", container);
- Set<ContainerReplica> replicas = replicaSet.getReplicas();
- try {
+ if (replicaSet.isSufficientlyReplicated()
+ && placementStatus.isPolicySatisfied()) {
+ LOG.info("The container {} with replicas {} is sufficiently " +
+ "replicated and is not mis-replicated",
+ container.getContainerID(), replicaSet);
+ return;
+ }
- if (replicaSet.isSufficientlyReplicated()
- && placementStatus.isPolicySatisfied()) {
- LOG.info("The container {} with replicas {} is sufficiently " +
- "replicated and is not mis-replicated",
- container.getContainerID(), replicaSet);
- return;
- }
- int repDelta = replicaSet.additionalReplicaNeeded();
- final ContainerID id = container.containerID();
- final List<DatanodeDetails> deletionInFlight
- = inflightDeletion.getDatanodeDetails(id);
- final List<DatanodeDetails> replicationInFlight
- = inflightReplication.getDatanodeDetails(id);
- final List<DatanodeDetails> source = replicas.stream()
- .filter(r ->
- r.getState() == State.QUASI_CLOSED ||
- r.getState() == State.CLOSED)
- // Exclude stale and dead nodes. This is particularly important for
- // maintenance nodes, as the replicas will remain present in the
- // container manager, even when they go dead.
- .filter(r ->
- getNodeStatus(r.getDatanodeDetails()).isHealthy())
- .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
- .sorted((r1, r2) -> r2.getSequenceId().compareTo(r1.getSequenceId()))
- .map(ContainerReplica::getDatanodeDetails)
- .collect(Collectors.toList());
- if (source.size() > 0) {
- final int replicationFactor = container
- .getReplicationConfig().getRequiredNodes();
- // Want to check if the container is mis-replicated after considering
- // inflight add and delete.
- // Create a new list from source (healthy replicas minus pending delete)
- List<DatanodeDetails> targetReplicas = new ArrayList<>(source);
- // Then add any pending additions
- targetReplicas.addAll(replicationInFlight);
- final ContainerPlacementStatus inFlightplacementStatus =
- containerPlacement.validateContainerPlacement(
- targetReplicas, replicationFactor);
- final int misRepDelta = inFlightplacementStatus.misReplicationCount();
- final int replicasNeeded
- = repDelta < misRepDelta ? misRepDelta : repDelta;
- if (replicasNeeded <= 0) {
- LOG.debug("Container {} meets replication requirement with " +
- "inflight replicas", id);
- return;
- }
+ List<ContainerReplica> allReplicas = replicaSet.getReplicas();
+ int numCloseCommandsSent = closeReplicasIfPossible(container, allReplicas);
+ int replicasNeeded =
+ replicaSet.additionalReplicaNeeded() - numCloseCommandsSent;
- // We should ensure that the target datanode has enough space
- // for a complete container to be created, but since the container
- // size may be changed smaller than origin, we should be defensive.
- final long dataSizeRequired = Math.max(container.getUsedBytes(),
- currentContainerSize);
- final List<DatanodeDetails> excludeList = replicas.stream()
- .map(ContainerReplica::getDatanodeDetails)
- .collect(Collectors.toList());
- excludeList.addAll(replicationInFlight);
- final List<DatanodeDetails> selectedDatanodes = containerPlacement
- .chooseDatanodes(excludeList, null, replicasNeeded,
- 0, dataSizeRequired);
- if (repDelta > 0) {
- LOG.info("Container {} is under replicated. Expected replica count" +
- " is {}, but found {}.", id, replicationFactor,
- replicationFactor - repDelta);
- }
- int newMisRepDelta = misRepDelta;
- if (misRepDelta > 0) {
- LOG.info("Container: {}. {}",
- id, placementStatus.misReplicatedReason());
- // Check if the new target nodes (original plus newly selected nodes)
- // makes the placement policy valid.
- targetReplicas.addAll(selectedDatanodes);
- newMisRepDelta = containerPlacement.validateContainerPlacement(
- targetReplicas, replicationFactor).misReplicationCount();
- }
- if (repDelta > 0 || newMisRepDelta < misRepDelta) {
- // Only create new replicas if we are missing a replicas or
- // the number of pending mis-replication has improved. No point in
- // creating new replicas for mis-replicated containers unless it
- // improves things.
- for (DatanodeDetails datanode : selectedDatanodes) {
- sendReplicateCommand(container, datanode, source);
- }
- } else {
- LOG.warn("Container {} is mis-replicated, requiring {} additional " +
- "replicas. After selecting new nodes, mis-replication has not " +
- "improved. No additional replicas will be scheduled",
- id, misRepDelta);
- }
- } else {
- LOG.warn("Cannot replicate container {}, no healthy replica found.",
- container.containerID());
- }
- } catch (IOException | IllegalStateException ex) {
- LOG.warn("Exception while replicating container {}.",
- container.getContainerID(), ex);
+ if (replicasNeeded > 0) {
+ report.incrementAndSample(HealthState.UNDER_REPLICATED,
+ container.containerID());
}
+
+ List<ContainerReplica> replicationSources = getReplicationSources(container,
+ replicaSet.getReplicas(), State.CLOSED, State.QUASI_CLOSED);
+ // This method will handle topology even if replicasNeeded <= 0.
+ replicateAnyWithTopology(container, replicationSources,
+ placementStatus, replicasNeeded);
}
/**
* If the given container is over replicated, identify the datanode(s)
* to delete the container and send delete container command to the
- * identified datanode(s).
+ * identified datanode(s). This method ignores unhealthy containers.
*
* @param container ContainerInfo
* @param replicaSet An instance of ContainerReplicaCount, containing the
* current replica count and inflight adds and deletes
*/
- private void handleOverReplicatedContainer(final ContainerInfo container,
- final RatisContainerReplicaCount replicaSet) {
+ private void handleOverReplicatedHealthy(final ContainerInfo container,
+ final RatisContainerReplicaCount replicaSet,
+ ReplicationManagerReport report) {
- final Set<ContainerReplica> replicas = replicaSet.getReplicas();
final ContainerID id = container.containerID();
final int replicationFactor =
container.getReplicationConfig().getRequiredNodes();
int excess = replicaSet.additionalReplicaNeeded() * -1;
if (excess > 0) {
-
LOG.info("Container {} is over replicated. Expected replica count" +
- " is {}, but found {}.", id, replicationFactor,
- replicationFactor + excess);
+ " is {}, but found {}.", id, replicationFactor,
+ replicationFactor + excess);
+
+ report.incrementAndSample(HealthState.OVER_REPLICATED,
+ container.containerID());
+
+ // The list of replicas that we can potentially delete to fix the over
+ // replicated state. This method is only concerned with healthy replicas.
+ final List<ContainerReplica> deleteCandidates =
+ getHealthyDeletionCandidates(container, replicaSet.getReplicas());
+
+ if (container.getState() == LifeCycleState.CLOSED) {
+ // Container is closed, so all healthy replicas are equal.
+ // We can choose which ones to delete based on topology.
+ // TODO Legacy RM implementation can only handle topology when all
+ // container replicas are closed and equal.
+ deleteExcessWithTopology(excess, container, deleteCandidates);
+ } else {
+ // Container is not yet closed. Choose which healthy replicas to
+ // delete so that we do not lose any origin node IDs.
+ deleteExcessWithNonUniqueOriginNodeIDs(container,
+ replicaSet.getReplicas(),
+ deleteCandidates, excess);
+ }
+ }
+ }
- final List<ContainerReplica> eligibleReplicas = new ArrayList<>(replicas);
+ /**
+ * Processes replicas of the container when all replicas are unhealthy (in
+ * a state that does not match the container state).
+ *
+ * Unhealthy replicas will first be checked to see if they can be closed.
+ * If there are more unhealthy replicas than required, some may be deleted.
+ * If there are fewer unhealthy replicas than required, some may be
+ * replicated.
+ */
+ private void handleAllReplicasUnhealthy(ContainerInfo container,
+ RatisContainerReplicaCount replicaSet,
+ ContainerPlacementStatus placementStatus,
+ ReplicationManagerReport report) {
- // Iterate replicas in deterministic order to avoid potential data loss.
- // See https://issues.apache.org/jira/browse/HDDS-4589.
- // N.B., sort replicas by (containerID, datanodeDetails).
- eligibleReplicas.sort(
- Comparator.comparingLong(ContainerReplica::hashCode));
+ List<ContainerReplica> replicas = replicaSet.getReplicas();
+ int excessReplicas = replicas.size() -
+ container.getReplicationConfig().getRequiredNodes();
+ int missingReplicas = excessReplicas * -1;
+
+ if (missingReplicas > 0) {
+ handleUnderReplicatedAllUnhealthy(container, replicas,
+ placementStatus, missingReplicas, report);
+ } else if (excessReplicas > 0) {
+ handleOverReplicatedAllUnhealthy(container, replicas,
+ excessReplicas, report);
+ } else {
+ // We have the correct number of unhealthy replicas. See if any of them
+ // can be closed.
+ closeReplicasIfPossible(container, replicas);
+ }
+ }
- final Map<UUID, ContainerReplica> uniqueReplicas =
- new LinkedHashMap<>();
+ /**
+ * Handles a container which has the correct number of healthy replicas,
+ * but an excess of unhealthy replicas.
+ *
+ * If the container is closed, the unhealthy replicas can be deleted. If the
+ * container is not yet closed, the unhealthy replicas with non-unique
+ * origin node IDs can be deleted.
+ */
+ private void handleOverReplicatedExcessUnhealthy(
+ final ContainerInfo container,
+ final RatisContainerReplicaCount replicaSet,
+ ReplicationManagerReport report) {
+ // Note - ReplicationManager would reach here only if the
+ // following conditions are met:
+ // 1. Container is in either CLOSED or QUASI-CLOSED state
+ // 2. We have adequate healthy replicas with extra unhealthy
+ // replicas.
+
+ List<ContainerReplica> replicas = replicaSet.getReplicas();
+ List<ContainerReplica> unhealthyReplicas =
+ getUnhealthyDeletionCandidates(container, replicas);
+
+ // Only unhealthy replicas which cannot be closed will remain eligible
+ // for deletion, since this method is deleting unhealthy containers only.
+ closeReplicasIfPossible(container, unhealthyReplicas);
+ if (!unhealthyReplicas.isEmpty()) {
+ LOG.info("Container {} has {} excess unhealthy replicas. Excess " +
+ "unhealthy replicas will be deleted.",
+ container.getContainerID(), unhealthyReplicas.size());
+
+ report.incrementAndSample(HealthState.OVER_REPLICATED,
+ container.containerID());
+
+ int excessReplicaCount = replicas.size() -
+ container.getReplicationConfig().getRequiredNodes();
+ if (container.getState() == LifeCycleState.CLOSED) {
+ // The container is already closed. The unhealthy replicas are extras
+ // and unnecessary.
+ deleteExcess(container, unhealthyReplicas, excessReplicaCount);
+ } else {
+ // Container is not yet closed.
+ // We only need to save the unhealthy replicas if they
+ // represent unique origin node IDs. If recovering these replicas is
+ // possible in the future they could be used to close the container.
+ deleteExcessWithNonUniqueOriginNodeIDs(container,
+ replicaSet.getReplicas(), unhealthyReplicas, excessReplicaCount);
+ }
+ }
+ }
- if (container.getState() != LifeCycleState.CLOSED) {
- replicas.stream()
- .filter(r -> compareState(container.getState(), r.getState()))
- .forEach(r -> uniqueReplicas
- .putIfAbsent(r.getOriginDatanodeId(), r));
+ /**
+ * Returns the replicas from {@code replicas} that:
+ * - Do not have in flight deletions
+ * - Exist on healthy datanodes
+ * - Have a replica state matching one of {@code validReplicaStates}. If
+ * this parameter is empty, any replica state is valid.
+ */
+ private List<ContainerReplica> getReplicationSources(ContainerInfo container,
+ List<ContainerReplica> replicas, State... validReplicaStates) {
+ final List<DatanodeDetails> deletionInFlight
+ = inflightDeletion.getDatanodeDetails(container.containerID());
+ final Set<State> validReplicaStateSet = Arrays.stream(validReplicaStates)
+ .collect(Collectors.toSet());
+ return replicas.stream()
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> getNodeStatus(r.getDatanodeDetails()).isHealthy()
+ && !deletionInFlight.contains(r.getDatanodeDetails())
+ && (validReplicaStateSet.isEmpty() ||
+ validReplicaStateSet.contains(r.getState())))
+ .collect(Collectors.toList());
+ }
- eligibleReplicas.removeAll(uniqueReplicas.values());
- }
- // Replica which are maintenance or decommissioned are not eligible to
- // be removed, as they do not count toward over-replication and they
- // also many not be available
- eligibleReplicas.removeIf(r ->
- r.getDatanodeDetails().getPersistedOpState() !=
- NodeOperationalState.IN_SERVICE);
-
- final List<ContainerReplica> unhealthyReplicas = eligibleReplicas
- .stream()
- .filter(r -> !compareState(container.getState(), r.getState()))
- .collect(Collectors.toList());
+ private List<ContainerReplica> getHealthyDeletionCandidates(
+ ContainerInfo container, List<ContainerReplica> replicas) {
+ return getDeletionCandidates(container, replicas, true);
+ }
- // If there are unhealthy replicas, then we should remove them even if it
- // makes the container violate the placement policy, as excess unhealthy
- // containers are not really useful. It will be corrected later as a
- // mis-replicated container will be seen as under-replicated.
- for (ContainerReplica r : unhealthyReplicas) {
- if (excess > 0) {
- sendDeleteCommand(container, r.getDatanodeDetails(), true);
- excess -= 1;
- } else {
- break;
- }
- }
- eligibleReplicas.removeAll(unhealthyReplicas);
- removeExcessReplicasIfNeeded(excess, container, eligibleReplicas);
- }
+ private List<ContainerReplica> getUnhealthyDeletionCandidates(
+ ContainerInfo container, List<ContainerReplica> replicas) {
+ return getDeletionCandidates(container, replicas, false);
+ }
+
+ /**
+ * A replica is eligible for deletion if its datanode is healthy and
+ * IN_SERVICE.
+ */
+ private List<ContainerReplica> getDeletionCandidates(ContainerInfo container,
+ List<ContainerReplica> replicas, boolean healthy) {
+ return replicas.stream()
+ .filter(r -> getNodeStatus(r.getDatanodeDetails()).isHealthy()
+ && compareState(container.getState(), r.getState()) == healthy
+ && r.getDatanodeDetails().getPersistedOpState() == IN_SERVICE)
+ .collect(Collectors.toList());
}
/**
@@ -1355,57 +1389,6 @@ public class LegacyReplicationManager {
}
}
- /**
- * remove execess replicas if needed, replicationFactor and placement policy
- * will be take into consideration.
- *
- * @param excess the excess number after subtracting replicationFactor
- * @param container ContainerInfo
- * @param eligibleReplicas An list of replicas, which may have excess replicas
- */
- private void removeExcessReplicasIfNeeded(int excess,
- final ContainerInfo container,
- final List<ContainerReplica> eligibleReplicas) {
- // After removing all unhealthy replicas, if the container is still over
- // replicated then we need to check if it is already mis-replicated.
- // If it is, we do no harm by removing excess replicas. However, if it is
- // not mis-replicated, then we can only remove replicas if they don't
- // make the container become mis-replicated.
- if (excess > 0) {
- Set<ContainerReplica> eligibleSet = new HashSet<>(eligibleReplicas);
- final int replicationFactor =
- container.getReplicationConfig().getRequiredNodes();
- ContainerPlacementStatus ps =
- getPlacementStatus(eligibleSet, replicationFactor);
-
- for (ContainerReplica r : eligibleReplicas) {
- if (excess <= 0) {
- break;
- }
- // First remove the replica we are working on from the set, and then
- // check if the set is now mis-replicated.
- eligibleSet.remove(r);
- ContainerPlacementStatus nowPS =
- getPlacementStatus(eligibleSet, replicationFactor);
- if (isPlacementStatusActuallyEqual(ps, nowPS)) {
- // Remove the replica if the container was already unsatisfied
- // and losing this replica keep actual placement count unchanged.
- // OR if losing this replica still keep satisfied
- sendDeleteCommand(container, r.getDatanodeDetails(), true);
- excess -= 1;
- continue;
- }
- // If we decided not to remove this replica, put it back into the set
- eligibleSet.add(r);
- }
- if (excess > 0) {
- LOG.info("The container {} is over replicated with {} excess " +
- "replica. The excess replicas cannot be removed without " +
- "violating the placement policy", container, excess);
- }
- }
- }
-
/**
* whether the given two ContainerPlacementStatus are actually equal.
*
@@ -1436,65 +1419,6 @@ public class LegacyReplicationManager {
replicaDns, replicationFactor);
}
- /**
- * Handles unstable container.
- * A container is inconsistent if any of the replica state doesn't
- * match the container state. We have to take appropriate action
- * based on state of the replica.
- *
- * @param container ContainerInfo
- * @param replicas Set of ContainerReplicas
- */
- private void handleUnstableContainer(final ContainerInfo container,
- final Set<ContainerReplica> replicas) {
- // Find unhealthy replicas
- List<ContainerReplica> unhealthyReplicas = replicas.stream()
- .filter(r -> !compareState(container.getState(), r.getState()))
- .collect(Collectors.toList());
-
- Iterator<ContainerReplica> iterator = unhealthyReplicas.iterator();
- while (iterator.hasNext()) {
- final ContainerReplica replica = iterator.next();
- final State state = replica.getState();
- if (state == State.OPEN || state == State.CLOSING) {
- sendCloseCommand(container, replica.getDatanodeDetails(), false);
- iterator.remove();
- }
-
- if (state == State.QUASI_CLOSED) {
- // Send force close command if the BCSID matches
- if (container.getSequenceId() == replica.getSequenceId()) {
- sendCloseCommand(container, replica.getDatanodeDetails(), true);
- iterator.remove();
- }
- }
- }
-
- // Now we are left with the replicas which are either unhealthy or
- // the BCSID doesn't match. These replicas should be deleted.
-
- /*
- * If we have unhealthy replicas we go under replicated and then
- * replicate the healthy copy.
- *
- * We also make sure that we delete only one unhealthy replica at a time.
- *
- * If there are two unhealthy replica:
- * - Delete first unhealthy replica
- * - Re-replicate the healthy copy
- * - Delete second unhealthy replica
- * - Re-replicate the healthy copy
- *
- * Note: Only one action will be executed in a single ReplicationMonitor
- * iteration. So to complete all the above actions we need four
- * ReplicationMonitor iterations.
- */
-
- unhealthyReplicas.stream().findFirst().ifPresent(replica ->
- sendDeleteCommand(container, replica.getDatanodeDetails(), true));
-
- }
-
/**
* Sends close container command for the given container to the given
* datanode.
@@ -2009,5 +1933,325 @@ public class LegacyReplicationManager {
inflightMoveFuture.remove(cid);
}
}
-}
+ private int closeReplicasIfPossible(ContainerInfo container,
+ List<ContainerReplica> replicas) {
+ // This method should not be used on open containers.
+ if (container.getState() == LifeCycleState.OPEN) {
+ return 0;
+ }
+
+ int numCloseCmdsSent = 0;
+ Iterator<ContainerReplica> iterator = replicas.iterator();
+ while (iterator.hasNext()) {
+ final ContainerReplica replica = iterator.next();
+ final State state = replica.getState();
+ if (state == State.OPEN || state == State.CLOSING) {
+ sendCloseCommand(container, replica.getDatanodeDetails(), false);
+ numCloseCmdsSent++;
+ iterator.remove();
+ } else if (state == State.QUASI_CLOSED) {
+ // Send force close command if the BCSID matches
+ if (container.getSequenceId() == replica.getSequenceId()) {
+ sendCloseCommand(container, replica.getDatanodeDetails(), true);
+ numCloseCmdsSent++;
+ iterator.remove();
+ }
+ }
+ }
+
+ return numCloseCmdsSent;
+ }
+
+ /* HELPER METHODS FOR UNHEALTHY OVER AND UNDER REPLICATED CONTAINERS */
+
+ /**
+ * Process a container with more replicas than required where all replicas
+ * are unhealthy.
+ *
+ * First try to close any replicas that are unhealthy due to pending
+ * closure. Replicas that can be closed will become healthy and will not be
+ * processed by this method.
+ * If the container is closed, delete replicas with lower BCSIDs first.
+ * If the container is not yet closed, delete replicas with origin node IDs
+ * already represented by other replicas.
+ */
+ private void handleOverReplicatedAllUnhealthy(ContainerInfo container,
+ List<ContainerReplica> replicas, int excess,
+ ReplicationManagerReport report) {
+ List<ContainerReplica> deleteCandidates =
+ getUnhealthyDeletionCandidates(container, replicas);
+
+ // Only unhealthy replicas which cannot be closed will remain eligible
+ // for deletion, since this method is deleting unhealthy containers only.
+ closeReplicasIfPossible(container, deleteCandidates);
+ if (deleteCandidates.isEmpty()) {
+ return;
+ }
+
+ if (excess > 0) {
+ report.incrementAndSample(HealthState.OVER_REPLICATED,
+ container.containerID());
+ int replicationFactor = container.getReplicationFactor().getNumber();
+ LOG.info("Container {} has all unhealthy replicas and is over " +
+ "replicated. Expected replica count" +
+ " is {}, but found {}.", container.getContainerID(),
+ replicationFactor, replicationFactor + excess);
+ }
+
+ if (container.getState() == LifeCycleState.CLOSED) {
+ // Prefer to delete unhealthy replicas with lower BCS IDs.
+ // If the replica became unhealthy after the container was closed but
+ // before the replica could be closed, it may have a smaller BCSID.
+ deleteExcessLowestBcsIDs(container, deleteCandidates, excess);
+ } else {
+ // Container is not yet closed.
+ // We only need to save the unhealthy replicas if they
+ // represent unique origin node IDs. If recovering these replicas is
+ // possible in the future they could be used to close the container.
+ deleteExcessWithNonUniqueOriginNodeIDs(container,
+ replicas, deleteCandidates, excess);
+ }
+ }
+
+ /**
+ * Processes container replicas when all replicas are unhealthy and there
+ * are fewer than the required number of replicas.
+ *
+ * If any of these replicas unhealthy because they are pending closure and
+ * they can be closed, close them to create a healthy replica that can be
+ * replicated.
+ * If none of the replicas can be closed, use one of the unhealthy replicas
+ * to restore replica count while satisfying topology requirements.
+ */
+ private void handleUnderReplicatedAllUnhealthy(ContainerInfo container,
+ List<ContainerReplica> replicas, ContainerPlacementStatus placementStatus,
+ int additionalReplicasNeeded, ReplicationManagerReport report) {
+
+ report.incrementAndSample(HealthState.UNDER_REPLICATED,
+ container.containerID());
+ int numCloseCmdsSent = closeReplicasIfPossible(container, replicas);
+ // Only replicate unhealthy containers if none of the unhealthy replicas
+ // could be closed. If we sent a close command to an unhealthy replica,
+ // we should wait for that to complete and replicate it when it becomes
+ // healthy on a future iteration.
+ if (numCloseCmdsSent == 0) {
+ LOG.info("Container {} is under replicated missing {} replicas with all" +
+ " replicas unhealthy. Copying unhealthy replicas.",
+ container.getContainerID(), additionalReplicasNeeded);
+ // TODO Datanodes currently shuffle sources, so we cannot prioritize
+ // some replicas based on BCSID or origin node ID.
+ replicateAnyWithTopology(container,
+ getReplicationSources(container, replicas), placementStatus,
+ additionalReplicasNeeded);
+ }
+ }
+
+ /* HELPER METHODS FOR ALL OVER AND UNDER REPLICATED CONTAINERS */
+
+ /**
+ * Deletes the first {@code excess} replicas from {@code deleteCandidates}.
+ * Replicas whose datanode operation state is not IN_SERVICE will be skipped.
+ */
+ private void deleteExcess(ContainerInfo container,
+ List<ContainerReplica> deleteCandidates, int excess) {
+ // Replica which are maintenance or decommissioned are not eligible to
+ // be removed, as they do not count toward over-replication and they
+ // also may not be available
+ deleteCandidates.removeIf(r ->
+ r.getDatanodeDetails().getPersistedOpState() !=
+ NodeOperationalState.IN_SERVICE);
+
+ deleteCandidates.stream().limit(excess).forEach(r ->
+ sendDeleteCommand(container, r.getDatanodeDetails(), true));
+ }
+
+ /**
+ * remove execess replicas if needed, replicationFactor and placement policy
+ * will be take into consideration.
+ *
+ * @param excess the excess number after subtracting replicationFactor
+ * @param container ContainerInfo
+ * @param eligibleReplicas An list of replicas, which may have excess replicas
+ */
+ private void deleteExcessWithTopology(int excess,
+ final ContainerInfo container,
+ final List<ContainerReplica> eligibleReplicas) {
+ // After removing all unhealthy replicas, if the container is still over
+ // replicated then we need to check if it is already mis-replicated.
+ // If it is, we do no harm by removing excess replicas. However, if it is
+ // not mis-replicated, then we can only remove replicas if they don't
+ // make the container become mis-replicated.
+ if (excess > 0) {
+ Set<ContainerReplica> eligibleSet = new HashSet<>(eligibleReplicas);
+ final int replicationFactor =
+ container.getReplicationConfig().getRequiredNodes();
+ ContainerPlacementStatus ps =
+ getPlacementStatus(eligibleSet, replicationFactor);
+
+ for (ContainerReplica r : eligibleReplicas) {
+ if (excess <= 0) {
+ break;
+ }
+ // First remove the replica we are working on from the set, and then
+ // check if the set is now mis-replicated.
+ eligibleSet.remove(r);
+ ContainerPlacementStatus nowPS =
+ getPlacementStatus(eligibleSet, replicationFactor);
+ if (isPlacementStatusActuallyEqual(ps, nowPS)) {
+ // Remove the replica if the container was already unsatisfied
+ // and losing this replica keep actual placement count unchanged.
+ // OR if losing this replica still keep satisfied
+ sendDeleteCommand(container, r.getDatanodeDetails(), true);
+ excess -= 1;
+ continue;
+ }
+ // If we decided not to remove this replica, put it back into the set
+ eligibleSet.add(r);
+ }
+ if (excess > 0) {
+ LOG.info("The container {} is over replicated with {} excess " +
+ "replica. The excess replicas cannot be removed without " +
+ "violating the placement policy", container, excess);
+ }
+ }
+ }
+
+ private void deleteExcessWithNonUniqueOriginNodeIDs(ContainerInfo container,
+ List<ContainerReplica> allReplicas,
+ List<ContainerReplica> deleteCandidates, int excess) {
+ // Remove delete candidates whose origin node ID is not already covered
+ // by an existing replica.
+ // TODO topology handling must be improved to make an optimal
+ // choice as to which replica to keep.
+
+ // Gather the origin node IDs of replicas which are not candidates for
+ // deletion.
+ Set<UUID> existingOriginNodeIDs = allReplicas.stream()
+ .filter(r -> !deleteCandidates.contains(r))
+ .map(ContainerReplica::getOriginDatanodeId)
+ .collect(Collectors.toSet());
+
+ List<ContainerReplica> nonUniqueDeleteCandidates = new ArrayList<>();
+ for (ContainerReplica replica: deleteCandidates) {
+ if (existingOriginNodeIDs.contains(replica.getOriginDatanodeId())) {
+ nonUniqueDeleteCandidates.add(replica);
+ } else {
+ // Spare this replica with this new origin node ID from deletion.
+ // delete candidates seen later in the loop with this same origin
+ // node ID can be deleted.
+ existingOriginNodeIDs.add(replica.getOriginDatanodeId());
+ }
+ }
+
+ if (LOG.isDebugEnabled() && nonUniqueDeleteCandidates.size() < excess) {
+ LOG.debug("Unable to delete {} excess replicas of container {}. Only {}" +
+ " replicas can be deleted to preserve unique origin node IDs for " +
+ "this unclosed container.", excess, container.getContainerID(),
+ nonUniqueDeleteCandidates.size());
+ }
+ deleteExcess(container, nonUniqueDeleteCandidates, excess);
+ }
+
+ /**
+ * Delete {@code excess} replicas from {@code deleteCandidates}, deleting
+ * those with lowest BCSIDs first.
+ */
+ private void deleteExcessLowestBcsIDs(ContainerInfo container,
+ List<ContainerReplica> deleteCandidates, int excess) {
+ // Sort containers with lowest BCSID first. These will be the first ones
+ // deleted.
+ deleteCandidates.sort(
+ Comparator.comparingLong(ContainerReplica::getSequenceId).reversed());
+ deleteExcess(container, deleteCandidates, excess);
+ }
+
+ /**
+ * Choose {@code additionalReplicasNeeded} datanodes to make copies of some
+ * of the container replicas to restore replication factor or satisfy
+ * topology requirements.
+ */
+ private void replicateAnyWithTopology(ContainerInfo container,
+ List<ContainerReplica> replicas,
+ ContainerPlacementStatus placementStatus, int additionalReplicasNeeded) {
+ try {
+ final ContainerID id = container.containerID();
+
+ final List<DatanodeDetails> sourceDNs = replicas.stream()
+ .map(ContainerReplica::getDatanodeDetails)
+ .collect(Collectors.toList());
+ final List<DatanodeDetails> replicationInFlight
+ = inflightReplication.getDatanodeDetails(id);
+ if (sourceDNs.size() > 0) {
+ final int replicationFactor = container
+ .getReplicationConfig().getRequiredNodes();
+ // Want to check if the container is mis-replicated after considering
+ // inflight add and delete.
+ // Create a new list from source (healthy replicas minus pending delete)
+ List<DatanodeDetails> targetReplicas = new ArrayList<>(sourceDNs);
+ // Then add any pending additions
+ targetReplicas.addAll(replicationInFlight);
+ final ContainerPlacementStatus inFlightplacementStatus =
+ containerPlacement.validateContainerPlacement(
+ targetReplicas, replicationFactor);
+ final int misRepDelta = inFlightplacementStatus.misReplicationCount();
+ final int replicasNeeded =
+ Math.max(additionalReplicasNeeded, misRepDelta);
+ if (replicasNeeded <= 0) {
+ LOG.debug("Container {} meets replication requirement with " +
+ "inflight replicas", id);
+ return;
+ }
+
+ // We should ensure that the target datanode has enough space
+ // for a complete container to be created, but since the container
+ // size may be changed smaller than origin, we should be defensive.
+ final long dataSizeRequired = Math.max(container.getUsedBytes(),
+ currentContainerSize);
+ final List<DatanodeDetails> excludeList = replicas.stream()
+ .map(ContainerReplica::getDatanodeDetails)
+ .collect(Collectors.toList());
+ excludeList.addAll(replicationInFlight);
+ final List<DatanodeDetails> selectedDatanodes = containerPlacement
+ .chooseDatanodes(excludeList, null, replicasNeeded,
+ 0, dataSizeRequired);
+ if (additionalReplicasNeeded > 0) {
+ LOG.info("Container {} is under replicated. Expected replica count" +
+ " is {}, but found {}.", id, replicationFactor,
+ replicationFactor - additionalReplicasNeeded);
+ }
+ int newMisRepDelta = misRepDelta;
+ if (misRepDelta > 0) {
+ LOG.info("Container: {}. {}",
+ id, placementStatus.misReplicatedReason());
+ // Check if the new target nodes (original plus newly selected nodes)
+ // makes the placement policy valid.
+ targetReplicas.addAll(selectedDatanodes);
+ newMisRepDelta = containerPlacement.validateContainerPlacement(
+ targetReplicas, replicationFactor).misReplicationCount();
+ }
+ if (additionalReplicasNeeded > 0 || newMisRepDelta < misRepDelta) {
+ // Only create new replicas if we are missing a replicas or
+ // the number of pending mis-replication has improved. No point in
+ // creating new replicas for mis-replicated containers unless it
+ // improves things.
+ for (DatanodeDetails datanode : selectedDatanodes) {
+ sendReplicateCommand(container, datanode, sourceDNs);
+ }
+ } else {
+ LOG.warn("Container {} is mis-replicated, requiring {} additional " +
+ "replicas. After selecting new nodes, mis-replication has" +
+ "not improved. No additional replicas will be scheduled",
+ id, misRepDelta);
+ }
+ } else {
+ LOG.warn("Cannot replicate container {}, no healthy datanodes with " +
+ "replica found.",
+ container.containerID());
+ }
+ } catch (IOException | IllegalStateException ex) {
+ LOG.warn("Exception while replicating container {}.",
+ container.getContainerID(), ex);
+ }
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisContainerReplicaCount.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisContainerReplicaCount.java
index 1363ba992a..ca30d2d5b8 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisContainerReplicaCount.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisContainerReplicaCount.java
@@ -22,7 +22,11 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONED;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONING;
@@ -38,7 +42,8 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalSt
*/
public class RatisContainerReplicaCount implements ContainerReplicaCount {
- private int healthyCount;
+ private int healthyReplicaCount;
+ private int unhealthyReplicaCount;
private int decommissionCount;
private int maintenanceCount;
private final int inFlightAdd;
@@ -46,24 +51,32 @@ public class RatisContainerReplicaCount implements ContainerReplicaCount {
private final int repFactor;
private final int minHealthyForMaintenance;
private final ContainerInfo container;
- private final Set<ContainerReplica> replica;
+ private final List<ContainerReplica> replicas;
public RatisContainerReplicaCount(ContainerInfo container,
- Set<ContainerReplica> replica, int inFlightAdd,
+ Set<ContainerReplica> replicas,
+ int inFlightAdd,
int inFlightDelete, int replicationFactor,
int minHealthyForMaintenance) {
- this.healthyCount = 0;
+ this.unhealthyReplicaCount = 0;
+ this.healthyReplicaCount = 0;
this.decommissionCount = 0;
this.maintenanceCount = 0;
this.inFlightAdd = inFlightAdd;
this.inFlightDel = inFlightDelete;
this.repFactor = replicationFactor;
- this.replica = replica;
+ // Iterate replicas in deterministic order to avoid potential data loss
+ // on delete.
+ // See https://issues.apache.org/jira/browse/HDDS-4589.
+ // N.B., sort replicas by (containerID, datanodeDetails).
+ this.replicas = replicas.stream()
+ .sorted(Comparator.comparingLong(ContainerReplica::hashCode))
+ .collect(Collectors.toList());
this.minHealthyForMaintenance
= Math.min(this.repFactor, minHealthyForMaintenance);
this.container = container;
- for (ContainerReplica cr : this.replica) {
+ for (ContainerReplica cr : this.replicas) {
HddsProtos.NodeOperationalState state =
cr.getDatanodeDetails().getPersistedOpState();
if (state == DECOMMISSIONED || state == DECOMMISSIONING) {
@@ -71,13 +84,33 @@ public class RatisContainerReplicaCount implements ContainerReplicaCount {
} else if (state == IN_MAINTENANCE || state == ENTERING_MAINTENANCE) {
maintenanceCount++;
} else {
- healthyCount++;
+ if (LegacyReplicationManager.compareState(container.getState(),
+ cr.getState())) {
+ healthyReplicaCount++;
+ } else {
+ unhealthyReplicaCount++;
+ }
}
}
}
- public int getHealthyCount() {
- return healthyCount;
+ public int getHealthyReplicaCount() {
+ return healthyReplicaCount + healthyReplicaCountAdapter();
+ }
+
+ public int getUnhealthyReplicaCount() {
+ return unhealthyReplicaCount;
+ }
+
+ /**
+ * The new replication manager currently counts unhealthy and healthy
+ * replicas together. This should be updated when changes from HDDS-6447
+ * are integrated into the new replication manager. See
+ * {@link LegacyRatisContainerReplicaCount}, which overrides this method, for
+ * details.
+ */
+ protected int healthyReplicaCountAdapter() {
+ return getUnhealthyReplicaCount();
}
@Override
@@ -100,15 +133,16 @@ public class RatisContainerReplicaCount implements ContainerReplicaCount {
}
@Override
- public Set<ContainerReplica> getReplicas() {
- return replica;
+ public List<ContainerReplica> getReplicas() {
+ return new ArrayList<>(replicas);
}
@Override
public String toString() {
return "Container State: " + container.getState() +
- " Replica Count: " + replica.size() +
- " Healthy Count: " + healthyCount +
+ " Replica Count: " + replicas.size() +
+ " Healthy Count: " + healthyReplicaCount +
+ " Unhealthy Count: " + unhealthyReplicaCount +
" Decommission Count: " + decommissionCount +
" Maintenance Count: " + maintenanceCount +
" inFlightAdd Count: " + inFlightAdd +
@@ -211,7 +245,7 @@ public class RatisContainerReplicaCount implements ContainerReplicaCount {
* for under replicated and a negative value for over replicated.
*/
private int missingReplicas() {
- int delta = repFactor - healthyCount;
+ int delta = repFactor - getHealthyReplicaCount();
if (delta < 0) {
// Over replicated, so may need to remove a container.
@@ -220,7 +254,7 @@ public class RatisContainerReplicaCount implements ContainerReplicaCount {
// May be under-replicated, depending on maintenance.
delta = Math.max(0, delta - maintenanceCount);
int neededHealthy =
- Math.max(0, minHealthyForMaintenance - healthyCount);
+ Math.max(0, minHealthyForMaintenance - getHealthyReplicaCount());
delta = Math.max(neededHealthy, delta);
return delta;
} else { // delta == 0
@@ -346,7 +380,8 @@ public class RatisContainerReplicaCount implements ContainerReplicaCount {
*/
public int getRemainingRedundancy() {
return Math.max(0,
- healthyCount + decommissionCount + maintenanceCount - inFlightDel - 1);
+ getHealthyReplicaCount() + decommissionCount + maintenanceCount
+ - inFlightDel - 1);
}
/**
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java
index 0bd644a0cb..528d2dfef3 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java
@@ -37,6 +37,7 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayDeque;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.HashMap;
@@ -398,7 +399,7 @@ public class DatanodeAdminMonitorImpl implements DatanodeAdminMonitor {
return underReplicated == 0 && unhealthy == 0;
}
- private String replicaDetails(Set<ContainerReplica> replicas) {
+ private String replicaDetails(Collection<ContainerReplica> replicas) {
StringBuilder sb = new StringBuilder();
sb.append("Replicas{");
sb.append(replicas.stream()
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestLegacyReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestLegacyReplicationManager.java
index 69ed758b40..8c46db43ac 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestLegacyReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestLegacyReplicationManager.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdds.scm.container.replication;
import com.google.common.primitives.Longs;
import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -71,7 +70,10 @@ import org.apache.ozone.test.TestClock;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.Mockito;
import java.io.File;
@@ -79,7 +81,9 @@ import java.io.IOException;
import java.time.Instant;
import java.time.ZoneId;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -103,6 +107,8 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE;
import static org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.CLOSED;
+import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.QUASI_CLOSED;
+import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.UNHEALTHY;
import static org.apache.hadoop.hdds.scm.HddsTestUtils.CONTAINER_NUM_KEYS_DEFAULT;
import static org.apache.hadoop.hdds.scm.HddsTestUtils.CONTAINER_USED_BYTES_DEFAULT;
import static org.apache.hadoop.hdds.scm.HddsTestUtils.getContainer;
@@ -282,1726 +288,2132 @@ public class TestLegacyReplicationManager {
}
@AfterEach
- public void tearDown() throws Exception {
+ public void teardown() throws Exception {
containerStateManager.close();
+ replicationManager.stop();
if (dbStore != null) {
dbStore.close();
}
-
- FileUtil.fullyDelete(testDir);
- }
-
- /**
- * Checks if restarting of replication manager works.
- */
- @Test
- public void testReplicationManagerRestart() throws InterruptedException {
- Assertions.assertTrue(replicationManager.isRunning());
- replicationManager.stop();
- // Stop is a non-blocking call, it might take sometime for the
- // ReplicationManager to shutdown
- Thread.sleep(500);
- Assertions.assertFalse(replicationManager.isRunning());
- replicationManager.start();
- Assertions.assertTrue(replicationManager.isRunning());
- }
-
- /**
- * Open containers are not handled by ReplicationManager.
- * This test-case makes sure that ReplicationManages doesn't take
- * any action on OPEN containers.
- */
- @Test
- public void testOpenContainer() throws IOException, TimeoutException {
- final ContainerInfo container = getContainer(LifeCycleState.OPEN);
- containerStateManager.addContainer(container.getProtobuf());
- replicationManager.processAll();
- eventQueue.processAll(1000);
- ReplicationManagerReport report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.OPEN));
- Assertions.assertEquals(0, datanodeCommandHandler.getInvocation());
+ FileUtils.deleteDirectory(testDir);
}
- /**
- * If the container is in CLOSING state we resend close container command
- * to all the datanodes.
- */
- @Test
- public void testClosingContainer() throws IOException, TimeoutException {
- final ContainerInfo container = getContainer(LifeCycleState.CLOSING);
- final ContainerID id = container.containerID();
-
- containerStateManager.addContainer(container.getProtobuf());
-
- // Two replicas in CLOSING state
- final Set<ContainerReplica> replicas = getReplicas(id, State.CLOSING,
- randomDatanodeDetails(),
- randomDatanodeDetails());
-
- // One replica in OPEN state
- final DatanodeDetails datanode = randomDatanodeDetails();
- replicas.addAll(getReplicas(id, State.OPEN, datanode));
-
- for (ContainerReplica replica : replicas) {
- containerStateManager.updateContainerReplica(id, replica);
+ @Nested
+ class Misc {
+ /**
+ * Checks if restarting of replication manager works.
+ */
+ @Test
+ public void testReplicationManagerRestart() throws InterruptedException {
+ Assertions.assertTrue(replicationManager.isRunning());
+ replicationManager.stop();
+ // Stop is a non-blocking call, it might take sometime for the
+ // ReplicationManager to shutdown
+ Thread.sleep(500);
+ Assertions.assertFalse(replicationManager.isRunning());
+ replicationManager.start();
+ Assertions.assertTrue(replicationManager.isRunning());
}
- final int currentCloseCommandCount = datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.closeContainerCommand);
+ @Test
+ public void testGeneratedConfig() {
+ ReplicationManagerConfiguration rmc = OzoneConfiguration.newInstanceOf(
+ ReplicationManagerConfiguration.class);
- replicationManager.processAll();
- eventQueue.processAll(1000);
- Assertions.assertEquals(currentCloseCommandCount + 3, datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.closeContainerCommand));
+ //default is not included in ozone-site.xml but generated from annotation
+ //to the ozone-site-generated.xml which should be loaded by the
+ // OzoneConfiguration.
+ Assertions.assertEquals(1800000, rmc.getEventTimeout());
- // Update the OPEN to CLOSING
- for (ContainerReplica replica : getReplicas(id, State.CLOSING, datanode)) {
- containerStateManager.updateContainerReplica(id, replica);
}
-
- replicationManager.processAll();
- eventQueue.processAll(1000);
- Assertions.assertEquals(currentCloseCommandCount + 6, datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.closeContainerCommand));
- ReplicationManagerReport report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.CLOSING));
- }
-
-
- /**
- * The container is QUASI_CLOSED but two of the replica is still in
- * open state. ReplicationManager should resend close command to those
- * datanodes.
- */
- @Test
- public void testQuasiClosedContainerWithTwoOpenReplica()
- throws IOException, TimeoutException {
- final ContainerInfo container = getContainer(LifeCycleState.QUASI_CLOSED);
- final ContainerID id = container.containerID();
- final UUID originNodeId = UUID.randomUUID();
- final ContainerReplica replicaOne = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaTwo = getReplicas(
- id, State.OPEN, 1000L, originNodeId, randomDatanodeDetails());
- final DatanodeDetails datanodeDetails = randomDatanodeDetails();
- final ContainerReplica replicaThree = getReplicas(
- id, State.OPEN, 1000L, datanodeDetails.getUuid(), datanodeDetails);
-
- containerStateManager.addContainer(container.getProtobuf());
- containerStateManager.updateContainerReplica(id, replicaOne);
- containerStateManager.updateContainerReplica(id, replicaTwo);
- containerStateManager.updateContainerReplica(
- id, replicaThree);
-
- final int currentCloseCommandCount = datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.closeContainerCommand);
- // Two of the replicas are in OPEN state
- replicationManager.processAll();
- eventQueue.processAll(1000);
- Assertions.assertEquals(currentCloseCommandCount + 2, datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.closeContainerCommand));
- Assertions.assertTrue(datanodeCommandHandler.received(
- SCMCommandProto.Type.closeContainerCommand,
- replicaTwo.getDatanodeDetails()));
- Assertions.assertTrue(datanodeCommandHandler.received(
- SCMCommandProto.Type.closeContainerCommand,
- replicaThree.getDatanodeDetails()));
- ReplicationManagerReport report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
}
/**
- * When the container is in QUASI_CLOSED state and all the replicas are
- * also in QUASI_CLOSED state and doesn't have a quorum to force close
- * the container, ReplicationManager will not do anything.
+ * Tests replication manager with healthy open and closed containers. No
+ * quasi closed or unhealthy containers are involved.
*/
- @Test
- public void testHealthyQuasiClosedContainer()
- throws IOException, TimeoutException {
- final ContainerInfo container = getContainer(LifeCycleState.QUASI_CLOSED);
- final ContainerID id = container.containerID();
- final UUID originNodeId = UUID.randomUUID();
- final ContainerReplica replicaOne = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaTwo = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaThree = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
-
- containerStateManager.addContainer(container.getProtobuf());
- containerStateManager.updateContainerReplica(id, replicaOne);
- containerStateManager.updateContainerReplica(id, replicaTwo);
- containerStateManager.updateContainerReplica(
- id, replicaThree);
-
- // All the QUASI_CLOSED replicas have same originNodeId, so the
- // container will not be closed. ReplicationManager should take no action.
- replicationManager.processAll();
- eventQueue.processAll(1000);
- Assertions.assertEquals(0, datanodeCommandHandler.getInvocation());
- ReplicationManagerReport report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
- }
-
- /**
- * When a container is QUASI_CLOSED and we don't have quorum to force close
- * the container, the container should have all the replicas in QUASI_CLOSED
- * state, else ReplicationManager will take action.
- *
- * In this test case we make one of the replica unhealthy, replication manager
- * will send delete container command to the datanode which has the unhealthy
- * replica.
- */
- @Test
- public void testQuasiClosedContainerWithUnhealthyReplica()
- throws IOException, TimeoutException {
- final ContainerInfo container = getContainer(LifeCycleState.QUASI_CLOSED);
- container.setUsedBytes(100);
- final ContainerID id = container.containerID();
- final UUID originNodeId = UUID.randomUUID();
- final ContainerReplica replicaOne = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaTwo = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaThree = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
-
- containerStateManager.addContainer(container.getProtobuf());
- containerStateManager.updateContainerReplica(id, replicaOne);
- containerStateManager.updateContainerReplica(id, replicaTwo);
- containerStateManager.updateContainerReplica(
- id, replicaThree);
-
- final int currentDeleteCommandCount = datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
- final int currentReplicateCommandCount = datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.replicateContainerCommand);
-
- // All the QUASI_CLOSED replicas have same originNodeId, so the
- // container will not be closed. ReplicationManager should take no action.
- replicationManager.processAll();
- eventQueue.processAll(1000);
- Assertions.assertEquals(0, datanodeCommandHandler.getInvocation());
-
- // Make the first replica unhealthy
- final ContainerReplica unhealthyReplica = getReplicas(
- id, State.UNHEALTHY, 1000L, originNodeId,
- replicaOne.getDatanodeDetails());
- containerStateManager.updateContainerReplica(
- id, unhealthyReplica);
-
- replicationManager.processAll();
- eventQueue.processAll(1000);
- Assertions.assertEquals(currentDeleteCommandCount + 1,
- datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand));
- Assertions.assertTrue(datanodeCommandHandler.received(
- SCMCommandProto.Type.deleteContainerCommand,
- replicaOne.getDatanodeDetails()));
- Assertions.assertEquals(currentDeleteCommandCount + 1,
- replicationManager.getMetrics().getNumDeletionCmdsSent());
-
- // Now we will delete the unhealthy replica from in-memory.
- containerStateManager.removeContainerReplica(id, replicaOne);
-
- final long currentBytesToReplicate = replicationManager.getMetrics()
- .getNumReplicationBytesTotal();
-
- // The container is under replicated as unhealthy replica is removed
- replicationManager.processAll();
- eventQueue.processAll(1000);
-
- // We should get replicate command
- Assertions.assertEquals(currentReplicateCommandCount + 1,
- datanodeCommandHandler.getInvocationCount(
- SCMCommandProto.Type.replicateContainerCommand));
- Assertions.assertEquals(currentReplicateCommandCount + 1,
- replicationManager.getMetrics().getNumReplicationCmdsSent());
- Assertions.assertEquals(currentBytesToReplicate + 100L,
- replicationManager.getMetrics().getNumReplicationBytesTotal());
- Assertions.assertEquals(1, getInflightCount(InflightType.REPLICATION));
- Assertions.assertEquals(1, replicationManager.getMetrics()
- .getInflightReplication());
-
- // We should have one under replicated and one quasi_closed_stuck
- ReplicationManagerReport report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.UNDER_REPLICATED));
-
- // Now we add the missing replica back
- DatanodeDetails targetDn = replicationManager.getLegacyReplicationManager()
- .getFirstDatanode(InflightType.REPLICATION, id);
- final ContainerReplica replicatedReplicaOne = getReplicas(
- id, State.CLOSED, 1000L, originNodeId, targetDn);
- containerStateManager.updateContainerReplica(
- id, replicatedReplicaOne);
-
- final long currentReplicationCommandCompleted = replicationManager
- .getMetrics().getNumReplicationCmdsCompleted();
- final long currentBytesCompleted = replicationManager.getMetrics()
- .getNumReplicationBytesCompleted();
-
- replicationManager.processAll();
- eventQueue.processAll(1000);
-
- Assertions.assertEquals(0, getInflightCount(InflightType.REPLICATION));
- Assertions.assertEquals(0, replicationManager.getMetrics()
- .getInflightReplication());
- Assertions.assertEquals(currentReplicationCommandCompleted + 1,
- replicationManager.getMetrics().getNumReplicationCmdsCompleted());
- Assertions.assertEquals(currentBytesCompleted + 100L,
- replicationManager.getMetrics().getNumReplicationBytesCompleted());
-
- report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
- Assertions.assertEquals(0, report.getStat(
- ReplicationManagerReport.HealthState.UNDER_REPLICATED));
- }
-
- /**
- * When a QUASI_CLOSED container is over replicated, ReplicationManager
- * deletes the excess replicas.
- */
- @Test
- public void testOverReplicatedQuasiClosedContainer()
- throws IOException, TimeoutException {
- final ContainerInfo container = getContainer(LifeCycleState.QUASI_CLOSED);
- container.setUsedBytes(101);
- final ContainerID id = container.containerID();
- final UUID originNodeId = UUID.randomUUID();
- final ContainerReplica replicaOne = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaTwo = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaThree = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaFour = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
-
- containerStateManager.addContainer(container.getProtobuf());
- containerStateManager.updateContainerReplica(id, replicaOne);
- containerStateManager.updateContainerReplica(id, replicaTwo);
- containerStateManager.updateContainerReplica(
- id, replicaThree);
- containerStateManager.updateContainerReplica(id, replicaFour);
-
- final int currentDeleteCommandCount = datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
-
- replicationManager.processAll();
- eventQueue.processAll(1000);
- Assertions.assertEquals(currentDeleteCommandCount + 1,
- datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand));
- Assertions.assertEquals(currentDeleteCommandCount + 1,
- replicationManager.getMetrics().getNumDeletionCmdsSent());
- Assertions.assertEquals(1, getInflightCount(InflightType.DELETION));
- Assertions.assertEquals(1, replicationManager.getMetrics()
- .getInflightDeletion());
-
- ReplicationManagerReport report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.OVER_REPLICATED));
-
- // Now we remove the replica according to inflight
- DatanodeDetails targetDn = replicationManager.getLegacyReplicationManager()
- .getFirstDatanode(InflightType.DELETION, id);
- if (targetDn.equals(replicaOne.getDatanodeDetails())) {
- containerStateManager.removeContainerReplica(
- id, replicaOne);
- } else if (targetDn.equals(replicaTwo.getDatanodeDetails())) {
- containerStateManager.removeContainerReplica(
- id, replicaTwo);
- } else if (targetDn.equals(replicaThree.getDatanodeDetails())) {
- containerStateManager.removeContainerReplica(
- id, replicaThree);
- } else if (targetDn.equals(replicaFour.getDatanodeDetails())) {
- containerStateManager.removeContainerReplica(
- id, replicaFour);
+ @Nested
+ class StableReplicas {
+ /**
+ * Open containers are not handled by ReplicationManager.
+ * This test-case makes sure that ReplicationManages doesn't take
+ * any action on OPEN containers.
+ */
+ @Test
+ public void testOpenContainer() throws IOException, TimeoutException {
+ final ContainerInfo container = getContainer(LifeCycleState.OPEN);
+ containerStateManager.addContainer(container.getProtobuf());
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ ReplicationManagerReport report = replicationManager.getContainerReport();
+ Assertions.assertEquals(1, report.getStat(LifeCycleState.OPEN));
+ Assertions.assertEquals(0, datanodeCommandHandler.getInvocation());
}
- final long currentDeleteCommandCompleted = replicationManager.getMetrics()
- .getNumDeletionCmdsCompleted();
- final long deleteBytesCompleted =
- replicationManager.getMetrics().getNumDeletionBytesCompleted();
-
- replicationManager.processAll();
- eventQueue.processAll(1000);
- Assertions.assertEquals(0, getInflightCount(InflightType.DELETION));
- Assertions.assertEquals(0, replicationManager.getMetrics()
- .getInflightDeletion());
- Assertions.assertEquals(currentDeleteCommandCompleted + 1,
- replicationManager.getMetrics().getNumDeletionCmdsCompleted());
- Assertions.assertEquals(deleteBytesCompleted + 101,
- replicationManager.getMetrics().getNumDeletionBytesCompleted());
-
- report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
- Assertions.assertEquals(0, report.getStat(
- ReplicationManagerReport.HealthState.OVER_REPLICATED));
- }
-
- /**
- * When a QUASI_CLOSED container is over replicated, ReplicationManager
- * deletes the excess replicas. While choosing the replica for deletion
- * ReplicationManager should prioritize unhealthy replica over QUASI_CLOSED
- * replica.
- */
- @Test
- public void testOverReplicatedQuasiClosedContainerWithUnhealthyReplica()
- throws IOException, TimeoutException {
- final ContainerInfo container = getContainer(LifeCycleState.QUASI_CLOSED);
- final ContainerID id = container.containerID();
- final UUID originNodeId = UUID.randomUUID();
- final ContainerReplica replicaOne = getReplicas(
- id, State.UNHEALTHY, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaTwo = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaThree = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaFour = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
-
- containerStateManager.addContainer(container.getProtobuf());
- containerStateManager.updateContainerReplica(id, replicaOne);
- containerStateManager.updateContainerReplica(id, replicaTwo);
- containerStateManager.updateContainerReplica(
- id, replicaThree);
- containerStateManager.updateContainerReplica(id, replicaFour);
-
- final int currentDeleteCommandCount = datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
-
- replicationManager.processAll();
- eventQueue.processAll(1000);
- Assertions.assertEquals(currentDeleteCommandCount + 1,
- datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand));
- Assertions.assertTrue(datanodeCommandHandler.received(
- SCMCommandProto.Type.deleteContainerCommand,
- replicaOne.getDatanodeDetails()));
- Assertions.assertEquals(currentDeleteCommandCount + 1,
- replicationManager.getMetrics().getNumDeletionCmdsSent());
- Assertions.assertEquals(1, getInflightCount(InflightType.DELETION));
- Assertions.assertEquals(1, replicationManager.getMetrics()
- .getInflightDeletion());
-
- ReplicationManagerReport report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.OVER_REPLICATED));
-
- final long currentDeleteCommandCompleted = replicationManager.getMetrics()
- .getNumDeletionCmdsCompleted();
- // Now we remove the replica to simulate deletion complete
- containerStateManager.removeContainerReplica(id, replicaOne);
-
- replicationManager.processAll();
- eventQueue.processAll(1000);
-
- Assertions.assertEquals(currentDeleteCommandCompleted + 1,
- replicationManager.getMetrics().getNumDeletionCmdsCompleted());
- Assertions.assertEquals(0, getInflightCount(InflightType.DELETION));
- Assertions.assertEquals(0, replicationManager.getMetrics()
- .getInflightDeletion());
-
- report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
- Assertions.assertEquals(0, report.getStat(
- ReplicationManagerReport.HealthState.OVER_REPLICATED));
- }
-
- /**
- * ReplicationManager should replicate an QUASI_CLOSED replica if it is
- * under replicated.
- */
- @Test
- public void testUnderReplicatedQuasiClosedContainer()
- throws IOException, TimeoutException {
- final ContainerInfo container = getContainer(LifeCycleState.QUASI_CLOSED);
- container.setUsedBytes(100);
- final ContainerID id = container.containerID();
- final UUID originNodeId = UUID.randomUUID();
- final ContainerReplica replicaOne = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaTwo = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
-
- containerStateManager.addContainer(container.getProtobuf());
- containerStateManager.updateContainerReplica(id, replicaOne);
- containerStateManager.updateContainerReplica(id, replicaTwo);
-
- final int currentReplicateCommandCount = datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.replicateContainerCommand);
- final long currentBytesToReplicate = replicationManager.getMetrics()
- .getNumReplicationBytesTotal();
-
- replicationManager.processAll();
- eventQueue.processAll(1000);
- Assertions.assertEquals(currentReplicateCommandCount + 1,
- datanodeCommandHandler.getInvocationCount(
- SCMCommandProto.Type.replicateContainerCommand));
- Assertions.assertEquals(currentReplicateCommandCount + 1,
- replicationManager.getMetrics().getNumReplicationCmdsSent());
- Assertions.assertEquals(currentBytesToReplicate + 100,
- replicationManager.getMetrics().getNumReplicationBytesTotal());
- Assertions.assertEquals(1, getInflightCount(InflightType.REPLICATION));
- Assertions.assertEquals(1, replicationManager.getMetrics()
- .getInflightReplication());
+ /**
+ * 1 open replica
+ * 2 closing replicas
+ * Expectation: Close command is sent to the open replicas.
+ */
+ @Test
+ public void testClosingContainer() throws IOException, TimeoutException {
+ final ContainerInfo container = getContainer(LifeCycleState.CLOSING);
+ final ContainerID id = container.containerID();
- ReplicationManagerReport report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.UNDER_REPLICATED));
+ containerStateManager.addContainer(container.getProtobuf());
- final long currentReplicateCommandCompleted = replicationManager
- .getMetrics().getNumReplicationCmdsCompleted();
- final long currentReplicateBytesCompleted = replicationManager
- .getMetrics().getNumReplicationBytesCompleted();
+ // Two replicas in CLOSING state
+ final Set<ContainerReplica> replicas = getReplicas(id, State.CLOSING,
+ randomDatanodeDetails(),
+ randomDatanodeDetails());
- // Now we add the replicated new replica
- DatanodeDetails targetDn = replicationManager.getLegacyReplicationManager()
- .getFirstDatanode(InflightType.REPLICATION, id);
- final ContainerReplica replicatedReplicaThree = getReplicas(
- id, State.CLOSED, 1000L, originNodeId, targetDn);
- containerStateManager.updateContainerReplica(
- id, replicatedReplicaThree);
+ // One replica in OPEN state
+ final DatanodeDetails datanode = randomDatanodeDetails();
+ replicas.addAll(getReplicas(id, State.OPEN, datanode));
- replicationManager.processAll();
- eventQueue.processAll(1000);
+ for (ContainerReplica replica : replicas) {
+ containerStateManager.updateContainerReplica(id, replica);
+ }
- Assertions.assertEquals(currentReplicateCommandCompleted + 1,
- replicationManager.getMetrics().getNumReplicationCmdsCompleted());
- Assertions.assertEquals(currentReplicateBytesCompleted + 100,
- replicationManager.getMetrics().getNumReplicationBytesCompleted());
- Assertions.assertEquals(0, getInflightCount(InflightType.REPLICATION));
- Assertions.assertEquals(0, replicationManager.getMetrics()
- .getInflightReplication());
-
- report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
- Assertions.assertEquals(0, report.getStat(
- ReplicationManagerReport.HealthState.UNDER_REPLICATED));
- }
+ final int currentCloseCommandCount = datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.closeContainerCommand);
- /**
- * When a QUASI_CLOSED container is under replicated, ReplicationManager
- * should re-replicate it. If there are any unhealthy replica, it has to
- * be deleted.
- *
- * In this test case, the container is QUASI_CLOSED and is under replicated
- * and also has an unhealthy replica.
- *
- * In the first iteration of ReplicationManager, it should re-replicate
- * the container so that it has enough replicas.
- *
- * In the second iteration, ReplicationManager should delete the unhealthy
- * replica.
- *
- * In the third iteration, ReplicationManager will re-replicate as the
- * container has again become under replicated after the unhealthy
- * replica has been deleted.
- *
- */
- @Test
- public void testUnderReplicatedQuasiClosedContainerWithUnhealthyReplica()
- throws IOException, InterruptedException,
- TimeoutException {
- final ContainerInfo container = getContainer(LifeCycleState.QUASI_CLOSED);
- container.setUsedBytes(99);
- final ContainerID id = container.containerID();
- final UUID originNodeId = UUID.randomUUID();
- final ContainerReplica replicaOne = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaTwo = getReplicas(
- id, State.UNHEALTHY, 1000L, originNodeId, randomDatanodeDetails());
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ Assertions.assertEquals(currentCloseCommandCount + 3,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.closeContainerCommand));
- containerStateManager.addContainer(container.getProtobuf());
- containerStateManager.updateContainerReplica(id, replicaOne);
- containerStateManager.updateContainerReplica(id, replicaTwo);
+ // Update the OPEN to CLOSING
+ for (ContainerReplica replica: getReplicas(id, State.CLOSING, datanode)) {
+ containerStateManager.updateContainerReplica(id, replica);
+ }
- final int currentReplicateCommandCount = datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.replicateContainerCommand);
- final int currentDeleteCommandCount = datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
- final long currentBytesToDelete = replicationManager.getMetrics()
- .getNumDeletionBytesTotal();
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ Assertions.assertEquals(currentCloseCommandCount + 6,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.closeContainerCommand));
+ ReplicationManagerReport report = replicationManager.getContainerReport();
+ Assertions.assertEquals(1, report.getStat(LifeCycleState.CLOSING));
+ }
- replicationManager.processAll();
- GenericTestUtils.waitFor(
- () -> (currentReplicateCommandCount + 1) == datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.replicateContainerCommand),
- 50, 5000);
+ @Test
+ public void testReplicateCommandTimeout()
+ throws IOException, TimeoutException {
+ long timeout = new ReplicationManagerConfiguration().getEventTimeout();
+
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ assertReplicaScheduled(1);
+
+ // Already a pending replica, so nothing scheduled
+ assertReplicaScheduled(0);
+
+ // Advance the clock past the timeout, and there should be a replica
+ // scheduled
+ clock.fastForward(timeout + 1000);
+ assertReplicaScheduled(1);
+ Assertions.assertEquals(1, replicationManager.getMetrics()
+ .getNumReplicationCmdsTimeout());
+ }
- Optional<CommandForDatanode> replicateCommand = datanodeCommandHandler
- .getReceivedCommands().stream()
- .filter(c -> c.getCommand().getType()
- .equals(SCMCommandProto.Type.replicateContainerCommand))
- .findFirst();
+ @Test
+ public void testDeleteCommandTimeout()
+ throws IOException, TimeoutException {
+ long timeout = new ReplicationManagerConfiguration().getEventTimeout();
+
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ assertDeleteScheduled(1);
+
+ // Already a pending replica, so nothing scheduled
+ assertReplicaScheduled(0);
+
+ // Advance the clock past the timeout, and there should be a replica
+ // scheduled
+ clock.fastForward(timeout + 1000);
+ assertDeleteScheduled(1);
+ Assertions.assertEquals(1, replicationManager.getMetrics()
+ .getNumDeletionCmdsTimeout());
+ }
- Assertions.assertTrue(replicateCommand.isPresent());
+ /**
+ * A closed empty container with all the replicas also closed and empty
+ * should be deleted.
+ * A container/ replica should be deemed empty when it has 0 keyCount even
+ * if the usedBytes is not 0 (usedBytes should not be used to determine if
+ * the container or replica is empty).
+ */
+ @Test
+ public void testDeleteEmptyContainer() throws Exception {
+ runTestDeleteEmptyContainer(3);
+ }
- DatanodeDetails newNode = createDatanodeDetails(
- replicateCommand.get().getDatanodeId());
- ContainerReplica newReplica = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, newNode);
- containerStateManager.updateContainerReplica(id, newReplica);
+ Void runTestDeleteEmptyContainer(int expectedDelete) throws Exception {
+ // Create container with usedBytes = 1000 and keyCount = 0
+ final ContainerInfo container = createContainer(
+ LifeCycleState.CLOSED, 1000, 0);
+ addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ // Create a replica with usedBytes != 0 and keyCount = 0
+ addReplica(container,
+ new NodeStatus(IN_SERVICE, HEALTHY), CLOSED, 100, 0);
+
+ assertDeleteScheduled(expectedDelete);
+ return null;
+ }
- ReplicationManagerReport report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.UNDER_REPLICATED));
- Assertions.assertEquals(0, report.getStat(
- ReplicationManagerReport.HealthState.UNHEALTHY));
+ @Test
+ public void testDeletionLimit() throws Exception {
+ runTestLimit(0, 2, 0, 1,
+ () -> runTestDeleteEmptyContainer(2));
+ }
- /*
- * We have report the replica to SCM, in the next ReplicationManager
- * iteration it should delete the unhealthy replica.
+ /**
+ * A closed empty container with a non-empty replica should not be deleted.
*/
+ @Test
+ public void testDeleteEmptyContainerNonEmptyReplica() throws Exception {
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED, 0,
+ 0);
+ addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ // Create the 3rd replica with non-zero key count and used bytes
+ addReplica(container,
+ new NodeStatus(IN_SERVICE, HEALTHY), CLOSED, 100, 1);
+ assertDeleteScheduled(0);
+ }
- replicationManager.processAll();
- eventQueue.processAll(1000);
- Assertions.assertEquals(currentDeleteCommandCount + 1,
- datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand));
- // ReplicaTwo should be deleted, that is the unhealthy one
- Assertions.assertTrue(datanodeCommandHandler.received(
- SCMCommandProto.Type.deleteContainerCommand,
- replicaTwo.getDatanodeDetails()));
- Assertions.assertEquals(currentDeleteCommandCount + 1,
- replicationManager.getMetrics().getNumDeletionCmdsSent());
- Assertions.assertEquals(currentBytesToDelete + 99,
- replicationManager.getMetrics().getNumDeletionBytesTotal());
- Assertions.assertEquals(1, getInflightCount(InflightType.DELETION));
- Assertions.assertEquals(1, replicationManager.getMetrics()
- .getInflightDeletion());
-
- containerStateManager.removeContainerReplica(id, replicaTwo);
-
- final long currentDeleteCommandCompleted = replicationManager.getMetrics()
- .getNumDeletionCmdsCompleted();
-
- report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
- Assertions.assertEquals(0, report.getStat(
- ReplicationManagerReport.HealthState.UNDER_REPLICATED));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.UNHEALTHY));
- /*
- * We have now removed unhealthy replica, next iteration of
- * ReplicationManager should re-replicate the container as it
- * is under replicated now
+ /**
+ * ReplicationManager should replicate zero replica when all copies
+ * are missing.
*/
-
- replicationManager.processAll();
- eventQueue.processAll(1000);
-
- Assertions.assertEquals(0, getInflightCount(InflightType.DELETION));
- Assertions.assertEquals(0, replicationManager.getMetrics()
- .getInflightDeletion());
- Assertions.assertEquals(currentDeleteCommandCompleted + 1,
- replicationManager.getMetrics().getNumDeletionCmdsCompleted());
-
- Assertions.assertEquals(currentReplicateCommandCount + 2,
- datanodeCommandHandler.getInvocationCount(
- SCMCommandProto.Type.replicateContainerCommand));
- Assertions.assertEquals(currentReplicateCommandCount + 2,
- replicationManager.getMetrics().getNumReplicationCmdsSent());
- Assertions.assertEquals(1, getInflightCount(InflightType.REPLICATION));
- Assertions.assertEquals(1, replicationManager.getMetrics()
- .getInflightReplication());
-
- report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.UNDER_REPLICATED));
- Assertions.assertEquals(0, report.getStat(
- ReplicationManagerReport.HealthState.UNHEALTHY));
- }
-
-
- /**
- * When a container is QUASI_CLOSED and it has >50% of its replica
- * in QUASI_CLOSED state with unique origin node id,
- * ReplicationManager should force close the replica(s) with
- * highest BCSID.
- */
- @Test
- public void testQuasiClosedToClosed() throws IOException, TimeoutException {
- final ContainerInfo container = getContainer(LifeCycleState.QUASI_CLOSED);
- final ContainerID id = container.containerID();
- final Set<ContainerReplica> replicas = getReplicas(id, State.QUASI_CLOSED,
- randomDatanodeDetails(),
- randomDatanodeDetails(),
- randomDatanodeDetails());
- containerStateManager.addContainer(container.getProtobuf());
- for (ContainerReplica replica : replicas) {
- containerStateManager.updateContainerReplica(id, replica);
+ @Test
+ public void testContainerWithMissingReplicas()
+ throws IOException, TimeoutException {
+ createContainer(LifeCycleState.CLOSED);
+ assertReplicaScheduled(0);
+ assertUnderReplicatedCount(1);
+ assertMissingCount(1);
}
- final int currentCloseCommandCount = datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.closeContainerCommand);
-
- replicationManager.processAll();
- eventQueue.processAll(1000);
+ /**
+ * 3 healthy closed replicas.
+ * Expectation: No action.
+ *
+ * ReplicationManager should not take any action if the container is
+ * CLOSED and healthy.
+ */
+ @Test
+ public void testHealthyClosedContainer()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
+ final ContainerID id = container.containerID();
+ final Set<ContainerReplica> replicas = getReplicas(id, State.CLOSED,
+ randomDatanodeDetails(),
+ randomDatanodeDetails(),
+ randomDatanodeDetails());
+
+ containerStateManager.addContainer(container.getProtobuf());
+ for (ContainerReplica replica : replicas) {
+ containerStateManager.updateContainerReplica(id, replica);
+ }
- // All the replicas have same BCSID, so all of them will be closed.
- Assertions.assertEquals(currentCloseCommandCount + 3, datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.closeContainerCommand));
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ Assertions.assertEquals(0, datanodeCommandHandler.getInvocation());
- ReplicationManagerReport report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
- Assertions.assertEquals(0, report.getStat(
- ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
+ ReplicationManagerReport report = replicationManager.getContainerReport();
+ Assertions.assertEquals(1, report.getStat(LifeCycleState.CLOSED));
+ for (ReplicationManagerReport.HealthState s :
+ ReplicationManagerReport.HealthState.values()) {
+ Assertions.assertEquals(0, report.getStat(s));
+ }
+ }
}
-
/**
- * ReplicationManager should not take any action if the container is
- * CLOSED and healthy.
+ * Tests replication manager with unhealthy and quasi-closed container
+ * replicas.
*/
- @Test
- public void testHealthyClosedContainer()
- throws IOException, TimeoutException {
- final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
- final ContainerID id = container.containerID();
- final Set<ContainerReplica> replicas = getReplicas(id, State.CLOSED,
- randomDatanodeDetails(),
- randomDatanodeDetails(),
- randomDatanodeDetails());
+ @Nested
+ class UnstableReplicas {
+ /**
+ * 2 open replicas
+ * 1 quasi-closed replica
+ * Expectation: close command is sent to the open replicas.
+ */
+ @Test
+ public void testQuasiClosedContainerWithTwoOpenReplica()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = getContainer(LifeCycleState.QUASI_CLOSED);
+ final ContainerID id = container.containerID();
+ final UUID originNodeId = UUID.randomUUID();
+ final ContainerReplica replicaOne = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaTwo = getReplicas(
+ id, State.OPEN, 1000L, originNodeId, randomDatanodeDetails());
+ final DatanodeDetails datanodeDetails = randomDatanodeDetails();
+ final ContainerReplica replicaThree = getReplicas(
+ id, State.OPEN, 1000L, datanodeDetails.getUuid(), datanodeDetails);
+
+ containerStateManager.addContainer(container.getProtobuf());
+ containerStateManager.updateContainerReplica(id, replicaOne);
+ containerStateManager.updateContainerReplica(id, replicaTwo);
+ containerStateManager.updateContainerReplica(
+ id, replicaThree);
- containerStateManager.addContainer(container.getProtobuf());
- for (ContainerReplica replica : replicas) {
- containerStateManager.updateContainerReplica(id, replica);
+ // First iteration
+
+ final int currentCloseCommandCount = datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.closeContainerCommand);
+ // Two of the replicas are in OPEN state
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ Assertions.assertEquals(currentCloseCommandCount + 2,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.closeContainerCommand));
+ Assertions.assertTrue(datanodeCommandHandler.received(
+ SCMCommandProto.Type.closeContainerCommand,
+ replicaTwo.getDatanodeDetails()));
+ Assertions.assertTrue(datanodeCommandHandler.received(
+ SCMCommandProto.Type.closeContainerCommand,
+ replicaThree.getDatanodeDetails()));
+ ReplicationManagerReport report = replicationManager.getContainerReport();
+ Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
}
- replicationManager.processAll();
- eventQueue.processAll(1000);
- Assertions.assertEquals(0, datanodeCommandHandler.getInvocation());
+ /**
+ * 3 quasi closed replicas with the same origin node ID.
+ * Expectation: No action taken.
+ *
+ * When the container is in QUASI_CLOSED state and all the replicas are
+ * also in QUASI_CLOSED state and doesn't have a quorum to force close
+ * the container, ReplicationManager will not do anything.
+ */
+ @Test
+ public void testHealthyQuasiClosedContainer()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = getContainer(LifeCycleState.QUASI_CLOSED);
+ final ContainerID id = container.containerID();
+ final UUID originNodeId = UUID.randomUUID();
+ final ContainerReplica replicaOne = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaTwo = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaThree = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+
+ containerStateManager.addContainer(container.getProtobuf());
+ containerStateManager.updateContainerReplica(id, replicaOne);
+ containerStateManager.updateContainerReplica(id, replicaTwo);
+ containerStateManager.updateContainerReplica(
+ id, replicaThree);
- ReplicationManagerReport report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.CLOSED));
- for (ReplicationManagerReport.HealthState s :
- ReplicationManagerReport.HealthState.values()) {
- Assertions.assertEquals(0, report.getStat(s));
+ // All the QUASI_CLOSED replicas have same originNodeId, so the
+ // container will not be closed. ReplicationManager should take no action.
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ Assertions.assertEquals(0, datanodeCommandHandler.getInvocation());
+ ReplicationManagerReport report = replicationManager.getContainerReport();
+ Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
}
- }
- /**
- * ReplicationManager should close the unhealthy OPEN container.
- */
- @Test
- public void testUnhealthyOpenContainer()
- throws IOException, TimeoutException {
- final ContainerInfo container = getContainer(LifeCycleState.OPEN);
- final ContainerID id = container.containerID();
- final Set<ContainerReplica> replicas = getReplicas(id, State.OPEN,
- randomDatanodeDetails(),
- randomDatanodeDetails());
- replicas.addAll(getReplicas(id, State.UNHEALTHY, randomDatanodeDetails()));
+ /**
+ * 2 quasi-closed replicas.
+ * 1 unhealthy replica.
+ * All replicas have same origin node ID.
+ * Expectation:
+ * Round 1: Quasi closed replica is replicated.
+ * Round 2: Unhealthy replica is deleted.
+ *
+ * When a container is QUASI_CLOSED and we don't have quorum to force close
+ * the container, the container should have all the replicas in QUASI_CLOSED
+ * state, else ReplicationManager will take action.
+ *
+ * In this test case we make one of the replica unhealthy, replication
+ * manager will send delete container command to the datanode which has the
+ * unhealthy replica.
+ */
+ @Test
+ public void testQuasiClosedContainerWithUnhealthyReplica()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = getContainer(LifeCycleState.QUASI_CLOSED);
+ container.setUsedBytes(100);
+ final ContainerID id = container.containerID();
+ final UUID originNodeId = UUID.randomUUID();
+ final ContainerReplica replicaOne = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaTwo = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaThree = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+
+ containerStateManager.addContainer(container.getProtobuf());
+ containerStateManager.updateContainerReplica(id, replicaOne);
+ containerStateManager.updateContainerReplica(id, replicaTwo);
+ containerStateManager.updateContainerReplica(id, replicaThree);
+
+ int currentReplicateCommandCount = datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.replicateContainerCommand);
+
+ // All the QUASI_CLOSED replicas have same originNodeId, so the
+ // container will not be closed. ReplicationManager should take no action.
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ Assertions.assertEquals(0, datanodeCommandHandler.getInvocation());
+
+ // Make the first replica unhealthy
+ final ContainerReplica unhealthyReplica = getReplicas(
+ id, UNHEALTHY, 1000L, originNodeId,
+ replicaOne.getDatanodeDetails());
+ containerStateManager.updateContainerReplica(
+ id, unhealthyReplica);
+
+ long currentBytesToReplicate = replicationManager.getMetrics()
+ .getNumReplicationBytesTotal();
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ // Under replication handler should first re-replicate one of the quasi
+ // closed containers.
+ // The unhealthy container should not have been deleted in the first pass.
+ assertDeleteScheduled(0);
+ currentReplicateCommandCount += 1;
+ currentBytesToReplicate += 100L;
+ Assertions.assertEquals(currentReplicateCommandCount,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.replicateContainerCommand));
+ Assertions.assertEquals(currentReplicateCommandCount,
+ replicationManager.getMetrics().getNumReplicationCmdsSent());
+ Assertions.assertEquals(currentBytesToReplicate,
+ replicationManager.getMetrics().getNumReplicationBytesTotal());
+ Assertions.assertEquals(1, getInflightCount(InflightType.REPLICATION));
+ Assertions.assertEquals(1, replicationManager.getMetrics()
+ .getInflightReplication());
+
+ // The quasi closed container cannot be closed, but it should have been
+ // restored to full replication on the previous run.
+ // The unhealthy replica should remain until the next iteration.
+ ReplicationManagerReport report = replicationManager.getContainerReport();
+ Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
+ Assertions.assertEquals(0, report.getStat(
+ ReplicationManagerReport.HealthState.UNDER_REPLICATED));
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.UNHEALTHY));
+
+ // Create the replica so replication manager sees it on the next run.
+ List<CommandForDatanode> replicateCommands = datanodeCommandHandler
+ .getReceivedCommands().stream()
+ .filter(c -> c.getCommand().getType()
+ .equals(SCMCommandProto.Type.replicateContainerCommand))
+ .collect(Collectors.toList());
+ for (CommandForDatanode replicateCommand: replicateCommands) {
+ DatanodeDetails newNode = createDatanodeDetails(
+ replicateCommand.getDatanodeId());
+ ContainerReplica newReplica = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, newNode);
+ containerStateManager.updateContainerReplica(id, newReplica);
+ }
- containerStateManager.addContainer(container.getProtobuf());
- for (ContainerReplica replica : replicas) {
- containerStateManager.updateContainerReplica(id, replica);
+ // On the next run, the unhealthy container should be scheduled for
+ // deletion, since the quasi closed container is now sufficiently
+ // replicated.
+ // This method runs an iteration of replication manager.
+ assertDeleteScheduled(1);
+ assertExactDeleteTargets(unhealthyReplica.getDatanodeDetails());
+ // Replication should have finished on the previous iteration, leaving
+ // these numbers unchanged.
+ Assertions.assertEquals(currentReplicateCommandCount,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.replicateContainerCommand));
+ Assertions.assertEquals(currentReplicateCommandCount,
+ replicationManager.getMetrics().getNumReplicationCmdsSent());
+ Assertions.assertEquals(currentBytesToReplicate,
+ replicationManager.getMetrics().getNumReplicationBytesTotal());
+ Assertions.assertEquals(0, getInflightCount(InflightType.REPLICATION));
+ Assertions.assertEquals(0, replicationManager.getMetrics()
+ .getInflightReplication());
+
+ // Now we will delete the unhealthy replica.
+ containerStateManager.removeContainerReplica(id, unhealthyReplica);
+
+ // There should be no work left on the following runs.
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ // The two commands shown are the previous delete and replicate commands.
+ Assertions.assertEquals(2, datanodeCommandHandler.getInvocation());
}
- final CloseContainerEventHandler closeContainerHandler =
- Mockito.mock(CloseContainerEventHandler.class);
- eventQueue.addHandler(SCMEvents.CLOSE_CONTAINER, closeContainerHandler);
-
- replicationManager.processAll();
- eventQueue.processAll(1000);
- Mockito.verify(closeContainerHandler, Mockito.times(1))
- .onMessage(id, eventQueue);
-
- ReplicationManagerReport report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.OPEN));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.OPEN_UNHEALTHY));
- }
- /**
- * ReplicationManager should skip send close command to unhealthy replica.
- */
- @Test
- public void testCloseUnhealthyReplica()
- throws IOException, TimeoutException {
- final ContainerInfo container = getContainer(LifeCycleState.CLOSING);
- final ContainerID id = container.containerID();
- final Set<ContainerReplica> replicas = getReplicas(id, State.UNHEALTHY,
- randomDatanodeDetails());
- replicas.addAll(getReplicas(id, State.OPEN, randomDatanodeDetails()));
- replicas.addAll(getReplicas(id, State.OPEN, randomDatanodeDetails()));
-
- containerStateManager.addContainer(container.getProtobuf());
- for (ContainerReplica replica : replicas) {
- containerStateManager.updateContainerReplica(id, replica);
+ /**
+ * Container is quasi closed.
+ * 3 quasi-closed replicas with the same origin node ID.
+ * 1 unhealthy replica with unique origin node ID.
+ *
+ * Expectation:
+ * No action taken. 3 healthy replicas are present. The unhealthy replica
+ * should not be deleted since it has a unique origin node ID. The
+ * container cannot be closed because there are not enough healthy unique
+ * origin node IDs.
+ */
+ @Test
+ public void testQuasiClosedContainerWithUniqueUnhealthyReplica()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = getContainer(LifeCycleState.QUASI_CLOSED);
+ container.setUsedBytes(100);
+ final ContainerID id = container.containerID();
+ final UUID originNodeId = UUID.randomUUID();
+ final ContainerReplica replicaOne = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaTwo = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaThree = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replica4 = getReplicas(
+ id, UNHEALTHY, 1000L, randomDatanodeDetails().getUuid(),
+ randomDatanodeDetails());
+
+ containerStateManager.addContainer(container.getProtobuf());
+ containerStateManager.updateContainerReplica(id, replicaOne);
+ containerStateManager.updateContainerReplica(id, replicaTwo);
+ containerStateManager.updateContainerReplica(id, replicaThree);
+ containerStateManager.updateContainerReplica(id, replica4);
+
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+
+ Assertions.assertEquals(0,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.replicateContainerCommand));
+
+ Assertions.assertEquals(0,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.deleteContainerCommand));
}
- replicationManager.processAll();
- // Wait for EventQueue to call the event handler
- eventQueue.processAll(1000);
- Assertions.assertEquals(2, datanodeCommandHandler.getInvocation());
- }
-
- @Test
- public void testGeneratedConfig() {
- ReplicationManagerConfiguration rmc =
- OzoneConfiguration.newInstanceOf(ReplicationManagerConfiguration.class);
-
- //default is not included in ozone-site.xml but generated from annotation
- //to the ozone-site-generated.xml which should be loaded by the
- // OzoneConfiguration.
- Assertions.assertEquals(1800000, rmc.getEventTimeout());
+ /**
+ * Container is closed.
+ * 2 quasi-closed replicas.
+ * 1 unhealthy replica.
+ * All replicas have unique origin node IDs.
+ * Quasi closed replicas BCS IDs match closed container's BCS ID.
+ *
+ * Expectation:
+ * Iteration 1: Quasi closed replicas are closed since their BCS IDs
+ * match the closed container state.
+ * Iteration 2: The now closed replicas are replicated.
+ * Iteration 3: The unhealthy replica is deleted.
+ */
+ @Test
+ public void testCloseableContainerWithUniqueUnhealthyReplica()
+ throws Exception {
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ ContainerReplica quasi1 = addReplicaToDn(container,
+ randomDatanodeDetails(), QUASI_CLOSED, container.getSequenceId());
+ ContainerReplica quasi2 = addReplicaToDn(container,
+ randomDatanodeDetails(), QUASI_CLOSED, container.getSequenceId());
+ ContainerReplica unhealthyReplica = addReplicaToDn(container,
+ randomDatanodeDetails(),
+ UNHEALTHY,
+ 900L);
+
+ // First RM iteration.
+ // The quasi containers should be closed since their BCSIDs match the
+ // closed container's state.
+ assertDeleteScheduled(0);
+ // All the containers are unhealthy, so it will not be counted as under
+ // replicated.
+ assertUnderReplicatedCount(0);
+ Assertions.assertEquals(2,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.closeContainerCommand));
+
+ // Update RM with the results of the close commands.
+ ContainerReplica closedRep1 = getReplicas(
+ container.containerID(), CLOSED,
+ container.getSequenceId(), quasi1.getDatanodeDetails())
+ .stream().findFirst().get();
+ ContainerReplica closedRep2 = getReplicas(
+ container.containerID(), CLOSED,
+ container.getSequenceId(), quasi2.getDatanodeDetails())
+ .stream().findFirst().get();
+
+ containerStateManager.updateContainerReplica(container.containerID(),
+ closedRep1);
+ containerStateManager.updateContainerReplica(container.containerID(),
+ closedRep2);
+
+ // Second RM iteration
+ // Now that we have healthy replicas, they should be replicated.
+ assertDeleteScheduled(0);
+ assertUnderReplicatedCount(1);
+ Assertions.assertEquals(1,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.replicateContainerCommand));
+
+
+ // Process the replicate command and report the replica back to SCM.
+ List<CommandForDatanode> replicateCommands = datanodeCommandHandler
+ .getReceivedCommands().stream()
+ .filter(c -> c.getCommand().getType()
+ .equals(SCMCommandProto.Type.replicateContainerCommand))
+ .collect(Collectors.toList());
+
+ // Report the new replica to SCM.
+ for (CommandForDatanode replicateCommand: replicateCommands) {
+ DatanodeDetails newNode = createDatanodeDetails(
+ replicateCommand.getDatanodeId());
+ ContainerReplica newReplica = getReplicas(
+ container.containerID(), CLOSED,
+ container.getSequenceId(), newNode.getUuid(), newNode);
+ containerStateManager.updateContainerReplica(container.containerID(),
+ newReplica);
+ }
- }
+ // Third RM iteration
+ // The unhealthy replica can be deleted since we have 3 healthy copies
+ // of a closed container.
+ assertDeleteScheduled(1);
+ assertUnderReplicatedCount(0);
+ Assertions.assertEquals(1,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.deleteContainerCommand));
+ assertExactDeleteTargets(unhealthyReplica.getDatanodeDetails());
+ }
- @Test
- public void additionalReplicaScheduledWhenMisReplicated()
- throws IOException, TimeoutException {
- final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
- container.setUsedBytes(100);
- final ContainerID id = container.containerID();
- final UUID originNodeId = UUID.randomUUID();
- final ContainerReplica replicaOne = getReplicas(
- id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaTwo = getReplicas(
- id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaThree = getReplicas(
- id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- containerStateManager.addContainer(container.getProtobuf());
- containerStateManager.updateContainerReplica(id, replicaOne);
- containerStateManager.updateContainerReplica(id, replicaTwo);
- containerStateManager.updateContainerReplica(
- id, replicaThree);
-
- // Ensure a mis-replicated status is returned for any containers in this
- // test where there are 3 replicas. When there are 2 or 4 replicas
- // the status returned will be healthy.
- Mockito.when(ratisContainerPlacementPolicy.validateContainerPlacement(
- Mockito.argThat(list -> list.size() == 3),
- Mockito.anyInt()
- )).thenAnswer(invocation -> {
- return new ContainerPlacementStatusDefault(1, 2, 3);
- });
+ /**
+ * $numReplicas unhealthy replicas.
+ * Expectation: The remaining replicas are scheduled.
+ */
+ @ParameterizedTest
+ @ValueSource(ints = {1, 2})
+ public void testUnderReplicatedWithOnlyUnhealthyReplicas(int numReplicas)
+ throws Exception {
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ for (int i = 0; i < numReplicas; i++) {
+ addReplica(container, NodeStatus.inServiceHealthy(), UNHEALTHY);
+ }
+ int numReplicasNeeded = HddsProtos.ReplicationFactor.THREE_VALUE -
+ numReplicas;
+ assertReplicaScheduled(numReplicasNeeded);
+ assertUnderReplicatedCount(1);
+ }
- int currentReplicateCommandCount = datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.replicateContainerCommand);
- final long currentBytesToReplicate = replicationManager.getMetrics()
- .getNumReplicationBytesTotal();
+ /**
+ * 1 unhealthy replica.
+ * 4 closed replicas.
+ * Expectation:
+ * Iteration 1: The unhealthy replica should be deleted.
+ * Iteration 2: One of the closed replicas should be deleted.
+ */
+ @Test
+ public void testOverReplicatedClosedAndUnhealthy() throws Exception {
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ ContainerReplica unhealthy = addReplica(container,
+ NodeStatus.inServiceHealthy(), UNHEALTHY);
+ addReplica(container, NodeStatus.inServiceHealthy(), CLOSED);
+ addReplica(container, NodeStatus.inServiceHealthy(), CLOSED);
+ addReplica(container, NodeStatus.inServiceHealthy(), CLOSED);
+ addReplica(container, NodeStatus.inServiceHealthy(), CLOSED);
+
+ // This method does one run of replication manager.
+ assertReplicaScheduled(0);
+ assertUnderReplicatedCount(0);
+ boolean unhealthyDeleted = false;
+ boolean closedDeleted = false;
+ UUID closedDeletedUUID = null;
+
+ for (CommandForDatanode<?> command :
+ datanodeCommandHandler.getReceivedCommands()) {
+ if (command.getCommand().getType() ==
+ SCMCommandProto.Type.deleteContainerCommand) {
+ if (command.getDatanodeId() ==
+ unhealthy.getDatanodeDetails().getUuid()) {
+ unhealthyDeleted = true;
+ } else {
+ closedDeleted = true;
+ closedDeletedUUID = command.getDatanodeId();
+ }
+ }
+ }
- replicationManager.processAll();
- eventQueue.processAll(1000);
- // At this stage, due to the mocked calls to validateContainerPlacement
- // the policy will not be satisfied, and replication will be triggered.
+ Assertions.assertFalse(unhealthyDeleted);
+ Assertions.assertTrue(closedDeleted);
+
+ // Do a second run.
+ assertReplicaScheduled(0);
+ assertUnderReplicatedCount(0);
+ unhealthyDeleted = false;
+ closedDeleted = false;
+ for (CommandForDatanode<?> command :
+ datanodeCommandHandler.getReceivedCommands()) {
+ if (command.getCommand().getType() ==
+ SCMCommandProto.Type.deleteContainerCommand) {
+ if (command.getDatanodeId() ==
+ unhealthy.getDatanodeDetails().getUuid()) {
+ unhealthyDeleted = true;
+ } else {
+ closedDeleted = true;
+ // The delete command should have been left over from the last run.
+ Assertions.assertEquals(closedDeletedUUID, command.getDatanodeId());
+ }
+ }
+ }
- Assertions.assertEquals(currentReplicateCommandCount + 1,
- datanodeCommandHandler.getInvocationCount(
- SCMCommandProto.Type.replicateContainerCommand));
- Assertions.assertEquals(currentReplicateCommandCount + 1,
- replicationManager.getMetrics().getNumReplicationCmdsSent());
- Assertions.assertEquals(currentBytesToReplicate + 100,
- replicationManager.getMetrics().getNumReplicationBytesTotal());
- Assertions.assertEquals(1, getInflightCount(InflightType.REPLICATION));
- Assertions.assertEquals(1, replicationManager.getMetrics()
- .getInflightReplication());
+ Assertions.assertTrue(unhealthyDeleted);
+ Assertions.assertTrue(closedDeleted);
+ }
- ReplicationManagerReport report = replicationManager.getContainerReport();
- Assertions.assertEquals(1, report.getStat(LifeCycleState.CLOSED));
- Assertions.assertEquals(1, report.getStat(
- ReplicationManagerReport.HealthState.MIS_REPLICATED));
+ /**
+ * 4 unhealthy replicas.
+ * Expectation: One unhealthy replica should be deleted.
+ */
+ @Test
+ public void testOverReplicatedUnhealthy() throws Exception {
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ Set<UUID> unhealthyContainerDNIDs = new HashSet<>();
+
+ final int numReplicas = 4;
+ for (int i = 0; i < numReplicas; i++) {
+ ContainerReplica replica = addReplica(container,
+ NodeStatus.inServiceHealthy(), UNHEALTHY);
+ unhealthyContainerDNIDs.add(replica.getDatanodeDetails().getUuid());
+ }
- // Now make it so that all containers seem mis-replicated no matter how
- // many replicas. This will test replicas are not scheduled if the new
- // replica does not fix the mis-replication.
- Mockito.when(ratisContainerPlacementPolicy.validateContainerPlacement(
- Mockito.anyList(),
- Mockito.anyInt()
- )).thenAnswer(invocation -> {
- return new ContainerPlacementStatusDefault(1, 2, 3);
- });
+ // No replications should be scheduled.
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ Assertions.assertEquals(0,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.replicateContainerCommand));
+ assertUnderReplicatedCount(0);
+
+ // One replica should be deleted.
+ Assertions.assertEquals(1,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.deleteContainerCommand));
+ Assertions.assertTrue(
+ datanodeCommandHandler.getReceivedCommands().stream()
+ .anyMatch(c -> c.getCommand().getType() ==
+ SCMCommandProto.Type.deleteContainerCommand &&
+ unhealthyContainerDNIDs.contains(c.getDatanodeId())));
+ }
- currentReplicateCommandCount = datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.replicateContainerCommand);
+ /**
+ * 4 quasi-closed replicas.
+ * All have same origin node ID.
+ * Expectation: One of the replicas is deleted.
+ */
+ @Test
+ public void testOverReplicatedQuasiClosedContainer()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = getContainer(LifeCycleState.QUASI_CLOSED);
+ container.setUsedBytes(101);
+ final ContainerID id = container.containerID();
+ final UUID originNodeId = UUID.randomUUID();
+ final ContainerReplica replicaOne = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaTwo = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaThree = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaFour = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+
+ containerStateManager.addContainer(container.getProtobuf());
+ containerStateManager.updateContainerReplica(id, replicaOne);
+ containerStateManager.updateContainerReplica(id, replicaTwo);
+ containerStateManager.updateContainerReplica(
+ id, replicaThree);
+ containerStateManager.updateContainerReplica(id, replicaFour);
+
+ final int currentDeleteCommandCount = datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
+
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ Assertions.assertEquals(currentDeleteCommandCount + 1,
+ datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand));
+ Assertions.assertEquals(currentDeleteCommandCount + 1,
+ replicationManager.getMetrics().getNumDeletionCmdsSent());
+ Assertions.assertEquals(1, getInflightCount(InflightType.DELETION));
+ Assertions.assertEquals(1, replicationManager.getMetrics()
+ .getInflightDeletion());
+
+ ReplicationManagerReport report = replicationManager.getContainerReport();
+ Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.OVER_REPLICATED));
+
+ // Now we remove the replica according to inflight
+ DatanodeDetails targetDn =
+ replicationManager.getLegacyReplicationManager()
+ .getFirstDatanode(InflightType.DELETION, id);
+ if (targetDn.equals(replicaOne.getDatanodeDetails())) {
+ containerStateManager.removeContainerReplica(
+ id, replicaOne);
+ } else if (targetDn.equals(replicaTwo.getDatanodeDetails())) {
+ containerStateManager.removeContainerReplica(
+ id, replicaTwo);
+ } else if (targetDn.equals(replicaThree.getDatanodeDetails())) {
+ containerStateManager.removeContainerReplica(
+ id, replicaThree);
+ } else if (targetDn.equals(replicaFour.getDatanodeDetails())) {
+ containerStateManager.removeContainerReplica(
+ id, replicaFour);
+ }
- replicationManager.processAll();
- eventQueue.processAll(1000);
- // At this stage, due to the mocked calls to validateContainerPlacement
- // the mis-replicated racks will not have improved, so expect to see nothing
- // scheduled.
- Assertions.assertEquals(currentReplicateCommandCount, datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.replicateContainerCommand));
- Assertions.assertEquals(currentReplicateCommandCount,
- replicationManager.getMetrics().getNumReplicationCmdsSent());
- Assertions.assertEquals(1, getInflightCount(InflightType.REPLICATION));
- Assertions.assertEquals(1, replicationManager.getMetrics()
- .getInflightReplication());
- }
+ final long currentDeleteCommandCompleted = replicationManager.getMetrics()
+ .getNumDeletionCmdsCompleted();
+ final long deleteBytesCompleted =
+ replicationManager.getMetrics().getNumDeletionBytesCompleted();
+
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ Assertions.assertEquals(0, getInflightCount(InflightType.DELETION));
+ Assertions.assertEquals(0, replicationManager.getMetrics()
+ .getInflightDeletion());
+ Assertions.assertEquals(currentDeleteCommandCompleted + 1,
+ replicationManager.getMetrics().getNumDeletionCmdsCompleted());
+ Assertions.assertEquals(deleteBytesCompleted + 101,
+ replicationManager.getMetrics().getNumDeletionBytesCompleted());
+
+ report = replicationManager.getContainerReport();
+ Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
+ Assertions.assertEquals(0, report.getStat(
+ ReplicationManagerReport.HealthState.OVER_REPLICATED));
+ }
- @Test
- public void overReplicatedButRemovingMakesMisReplicated()
- throws IOException, TimeoutException {
- // In this test, the excess replica should not be removed.
- final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
- final ContainerID id = container.containerID();
- final UUID originNodeId = UUID.randomUUID();
- final ContainerReplica replicaOne = getReplicas(
- id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaTwo = getReplicas(
- id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaThree = getReplicas(
- id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaFour = getReplicas(
- id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaFive = getReplicas(
- id, State.UNHEALTHY, 1000L, originNodeId, randomDatanodeDetails());
+ /**
+ * 2 open replicas.
+ * 1 unhealthy replica.
+ * Expectation: Container is closed.
+ *
+ * ReplicationManager should close the unhealthy OPEN container.
+ */
+ @Test
+ public void testUnhealthyOpenContainer()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = getContainer(LifeCycleState.OPEN);
+ final ContainerID id = container.containerID();
+ final Set<ContainerReplica> replicas = getReplicas(id, State.OPEN,
+ randomDatanodeDetails(),
+ randomDatanodeDetails());
+ replicas.addAll(getReplicas(id, UNHEALTHY, randomDatanodeDetails()));
+
+ containerStateManager.addContainer(container.getProtobuf());
+ for (ContainerReplica replica : replicas) {
+ containerStateManager.updateContainerReplica(id, replica);
+ }
- containerStateManager.addContainer(container.getProtobuf());
- containerStateManager.updateContainerReplica(id, replicaOne);
- containerStateManager.updateContainerReplica(id, replicaTwo);
- containerStateManager.updateContainerReplica(
- id, replicaThree);
- containerStateManager.updateContainerReplica(id, replicaFour);
- containerStateManager.updateContainerReplica(id, replicaFive);
-
- // Ensure a mis-replicated status is returned for any containers in this
- // test where there are exactly 3 replicas checked.
- Mockito.when(ratisContainerPlacementPolicy.validateContainerPlacement(
- Mockito.argThat(list -> list.size() == 3),
- Mockito.anyInt()
- )).thenAnswer(
- invocation -> new ContainerPlacementStatusDefault(1, 2, 3));
+ final CloseContainerEventHandler closeContainerHandler =
+ Mockito.mock(CloseContainerEventHandler.class);
+ eventQueue.addHandler(SCMEvents.CLOSE_CONTAINER, closeContainerHandler);
- int currentDeleteCommandCount = datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ Mockito.verify(closeContainerHandler, Mockito.times(1))
+ .onMessage(id, eventQueue);
- replicationManager.processAll();
- eventQueue.processAll(1000);
- // The unhealthy replica should be removed, but not the other replica
- // as each time we test with 3 replicas, Mockito ensures it returns
- // mis-replicated
- Assertions.assertEquals(currentDeleteCommandCount + 1,
- datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand));
- Assertions.assertEquals(currentDeleteCommandCount + 1,
- replicationManager.getMetrics().getNumDeletionCmdsSent());
+ ReplicationManagerReport report = replicationManager.getContainerReport();
+ Assertions.assertEquals(1, report.getStat(LifeCycleState.OPEN));
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.OPEN_UNHEALTHY));
+ }
- Assertions.assertTrue(datanodeCommandHandler.received(
- SCMCommandProto.Type.deleteContainerCommand,
- replicaFive.getDatanodeDetails()));
- Assertions.assertEquals(1, getInflightCount(InflightType.DELETION));
- Assertions.assertEquals(1, replicationManager.getMetrics()
- .getInflightDeletion());
- assertOverReplicatedCount(1);
- }
+ /**
+ * 1 unhealthy replica.
+ * 2 open replicas.
+ * Expectation: Close command should be sent to open replicas only.
+ *
+ * ReplicationManager should skip send close command to unhealthy replica.
+ */
+ @Test
+ public void testCloseUnhealthyReplica()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = getContainer(LifeCycleState.CLOSING);
+ final ContainerID id = container.containerID();
+ final Set<ContainerReplica> replicas = getReplicas(id, UNHEALTHY,
+ randomDatanodeDetails());
+ replicas.addAll(getReplicas(id, State.OPEN, randomDatanodeDetails()));
+ replicas.addAll(getReplicas(id, State.OPEN, randomDatanodeDetails()));
+
+ containerStateManager.addContainer(container.getProtobuf());
+ for (ContainerReplica replica : replicas) {
+ containerStateManager.updateContainerReplica(id, replica);
+ }
- @Test
- public void testOverReplicatedAndPolicySatisfied()
- throws IOException, TimeoutException {
- final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
- final ContainerID id = container.containerID();
- final UUID originNodeId = UUID.randomUUID();
- final ContainerReplica replicaOne = getReplicas(
- id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaTwo = getReplicas(
- id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaThree = getReplicas(
- id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaFour = getReplicas(
- id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ replicationManager.processAll();
+ // Wait for EventQueue to call the event handler
+ eventQueue.processAll(1000);
+ Assertions.assertEquals(2,
+ datanodeCommandHandler.getInvocation());
+ }
- containerStateManager.addContainer(container.getProtobuf());
- containerStateManager.updateContainerReplica(id, replicaOne);
- containerStateManager.updateContainerReplica(id, replicaTwo);
- containerStateManager.updateContainerReplica(
- id, replicaThree);
- containerStateManager.updateContainerReplica(id, replicaFour);
+ /**
+ * 1 unhealthy replica.
+ * 3 quasi closed replicas.
+ * All have same origin node ID.
+ * Expectation: Unhealthy replica is deleted.
+ *
+ * When a QUASI_CLOSED container is over replicated, ReplicationManager
+ * deletes the excess replicas. While choosing the replica for deletion
+ * ReplicationManager should prioritize deleting the unhealthy replica over
+ * QUASI_CLOSED replica if the unhealthy replica does not have a unique
+ * origin node ID.
+ */
+ @Test
+ public void testQuasiClosedContainerWithExtraUnhealthyReplica()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = getContainer(LifeCycleState.QUASI_CLOSED);
+ final ContainerID id = container.containerID();
+ final UUID originNodeId = UUID.randomUUID();
+ final ContainerReplica unhealthyReplica = getReplicas(
+ id, UNHEALTHY, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaTwo = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaThree = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaFour = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+
+ containerStateManager.addContainer(container.getProtobuf());
+ containerStateManager.updateContainerReplica(id, unhealthyReplica);
+ containerStateManager.updateContainerReplica(id, replicaTwo);
+ containerStateManager.updateContainerReplica(id, replicaThree);
+ containerStateManager.updateContainerReplica(id, replicaFour);
+
+ assertDeleteScheduled(1);
+ Assertions.assertTrue(
+ datanodeCommandHandler.getReceivedCommands().stream()
+ .anyMatch(c -> c.getCommand().getType() ==
+ SCMCommandProto.Type.deleteContainerCommand &&
+ c.getDatanodeId().equals(
+ unhealthyReplica.getDatanodeDetails().getUuid())));
+
+ ReplicationManagerReport report = replicationManager.getContainerReport();
+ Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
+ // Container should have been considered over replicated including the
+ // unhealthy replica.
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.OVER_REPLICATED));
+
+ final long currentDeleteCommandCompleted = replicationManager.getMetrics()
+ .getNumDeletionCmdsCompleted();
+ // Now we remove the replica to simulate deletion complete
+ containerStateManager.removeContainerReplica(id, unhealthyReplica);
+
+ // On the next run, the over replicated status should be reconciled and
+ // the delete completed.
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+
+ Assertions.assertEquals(currentDeleteCommandCompleted + 1,
+ replicationManager.getMetrics().getNumDeletionCmdsCompleted());
+ Assertions.assertEquals(0, getInflightCount(InflightType.DELETION));
+ Assertions.assertEquals(0, replicationManager.getMetrics()
+ .getInflightDeletion());
+
+ report = replicationManager.getContainerReport();
+ Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
+ Assertions.assertEquals(0, report.getStat(
+ ReplicationManagerReport.HealthState.OVER_REPLICATED));
+ }
- Mockito.when(ratisContainerPlacementPolicy.validateContainerPlacement(
- Mockito.argThat(list -> list.size() == 3),
- Mockito.anyInt()
- )).thenAnswer(
- invocation -> new ContainerPlacementStatusDefault(2, 2, 3));
+ /**
+ * 2 quasi-closed replicas.
+ * Expectation: Replicate one of the replicas.
+ */
+ @Test
+ public void testUnderReplicatedQuasiClosedContainer()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = getContainer(LifeCycleState.QUASI_CLOSED);
+ container.setUsedBytes(100);
+ final ContainerID id = container.containerID();
+ final UUID originNodeId = UUID.randomUUID();
+ final ContainerReplica replicaOne = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaTwo = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+
+ containerStateManager.addContainer(container.getProtobuf());
+ containerStateManager.updateContainerReplica(id, replicaOne);
+ containerStateManager.updateContainerReplica(id, replicaTwo);
+
+ final int currentReplicateCommandCount = datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.replicateContainerCommand);
+ final long currentBytesToReplicate = replicationManager.getMetrics()
+ .getNumReplicationBytesTotal();
+
+ // On the first iteration, one of the quasi closed replicas should be
+ // replicated.
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ Assertions.assertEquals(currentReplicateCommandCount + 1,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.replicateContainerCommand));
+ Assertions.assertEquals(currentReplicateCommandCount + 1,
+ replicationManager.getMetrics().getNumReplicationCmdsSent());
+ Assertions.assertEquals(currentBytesToReplicate + 100,
+ replicationManager.getMetrics().getNumReplicationBytesTotal());
+ Assertions.assertEquals(1, getInflightCount(InflightType.REPLICATION));
+ Assertions.assertEquals(1, replicationManager.getMetrics()
+ .getInflightReplication());
+
+ ReplicationManagerReport report = replicationManager.getContainerReport();
+ Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.UNDER_REPLICATED));
+
+ final long currentReplicateCommandCompleted = replicationManager
+ .getMetrics().getNumReplicationCmdsCompleted();
+ final long currentReplicateBytesCompleted = replicationManager
+ .getMetrics().getNumReplicationBytesCompleted();
+
+ // Now we add the replicated new replica
+ DatanodeDetails targetDn =
+ replicationManager.getLegacyReplicationManager()
+ .getFirstDatanode(InflightType.REPLICATION, id);
+ final ContainerReplica replicatedReplicaThree = getReplicas(
+ id, State.QUASI_CLOSED, 1000L, originNodeId, targetDn);
+ containerStateManager.updateContainerReplica(
+ id, replicatedReplicaThree);
+
+ // On the next run, no additional replications should be scheduled.
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+
+ Assertions.assertEquals(currentReplicateCommandCompleted + 1,
+ replicationManager.getMetrics().getNumReplicationCmdsCompleted());
+ Assertions.assertEquals(currentReplicateBytesCompleted + 100,
+ replicationManager.getMetrics().getNumReplicationBytesCompleted());
+ Assertions.assertEquals(0, getInflightCount(InflightType.REPLICATION));
+ Assertions.assertEquals(0, replicationManager.getMetrics()
+ .getInflightReplication());
+
+ report = replicationManager.getContainerReport();
+ Assertions.assertEquals(1, report.getStat(LifeCycleState.QUASI_CLOSED));
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
+ Assertions.assertEquals(0, report.getStat(
+ ReplicationManagerReport.HealthState.UNDER_REPLICATED));
+ }
- final int currentDeleteCommandCount = datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
+ /**
+ * 1 quasi-closed replica.
+ * 1 unhealthy replica.
+ * All have same origin node ID.
+ * Expectation:
+ *
+ * In the first iteration of ReplicationManager, it should re-replicate
+ * the quasi closed replicas so that there are 3 of them.
+ *
+ * In the second iteration, ReplicationManager should delete the unhealthy
+ * replica since its origin node ID is not unique.
+ */
+ @Test
+ public void testUnderReplicatedQuasiClosedContainerWithUnhealthyReplica()
+ throws IOException, InterruptedException,
+ TimeoutException {
+ final ContainerInfo container = getContainer(LifeCycleState.QUASI_CLOSED);
+ container.setUsedBytes(99);
+ final ContainerID id = container.containerID();
+ final UUID originNodeId = UUID.randomUUID();
+ final ContainerReplica replicaOne = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId,
+ randomDatanodeDetails());
+ final ContainerReplica unhealthyReplica = getReplicas(
+ id, UNHEALTHY, 1000L, originNodeId,
+ randomDatanodeDetails());
+
+ containerStateManager.addContainer(container.getProtobuf());
+ containerStateManager.updateContainerReplica(id, replicaOne);
+ containerStateManager.updateContainerReplica(id, unhealthyReplica);
+
+ final int currentReplicateCommandCount = datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.replicateContainerCommand);
+ final int currentDeleteCommandCount = datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
+ final long currentBytesToDelete = replicationManager.getMetrics()
+ .getNumDeletionBytesTotal();
+
+ // Run first iteraiton
+
+ replicationManager.processAll();
+ GenericTestUtils.waitFor(
+ () -> (currentReplicateCommandCount + 2) == datanodeCommandHandler
+ .getInvocationCount(
+ SCMCommandProto.Type.replicateContainerCommand),
+ 50, 5000);
+
+ ReplicationManagerReport report = replicationManager.getContainerReport();
+ Assertions.assertEquals(1,
+ report.getStat(LifeCycleState.QUASI_CLOSED));
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.UNDER_REPLICATED));
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.UNHEALTHY));
+
+ List<CommandForDatanode> replicateCommands = datanodeCommandHandler
+ .getReceivedCommands().stream()
+ .filter(c -> c.getCommand().getType()
+ .equals(SCMCommandProto.Type.replicateContainerCommand))
+ .collect(Collectors.toList());
+
+ Assertions.assertEquals(2, replicateCommands.size());
+
+ // Report the two new replicas to SCM.
+ for (CommandForDatanode replicateCommand: replicateCommands) {
+ DatanodeDetails newNode = createDatanodeDetails(
+ replicateCommand.getDatanodeId());
+ ContainerReplica newReplica = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, newNode);
+ containerStateManager.updateContainerReplica(id, newReplica);
+ }
- replicationManager.processAll();
- eventQueue.processAll(1000);
- Assertions.assertEquals(currentDeleteCommandCount + 1,
- datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand));
- Assertions.assertEquals(currentDeleteCommandCount + 1,
- replicationManager.getMetrics().getNumDeletionCmdsSent());
- Assertions.assertEquals(1, getInflightCount(InflightType.DELETION));
- Assertions.assertEquals(1, replicationManager.getMetrics()
- .getInflightDeletion());
+ // Run second iteration.
+ // Now that the quasi closed replica is sufficiently replicated, SCM
+ // should delete the unhealthy replica on the next iteration.
- assertOverReplicatedCount(1);
- }
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ Assertions.assertEquals(currentDeleteCommandCount + 1,
+ datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand));
+ Assertions.assertTrue(datanodeCommandHandler.received(
+ SCMCommandProto.Type.deleteContainerCommand,
+ unhealthyReplica.getDatanodeDetails()));
+ Assertions.assertEquals(currentDeleteCommandCount + 1,
+ replicationManager.getMetrics().getNumDeletionCmdsSent());
+ Assertions.assertEquals(currentBytesToDelete + 99,
+ replicationManager.getMetrics().getNumDeletionBytesTotal());
+ Assertions.assertEquals(1,
+ getInflightCount(InflightType.DELETION));
+ Assertions.assertEquals(1, replicationManager.getMetrics()
+ .getInflightDeletion());
+
+ containerStateManager.removeContainerReplica(id, unhealthyReplica);
+
+ report = replicationManager.getContainerReport();
+ Assertions.assertEquals(1,
+ report.getStat(LifeCycleState.QUASI_CLOSED));
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
+ Assertions.assertEquals(0, report.getStat(
+ ReplicationManagerReport.HealthState.UNDER_REPLICATED));
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.UNHEALTHY));
+ }
- @Test
- public void testOverReplicatedAndPolicyUnSatisfiedAndDeleted()
- throws IOException, TimeoutException {
- final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
- final ContainerID id = container.containerID();
- final UUID originNodeId = UUID.randomUUID();
- final ContainerReplica replicaOne = getReplicas(
- id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaTwo = getReplicas(
- id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaThree = getReplicas(
- id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaFour = getReplicas(
- id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- final ContainerReplica replicaFive = getReplicas(
- id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
- containerStateManager.addContainer(container.getProtobuf());
- containerStateManager.updateContainerReplica(id, replicaOne);
- containerStateManager.updateContainerReplica(id, replicaTwo);
- containerStateManager.updateContainerReplica(
- id, replicaThree);
- containerStateManager.updateContainerReplica(id, replicaFour);
- containerStateManager.updateContainerReplica(id, replicaFive);
+ /**
+ * 3 quasi-closed replicas.
+ * All unique origin IDs.
+ * Expectation: Container is closed.
+ *
+ * When a container is QUASI_CLOSED and it has >50% of its replica
+ * in QUASI_CLOSED state with unique origin node id,
+ * ReplicationManager should force close the replica(s) with
+ * highest BCSID.
+ */
+ @Test
+ public void testQuasiClosedToClosed() throws IOException, TimeoutException {
+ final ContainerInfo container = getContainer(LifeCycleState.QUASI_CLOSED);
+ final ContainerID id = container.containerID();
+ final Set<ContainerReplica> replicas = getReplicas(id, QUASI_CLOSED,
+ randomDatanodeDetails(),
+ randomDatanodeDetails(),
+ randomDatanodeDetails());
+ containerStateManager.addContainer(container.getProtobuf());
+ for (ContainerReplica replica : replicas) {
+ containerStateManager.updateContainerReplica(id, replica);
+ }
- Mockito.when(ratisContainerPlacementPolicy.validateContainerPlacement(
- Mockito.argThat(list -> list != null && list.size() <= 4),
- Mockito.anyInt()
- )).thenAnswer(
- invocation -> new ContainerPlacementStatusDefault(1, 2, 3));
+ final int currentCloseCommandCount = datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.closeContainerCommand);
- final int currentDeleteCommandCount = datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
- replicationManager.processAll();
- eventQueue.processAll(1000);
- Assertions.assertEquals(currentDeleteCommandCount + 2,
- datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand));
- Assertions.assertEquals(currentDeleteCommandCount + 2,
- replicationManager.getMetrics().getNumDeletionCmdsSent());
- Assertions.assertEquals(1, getInflightCount(InflightType.DELETION));
- Assertions.assertEquals(1, replicationManager.getMetrics()
- .getInflightDeletion());
- }
+ // All the replicas have same BCSID, so all of them will be closed.
+ Assertions.assertEquals(currentCloseCommandCount + 3,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.closeContainerCommand));
- /**
- * ReplicationManager should replicate an additional replica if there are
- * decommissioned replicas.
- */
- @Test
- public void testUnderReplicatedDueToDecommission()
- throws IOException, TimeoutException {
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), CLOSED);
- assertReplicaScheduled(2);
- assertUnderReplicatedCount(1);
+ ReplicationManagerReport report = replicationManager.getContainerReport();
+ Assertions.assertEquals(1,
+ report.getStat(LifeCycleState.QUASI_CLOSED));
+ Assertions.assertEquals(0, report.getStat(
+ ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
+ }
}
/**
- * ReplicationManager should replicate an additional replica when all copies
- * are decommissioning.
+ * Tests replication manager handling of decommissioning and maintainence
+ * mode datanodes.
*/
- @Test
- public void testUnderReplicatedDueToAllDecommission()
- throws IOException, TimeoutException {
- runTestUnderReplicatedDueToAllDecommission(3);
- }
-
- Void runTestUnderReplicatedDueToAllDecommission(int expectedReplication)
- throws IOException, TimeoutException {
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
- addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), CLOSED);
- assertReplicaScheduled(expectedReplication);
- assertUnderReplicatedCount(1);
- return null;
- }
+ @Nested
+ class DecomAndMaintenance {
+ /**
+ * ReplicationManager should replicate an additional replica if there are
+ * decommissioned replicas.
+ */
+ @Test
+ public void testUnderReplicatedDueToDecommission()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), CLOSED);
+ assertReplicaScheduled(2);
+ assertUnderReplicatedCount(1);
+ }
- @Test
- public void testReplicationLimit() throws Exception {
- runTestLimit(1, 0, 2, 0,
- () -> runTestUnderReplicatedDueToAllDecommission(1));
- }
+ /**
+ * ReplicationManager should replicate an additional replica when all copies
+ * are decommissioning.
+ */
+ @Test
+ public void testUnderReplicatedDueToAllDecommission()
+ throws IOException, TimeoutException {
+ runTestUnderReplicatedDueToAllDecommission(3);
+ }
- void runTestLimit(int replicationLimit, int deletionLimit,
- int expectedReplicationSkipped, int expectedDeletionSkipped,
- Callable<Void> testcase) throws Exception {
- createReplicationManager(replicationLimit, deletionLimit);
+ Void runTestUnderReplicatedDueToAllDecommission(int expectedReplication)
+ throws IOException, TimeoutException {
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), CLOSED);
+ assertReplicaScheduled(expectedReplication);
+ assertUnderReplicatedCount(1);
+ return null;
+ }
- final ReplicationManagerMetrics metrics = replicationManager.getMetrics();
- final long replicationSkipped = metrics.getInflightReplicationSkipped();
- final long deletionSkipped = metrics.getInflightDeletionSkipped();
+ @Test
+ public void testReplicationLimit() throws Exception {
+ runTestLimit(1, 0, 2, 0,
+ () -> runTestUnderReplicatedDueToAllDecommission(1));
+ }
- testcase.call();
+ /**
+ * ReplicationManager should not take any action when the container is
+ * correctly replicated with decommissioned replicas still present.
+ */
+ @Test
+ public void testCorrectlyReplicatedWithDecommission()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), CLOSED);
+ assertReplicaScheduled(0);
+ assertUnderReplicatedCount(0);
+ }
- Assertions.assertEquals(replicationSkipped + expectedReplicationSkipped,
- metrics.getInflightReplicationSkipped());
- Assertions.assertEquals(deletionSkipped + expectedDeletionSkipped,
- metrics.getInflightDeletionSkipped());
+ /**
+ * ReplicationManager should replicate an additional replica when min rep
+ * is not met for maintenance.
+ */
+ @Test
+ public void testUnderReplicatedDueToMaintenance()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
+ assertReplicaScheduled(1);
+ assertUnderReplicatedCount(1);
+ }
- //reset limits for other tests.
- createReplicationManager(0, 0);
- }
+ /**
+ * ReplicationManager should not replicate an additional replica when if
+ * min replica for maintenance is 1 and another replica is available.
+ */
+ @Test
+ public void testNotUnderReplicatedDueToMaintenanceMinRepOne()
+ throws Exception {
+ replicationManager.stop();
+ ReplicationManagerConfiguration newConf =
+ new ReplicationManagerConfiguration();
+ newConf.setMaintenanceReplicaMinimum(1);
+ dbStore.close();
+ createReplicationManager(newConf);
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
+ assertReplicaScheduled(0);
+ assertUnderReplicatedCount(0);
+ }
- /**
- * ReplicationManager should not take any action when the container is
- * correctly replicated with decommissioned replicas still present.
- */
- @Test
- public void testCorrectlyReplicatedWithDecommission()
- throws IOException, TimeoutException {
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), CLOSED);
- assertReplicaScheduled(0);
- assertUnderReplicatedCount(0);
- }
+ /**
+ * ReplicationManager should replicate an additional replica when all copies
+ * are going off line and min rep is 1.
+ */
+ @Test
+ public void testUnderReplicatedDueToMaintenanceMinRepOne()
+ throws Exception {
+ replicationManager.stop();
+ ReplicationManagerConfiguration newConf =
+ new ReplicationManagerConfiguration();
+ newConf.setMaintenanceReplicaMinimum(1);
+ dbStore.close();
+ createReplicationManager(newConf);
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
+ assertReplicaScheduled(1);
+ assertUnderReplicatedCount(1);
+ }
- /**
- * ReplicationManager should replicate an additional replica when min rep
- * is not met for maintenance.
- */
- @Test
- public void testUnderReplicatedDueToMaintenance()
- throws IOException, TimeoutException {
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
- assertReplicaScheduled(1);
- assertUnderReplicatedCount(1);
- }
+ /**
+ * ReplicationManager should replicate additional replica when all copies
+ * are going into maintenance.
+ */
+ @Test
+ public void testUnderReplicatedDueToAllMaintenance()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
+ assertReplicaScheduled(2);
+ assertUnderReplicatedCount(1);
+ }
- /**
- * ReplicationManager should not replicate an additional replica when if
- * min replica for maintenance is 1 and another replica is available.
- */
- @Test
- public void testNotUnderReplicatedDueToMaintenanceMinRepOne()
- throws Exception {
- replicationManager.stop();
- ReplicationManagerConfiguration newConf =
- new ReplicationManagerConfiguration();
- newConf.setMaintenanceReplicaMinimum(1);
- dbStore.close();
- createReplicationManager(newConf);
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
- assertReplicaScheduled(0);
- assertUnderReplicatedCount(0);
- }
+ /**
+ * ReplicationManager should not replicate additional replica sufficient
+ * replica are available.
+ */
+ @Test
+ public void testCorrectlyReplicatedWithMaintenance()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
+ assertReplicaScheduled(0);
+ assertUnderReplicatedCount(0);
+ }
- /**
- * ReplicationManager should replicate an additional replica when all copies
- * are going off line and min rep is 1.
- */
- @Test
- public void testUnderReplicatedDueToMaintenanceMinRepOne()
- throws Exception {
- replicationManager.stop();
- ReplicationManagerConfiguration newConf =
- new ReplicationManagerConfiguration();
- newConf.setMaintenanceReplicaMinimum(1);
- dbStore.close();
- createReplicationManager(newConf);
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
- addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
- assertReplicaScheduled(1);
- assertUnderReplicatedCount(1);
- }
+ /**
+ * ReplicationManager should replicate additional replica when all copies
+ * are decommissioning or maintenance.
+ */
+ @Test
+ public void testUnderReplicatedWithDecommissionAndMaintenance()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ addReplica(container, new NodeStatus(DECOMMISSIONED, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(DECOMMISSIONED, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
+ assertReplicaScheduled(2);
+ assertUnderReplicatedCount(1);
+ }
- /**
- * ReplicationManager should replicate additional replica when all copies
- * are going into maintenance.
- */
- @Test
- public void testUnderReplicatedDueToAllMaintenance()
- throws IOException, TimeoutException {
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
- addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
- assertReplicaScheduled(2);
- assertUnderReplicatedCount(1);
- }
+ /**
+ * When a CLOSED container is over replicated, ReplicationManager
+ * deletes the excess replicas. While choosing the replica for deletion
+ * ReplicationManager should not attempt to remove a DECOMMISSION or
+ * MAINTENANCE replica.
+ */
+ @Test
+ public void testOverReplicatedClosedContainerWithDecomAndMaint()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ addReplica(container, NodeStatus.inServiceHealthy(), CLOSED);
+ addReplica(container, new NodeStatus(DECOMMISSIONED, HEALTHY), CLOSED);
+ addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
+ addReplica(container, NodeStatus.inServiceHealthy(), CLOSED);
+ addReplica(container, NodeStatus.inServiceHealthy(), CLOSED);
+ addReplica(container, NodeStatus.inServiceHealthy(), CLOSED);
+ addReplica(container, NodeStatus.inServiceHealthy(), CLOSED);
+
+ final int currentDeleteCommandCount = datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
+
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ Assertions.assertEquals(currentDeleteCommandCount + 2,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.deleteContainerCommand));
+ Assertions.assertEquals(currentDeleteCommandCount + 2,
+ replicationManager.getMetrics().getNumDeletionCmdsSent());
+ Assertions.assertEquals(1, getInflightCount(InflightType.DELETION));
+ Assertions.assertEquals(1, replicationManager.getMetrics()
+ .getInflightDeletion());
+ // Get the DECOM and Maint replica and ensure none of them are scheduled
+ // for removal
+ Set<ContainerReplica> decom =
+ containerStateManager.getContainerReplicas(
+ container.containerID())
+ .stream()
+ .filter(r ->
+ r.getDatanodeDetails().getPersistedOpState() != IN_SERVICE)
+ .collect(Collectors.toSet());
+ for (ContainerReplica r : decom) {
+ Assertions.assertFalse(datanodeCommandHandler.received(
+ SCMCommandProto.Type.deleteContainerCommand,
+ r.getDatanodeDetails()));
+ }
+ assertOverReplicatedCount(1);
+ }
- /**
- * ReplicationManager should not replicate additional replica sufficient
- * replica are available.
- */
- @Test
- public void testCorrectlyReplicatedWithMaintenance()
- throws IOException, TimeoutException {
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
- assertReplicaScheduled(0);
- assertUnderReplicatedCount(0);
+ /**
+ * Replication Manager should not attempt to replicate from an unhealthy
+ * (stale or dead) node. To test this, setup a scenario where a replia needs
+ * to be created, but mark all nodes stale. That way, no new replica will be
+ * scheduled.
+ */
+ @Test
+ public void testUnderReplicatedNotHealthySource()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ addReplica(container, NodeStatus.inServiceStale(), CLOSED);
+ addReplica(container, new NodeStatus(DECOMMISSIONED, STALE), CLOSED);
+ addReplica(container, new NodeStatus(DECOMMISSIONED, STALE), CLOSED);
+ // There should be replica scheduled, but as all nodes are stale, nothing
+ // gets scheduled.
+ assertReplicaScheduled(0);
+ assertUnderReplicatedCount(1);
+ }
}
/**
- * ReplicationManager should replicate additional replica when all copies
- * are decommissioning or maintenance.
+ * Tests replication manager move command.
*/
- @Test
- public void testUnderReplicatedWithDecommissionAndMaintenance()
- throws IOException, TimeoutException {
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
- addReplica(container, new NodeStatus(DECOMMISSIONED, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(DECOMMISSIONED, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
- assertReplicaScheduled(2);
- assertUnderReplicatedCount(1);
- }
+ @Nested
+ class Move {
+ /**
+ * if all the prerequisites are satisfied, move should work as expected.
+ */
+ @Test
+ public void testMove() throws IOException, NodeNotFoundException,
+ InterruptedException, ExecutionException, TimeoutException {
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ ContainerID id = container.containerID();
+ ContainerReplica dn1 = addReplica(container,
+ new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container,
+ new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container,
+ new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ DatanodeDetails dn3 = addNode(new NodeStatus(IN_SERVICE, HEALTHY));
+ CompletableFuture<MoveResult> cf =
+ replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
+ Assertions.assertTrue(scmLogs.getOutput().contains(
+ "receive a move request about container"));
+ Thread.sleep(100L);
+ Assertions.assertTrue(datanodeCommandHandler.received(
+ SCMCommandProto.Type.replicateContainerCommand, dn3));
+ Assertions.assertEquals(1, datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.replicateContainerCommand));
+
+ //replicate container to dn3
+ addReplicaToDn(container, dn3, CLOSED);
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+
+ Assertions.assertTrue(datanodeCommandHandler.received(
+ SCMCommandProto.Type.deleteContainerCommand,
+ dn1.getDatanodeDetails()));
+ Assertions.assertEquals(1, datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.deleteContainerCommand));
+ containerStateManager.removeContainerReplica(id, dn1);
- /**
- * ReplicationManager should replicate zero replica when all copies
- * are missing.
- */
- @Test
- public void testContainerWithMissingReplicas()
- throws IOException, TimeoutException {
- createContainer(LifeCycleState.CLOSED);
- assertReplicaScheduled(0);
- assertUnderReplicatedCount(1);
- assertMissingCount(1);
- }
- /**
- * When a CLOSED container is over replicated, ReplicationManager
- * deletes the excess replicas. While choosing the replica for deletion
- * ReplicationManager should not attempt to remove a DECOMMISSION or
- * MAINTENANCE replica.
- */
- @Test
- public void testOverReplicatedClosedContainerWithDecomAndMaint()
- throws IOException, TimeoutException {
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
- addReplica(container, NodeStatus.inServiceHealthy(), CLOSED);
- addReplica(container, new NodeStatus(DECOMMISSIONED, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED);
- addReplica(container, NodeStatus.inServiceHealthy(), CLOSED);
- addReplica(container, NodeStatus.inServiceHealthy(), CLOSED);
- addReplica(container, NodeStatus.inServiceHealthy(), CLOSED);
- addReplica(container, NodeStatus.inServiceHealthy(), CLOSED);
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
- final int currentDeleteCommandCount = datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
+ Assertions.assertTrue(cf.isDone() && cf.get() == MoveResult.COMPLETED);
+ }
- replicationManager.processAll();
- eventQueue.processAll(1000);
- Assertions.assertEquals(currentDeleteCommandCount + 2,
- datanodeCommandHandler
- .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand));
- Assertions.assertEquals(currentDeleteCommandCount + 2,
- replicationManager.getMetrics().getNumDeletionCmdsSent());
- Assertions.assertEquals(1, getInflightCount(InflightType.DELETION));
- Assertions.assertEquals(1, replicationManager.getMetrics()
- .getInflightDeletion());
- // Get the DECOM and Maint replica and ensure none of them are scheduled
- // for removal
- Set<ContainerReplica> decom =
- containerStateManager.getContainerReplicas(
- container.containerID())
- .stream()
- .filter(r -> r.getDatanodeDetails().getPersistedOpState() != IN_SERVICE)
- .collect(Collectors.toSet());
- for (ContainerReplica r : decom) {
+ /**
+ * if crash happened and restarted, move option should work as expected.
+ */
+ @Test
+ public void testMoveCrashAndRestart() throws IOException,
+ NodeNotFoundException, InterruptedException, TimeoutException {
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ ContainerID id = container.containerID();
+ ContainerReplica dn1 = addReplica(container,
+ new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container,
+ new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container,
+ new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ DatanodeDetails dn3 = addNode(new NodeStatus(IN_SERVICE, HEALTHY));
+ replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
+ Assertions.assertTrue(scmLogs.getOutput().contains(
+ "receive a move request about container"));
+ Thread.sleep(100L);
+ Assertions.assertTrue(datanodeCommandHandler.received(
+ SCMCommandProto.Type.replicateContainerCommand, dn3));
+ Assertions.assertEquals(1, datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.replicateContainerCommand));
+
+ //crash happens, restart scm.
+ //clear current inflight actions and reload inflightMove from DBStore.
+ resetReplicationManager();
+ replicationManager.getMoveScheduler()
+ .reinitialize(SCMDBDefinition.MOVE.getTable(dbStore));
+ Assertions.assertTrue(replicationManager.getMoveScheduler()
+ .getInflightMove().containsKey(id));
+ MoveDataNodePair kv = replicationManager.getMoveScheduler()
+ .getInflightMove().get(id);
+ Assertions.assertEquals(kv.getSrc(), dn1.getDatanodeDetails());
+ Assertions.assertEquals(kv.getTgt(), dn3);
+ serviceManager.notifyStatusChanged();
+
+ Thread.sleep(100L);
+ // now, the container is not over-replicated,
+ // so no deleteContainerCommand will be sent
Assertions.assertFalse(datanodeCommandHandler.received(
SCMCommandProto.Type.deleteContainerCommand,
- r.getDatanodeDetails()));
+ dn1.getDatanodeDetails()));
+ //replica does not exist in target datanode, so a
+ // replicateContainerCommand will be sent again at
+ // notifyStatusChanged#onLeaderReadyAndOutOfSafeMode
+ Assertions.assertEquals(2, datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.replicateContainerCommand));
+
+
+ //replicate container to dn3, now, over-replicated
+ addReplicaToDn(container, dn3, CLOSED);
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+
+ //deleteContainerCommand is sent, but the src replica is not deleted now
+ Assertions.assertEquals(1, datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.deleteContainerCommand));
+
+ //crash happens, restart scm.
+ //clear current inflight actions and reload inflightMove from DBStore.
+ resetReplicationManager();
+ replicationManager.getMoveScheduler()
+ .reinitialize(SCMDBDefinition.MOVE.getTable(dbStore));
+ Assertions.assertTrue(replicationManager.getMoveScheduler()
+ .getInflightMove().containsKey(id));
+ kv = replicationManager.getMoveScheduler()
+ .getInflightMove().get(id);
+ Assertions.assertEquals(kv.getSrc(), dn1.getDatanodeDetails());
+ Assertions.assertEquals(kv.getTgt(), dn3);
+ serviceManager.notifyStatusChanged();
+
+ //after restart and the container is over-replicated now,
+ //deleteContainerCommand will be sent again
+ Assertions.assertEquals(2, datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.deleteContainerCommand));
+ containerStateManager.removeContainerReplica(id, dn1);
+
+ //replica in src datanode is deleted now
+ containerStateManager.removeContainerReplica(id, dn1);
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+
+ //since the move is complete,so after scm crash and restart
+ //inflightMove should not contain the container again
+ resetReplicationManager();
+ replicationManager.getMoveScheduler()
+ .reinitialize(SCMDBDefinition.MOVE.getTable(dbStore));
+ Assertions.assertFalse(replicationManager.getMoveScheduler()
+ .getInflightMove().containsKey(id));
+
+ //completeableFuture is not stored in DB, so after scm crash and
+ //restart ,completeableFuture is missing
}
- assertOverReplicatedCount(1);
- }
-
- /**
- * Replication Manager should not attempt to replicate from an unhealthy
- * (stale or dead) node. To test this, setup a scenario where a replia needs
- * to be created, but mark all nodes stale. That way, no new replica will be
- * scheduled.
- */
- @Test
- public void testUnderReplicatedNotHealthySource()
- throws IOException, TimeoutException {
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
- addReplica(container, NodeStatus.inServiceStale(), CLOSED);
- addReplica(container, new NodeStatus(DECOMMISSIONED, STALE), CLOSED);
- addReplica(container, new NodeStatus(DECOMMISSIONED, STALE), CLOSED);
- // There should be replica scheduled, but as all nodes are stale, nothing
- // gets scheduled.
- assertReplicaScheduled(0);
- assertUnderReplicatedCount(1);
- }
-
- /**
- * if all the prerequisites are satisfied, move should work as expected.
- */
- @Test
- public void testMove() throws IOException, NodeNotFoundException,
- InterruptedException, ExecutionException, TimeoutException {
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
- ContainerID id = container.containerID();
- ContainerReplica dn1 = addReplica(container,
- new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container,
- new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container,
- new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- DatanodeDetails dn3 = addNode(new NodeStatus(IN_SERVICE, HEALTHY));
- CompletableFuture<MoveResult> cf =
- replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
- Assertions.assertTrue(scmLogs.getOutput().contains(
- "receive a move request about container"));
- Thread.sleep(100L);
- Assertions.assertTrue(datanodeCommandHandler.received(
- SCMCommandProto.Type.replicateContainerCommand, dn3));
- Assertions.assertEquals(1, datanodeCommandHandler.getInvocationCount(
- SCMCommandProto.Type.replicateContainerCommand));
- //replicate container to dn3
- addReplicaToDn(container, dn3, CLOSED);
- replicationManager.processAll();
- eventQueue.processAll(1000);
+ /**
+ * make sure RM does not delete replica if placement policy is not
+ * satisfied.
+ */
+ @Test
+ public void testMoveNotDeleteSrcIfPolicyNotSatisfied()
+ throws IOException, NodeNotFoundException,
+ InterruptedException, ExecutionException, TimeoutException {
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ ContainerID id = container.containerID();
+ ContainerReplica dn1 = addReplica(container,
+ new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ ContainerReplica dn2 = addReplica(container,
+ new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container,
+ new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ DatanodeDetails dn4 = addNode(new NodeStatus(IN_SERVICE, HEALTHY));
+ CompletableFuture<MoveResult> cf =
+ replicationManager.move(id, dn1.getDatanodeDetails(), dn4);
+ Assertions.assertTrue(scmLogs.getOutput().contains(
+ "receive a move request about container"));
+ Thread.sleep(100L);
+ Assertions.assertTrue(datanodeCommandHandler.received(
+ SCMCommandProto.Type.replicateContainerCommand, dn4));
+ Assertions.assertEquals(1, datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.replicateContainerCommand));
+
+ //replicate container to dn4
+ addReplicaToDn(container, dn4, CLOSED);
+ //now, replication succeeds, but replica in dn2 lost,
+ //and there are only tree replicas totally, so rm should
+ //not delete the replica on dn1
+ containerStateManager.removeContainerReplica(id, dn2);
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+
+ Assertions.assertFalse(
+ datanodeCommandHandler.received(
+ SCMCommandProto.Type.deleteContainerCommand,
+ dn1.getDatanodeDetails()));
+
+ Assertions.assertTrue(cf.isDone() &&
+ cf.get() == MoveResult.DELETE_FAIL_POLICY);
+ }
- Assertions.assertTrue(datanodeCommandHandler.received(
- SCMCommandProto.Type.deleteContainerCommand, dn1.getDatanodeDetails()));
- Assertions.assertEquals(1, datanodeCommandHandler.getInvocationCount(
- SCMCommandProto.Type.deleteContainerCommand));
- containerStateManager.removeContainerReplica(id, dn1);
- replicationManager.processAll();
- eventQueue.processAll(1000);
+ /**
+ * test src and target datanode become unhealthy when moving.
+ */
+ @Test
+ public void testDnBecameUnhealthyWhenMoving() throws IOException,
+ NodeNotFoundException, InterruptedException, ExecutionException,
+ TimeoutException {
+ final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
+ ContainerID id = container.containerID();
+ ContainerReplica dn1 = addReplica(container,
+ new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container,
+ new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ addReplica(container,
+ new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ DatanodeDetails dn3 = addNode(new NodeStatus(IN_SERVICE, HEALTHY));
+ CompletableFuture<MoveResult> cf =
+ replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
+ Assertions.assertTrue(scmLogs.getOutput().contains(
+ "receive a move request about container"));
+
+ nodeManager.setNodeStatus(dn3, new NodeStatus(IN_SERVICE, STALE));
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+
+ Assertions.assertTrue(cf.isDone() && cf.get() ==
+ MoveResult.REPLICATION_FAIL_NODE_UNHEALTHY);
+
+ nodeManager.setNodeStatus(dn3, new NodeStatus(IN_SERVICE, HEALTHY));
+ cf = replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
+ addReplicaToDn(container, dn3, CLOSED);
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ nodeManager.setNodeStatus(dn1.getDatanodeDetails(),
+ new NodeStatus(IN_SERVICE, STALE));
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+
+ Assertions.assertTrue(cf.isDone() && cf.get() ==
+ MoveResult.DELETION_FAIL_NODE_UNHEALTHY);
+ }
- Assertions.assertTrue(cf.isDone() && cf.get() == MoveResult.COMPLETED);
+ /**
+ * before Replication Manager generates a completablefuture for a move
+ * option, some Prerequisites should be satisfied.
+ */
+ @Test
+ public void testMovePrerequisites() throws IOException,
+ NodeNotFoundException, InterruptedException, ExecutionException,
+ InvalidStateTransitionException, TimeoutException {
+ //all conditions is met
+ final ContainerInfo container = createContainer(LifeCycleState.OPEN);
+ ContainerID id = container.containerID();
+ ContainerReplica dn1 = addReplica(container,
+ new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ ContainerReplica dn2 = addReplica(container,
+ new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+ DatanodeDetails dn3 = addNode(new NodeStatus(IN_SERVICE, HEALTHY));
+ ContainerReplica dn4 = addReplica(container,
+ new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
+
+ CompletableFuture<MoveResult> cf;
+ //the above move is executed successfully, so there may be some item in
+ //inflightReplication or inflightDeletion. here we stop replication
+ // manager to clear these states, which may impact the tests below.
+ //we don't need a running replicationManamger now
+ replicationManager.stop();
+ Thread.sleep(100L);
+ cf = replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
+ Assertions.assertTrue(cf.isDone() && cf.get() ==
+ MoveResult.FAIL_NOT_RUNNING);
+ replicationManager.start();
+ Thread.sleep(100L);
+
+ //container in not in OPEN state
+ cf = replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
+ Assertions.assertTrue(cf.isDone() && cf.get() ==
+ MoveResult.REPLICATION_FAIL_CONTAINER_NOT_CLOSED);
+ //open -> closing
+ containerStateManager.updateContainerState(id.getProtobuf(),
+ LifeCycleEvent.FINALIZE);
+ cf = replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
+ Assertions.assertTrue(cf.isDone() && cf.get() ==
+ MoveResult.REPLICATION_FAIL_CONTAINER_NOT_CLOSED);
+ //closing -> quasi_closed
+ containerStateManager.updateContainerState(id.getProtobuf(),
+ LifeCycleEvent.QUASI_CLOSE);
+ cf = replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
+ Assertions.assertTrue(cf.isDone() && cf.get() ==
+ MoveResult.REPLICATION_FAIL_CONTAINER_NOT_CLOSED);
+
+ //quasi_closed -> closed
+ containerStateManager.updateContainerState(id.getProtobuf(),
+ LifeCycleEvent.FORCE_CLOSE);
+ Assertions.assertSame(LifeCycleState.CLOSED,
+ containerStateManager.getContainer(id).getState());
+
+ //Node is not in healthy state
+ for (HddsProtos.NodeState state : HddsProtos.NodeState.values()) {
+ if (state != HEALTHY) {
+ nodeManager.setNodeStatus(dn3,
+ new NodeStatus(IN_SERVICE, state));
+ cf = replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
+ Assertions.assertTrue(cf.isDone() && cf.get() ==
+ MoveResult.REPLICATION_FAIL_NODE_UNHEALTHY);
+ cf = replicationManager.move(id, dn3, dn1.getDatanodeDetails());
+ Assertions.assertTrue(cf.isDone() && cf.get() ==
+ MoveResult.REPLICATION_FAIL_NODE_UNHEALTHY);
+ }
+ }
+ nodeManager.setNodeStatus(dn3, new NodeStatus(IN_SERVICE, HEALTHY));
+
+ //Node is not in IN_SERVICE state
+ for (HddsProtos.NodeOperationalState state :
+ HddsProtos.NodeOperationalState.values()) {
+ if (state != IN_SERVICE) {
+ nodeManager.setNodeStatus(dn3,
+ new NodeStatus(state, HEALTHY));
+ cf = replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
+ Assertions.assertTrue(cf.isDone() && cf.get() ==
+ MoveResult.REPLICATION_FAIL_NODE_NOT_IN_SERVICE);
+ cf = replicationManager.move(id, dn3, dn1.getDatanodeDetails());
+ Assertions.assertTrue(cf.isDone() && cf.get() ==
+ MoveResult.REPLICATION_FAIL_NODE_NOT_IN_SERVICE);
+ }
+ }
+ nodeManager.setNodeStatus(dn3, new NodeStatus(IN_SERVICE, HEALTHY));
+
+ //container exists in target datanode
+ cf = replicationManager.move(id, dn1.getDatanodeDetails(),
+ dn2.getDatanodeDetails());
+ Assertions.assertTrue(cf.isDone() && cf.get() ==
+ MoveResult.REPLICATION_FAIL_EXIST_IN_TARGET);
+
+ //container does not exist in source datanode
+ cf = replicationManager.move(id, dn3, dn3);
+ Assertions.assertTrue(cf.isDone() && cf.get() ==
+ MoveResult.REPLICATION_FAIL_NOT_EXIST_IN_SOURCE);
+
+ //make container over relplicated to test the
+ // case that container is in inflightDeletion
+ ContainerReplica dn5 = addReplica(container,
+ new NodeStatus(IN_SERVICE, HEALTHY), State.CLOSED);
+ replicationManager.processAll();
+ //waiting for inflightDeletion generation
+ eventQueue.processAll(1000);
+ cf = replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
+ Assertions.assertTrue(cf.isDone() && cf.get() ==
+ MoveResult.REPLICATION_FAIL_INFLIGHT_DELETION);
+ resetReplicationManager();
+
+ //make the replica num be 2 to test the case
+ //that container is in inflightReplication
+ containerStateManager.removeContainerReplica(id, dn5);
+ containerStateManager.removeContainerReplica(id, dn4);
+ //replication manager should generate inflightReplication
+ replicationManager.processAll();
+ //waiting for inflightReplication generation
+ eventQueue.processAll(1000);
+ cf = replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
+ Assertions.assertTrue(cf.isDone() && cf.get() ==
+ MoveResult.REPLICATION_FAIL_INFLIGHT_REPLICATION);
+ }
}
/**
- * if crash happened and restarted, move option should work as expected.
+ * Tests mis-replicated containers with rack topology information.
*/
- @Test
- public void testMoveCrashAndRestart() throws IOException,
- NodeNotFoundException, InterruptedException, TimeoutException {
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
- ContainerID id = container.containerID();
- ContainerReplica dn1 = addReplica(container,
- new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container,
- new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container,
- new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- DatanodeDetails dn3 = addNode(new NodeStatus(IN_SERVICE, HEALTHY));
- replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
- Assertions.assertTrue(scmLogs.getOutput().contains(
- "receive a move request about container"));
- Thread.sleep(100L);
- Assertions.assertTrue(datanodeCommandHandler.received(
- SCMCommandProto.Type.replicateContainerCommand, dn3));
- Assertions.assertEquals(1, datanodeCommandHandler.getInvocationCount(
- SCMCommandProto.Type.replicateContainerCommand));
-
- //crash happens, restart scm.
- //clear current inflight actions and reload inflightMove from DBStore.
- resetReplicationManager();
- replicationManager.getMoveScheduler()
- .reinitialize(SCMDBDefinition.MOVE.getTable(dbStore));
- Assertions.assertTrue(replicationManager.getMoveScheduler()
- .getInflightMove().containsKey(id));
- MoveDataNodePair kv = replicationManager.getMoveScheduler()
- .getInflightMove().get(id);
- Assertions.assertEquals(kv.getSrc(), dn1.getDatanodeDetails());
- Assertions.assertEquals(kv.getTgt(), dn3);
- serviceManager.notifyStatusChanged();
+ @Nested
+ class MisReplicated {
+
+ @Test
+ public void additionalReplicaScheduledWhenMisReplicated()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
+ container.setUsedBytes(100);
+ final ContainerID id = container.containerID();
+ final UUID originNodeId = UUID.randomUUID();
+ final ContainerReplica replicaOne = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaTwo = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaThree = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+
+ containerStateManager.addContainer(container.getProtobuf());
+ containerStateManager.updateContainerReplica(id, replicaOne);
+ containerStateManager.updateContainerReplica(id, replicaTwo);
+ containerStateManager.updateContainerReplica(
+ id, replicaThree);
+
+ // Ensure a mis-replicated status is returned for any containers in this
+ // test where there are 3 replicas. When there are 2 or 4 replicas
+ // the status returned will be healthy.
+ Mockito.when(ratisContainerPlacementPolicy.validateContainerPlacement(
+ Mockito.argThat(list -> list.size() == 3),
+ Mockito.anyInt()
+ )).thenAnswer(invocation -> {
+ return new ContainerPlacementStatusDefault(1, 2, 3);
+ });
+
+ int currentReplicateCommandCount = datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.replicateContainerCommand);
+ final long currentBytesToReplicate = replicationManager.getMetrics()
+ .getNumReplicationBytesTotal();
+
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ // At this stage, due to the mocked calls to validateContainerPlacement
+ // the policy will not be satisfied, and replication will be triggered.
+
+ Assertions.assertEquals(currentReplicateCommandCount + 1,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.replicateContainerCommand));
+ Assertions.assertEquals(currentReplicateCommandCount + 1,
+ replicationManager.getMetrics().getNumReplicationCmdsSent());
+ Assertions.assertEquals(currentBytesToReplicate + 100,
+ replicationManager.getMetrics().getNumReplicationBytesTotal());
+ Assertions.assertEquals(1, getInflightCount(InflightType.REPLICATION));
+ Assertions.assertEquals(1, replicationManager.getMetrics()
+ .getInflightReplication());
+
+ ReplicationManagerReport report = replicationManager.getContainerReport();
+ Assertions.assertEquals(1, report.getStat(LifeCycleState.CLOSED));
+ Assertions.assertEquals(1, report.getStat(
+ ReplicationManagerReport.HealthState.MIS_REPLICATED));
+
+ // Now make it so that all containers seem mis-replicated no matter how
+ // many replicas. This will test replicas are not scheduled if the new
+ // replica does not fix the mis-replication.
+ Mockito.when(ratisContainerPlacementPolicy.validateContainerPlacement(
+ Mockito.anyList(),
+ Mockito.anyInt()
+ )).thenAnswer(invocation -> {
+ return new ContainerPlacementStatusDefault(1, 2, 3);
+ });
+
+ currentReplicateCommandCount = datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.replicateContainerCommand);
+
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ // At this stage, due to the mocked calls to validateContainerPlacement
+ // the mis-replicated racks will not have improved, so expect to see
+ // nothing scheduled.
+ Assertions.assertEquals(currentReplicateCommandCount,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.replicateContainerCommand));
+ Assertions.assertEquals(currentReplicateCommandCount,
+ replicationManager.getMetrics().getNumReplicationCmdsSent());
+ Assertions.assertEquals(1, getInflightCount(InflightType.REPLICATION));
+ Assertions.assertEquals(1, replicationManager.getMetrics()
+ .getInflightReplication());
+ }
- Thread.sleep(100L);
- // now, the container is not over-replicated,
- // so no deleteContainerCommand will be sent
- Assertions.assertFalse(datanodeCommandHandler.received(
- SCMCommandProto.Type.deleteContainerCommand, dn1.getDatanodeDetails()));
- //replica does not exist in target datanode, so a replicateContainerCommand
- //will be sent again at notifyStatusChanged#onLeaderReadyAndOutOfSafeMode
- Assertions.assertEquals(2, datanodeCommandHandler.getInvocationCount(
- SCMCommandProto.Type.replicateContainerCommand));
-
-
- //replicate container to dn3, now, over-replicated
- addReplicaToDn(container, dn3, CLOSED);
- replicationManager.processAll();
- eventQueue.processAll(1000);
+ @Test
+ public void overReplicatedButRemovingMakesMisReplicated()
+ throws IOException, TimeoutException {
+ // In this test, the excess replica should not be removed.
+ final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
+ final ContainerID id = container.containerID();
+ final UUID originNodeId = UUID.randomUUID();
+ final ContainerReplica replicaOne = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaTwo = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaThree = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaFour = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaFive = getReplicas(
+ id, UNHEALTHY, 1000L, originNodeId, randomDatanodeDetails());
+
+ containerStateManager.addContainer(container.getProtobuf());
+ containerStateManager.updateContainerReplica(id, replicaOne);
+ containerStateManager.updateContainerReplica(id, replicaTwo);
+ containerStateManager.updateContainerReplica(
+ id, replicaThree);
+ containerStateManager.updateContainerReplica(id, replicaFour);
+ containerStateManager.updateContainerReplica(id, replicaFive);
+
+ // Ensure a mis-replicated status is returned for any containers in this
+ // test where there are exactly 3 replicas checked.
+ Mockito.when(ratisContainerPlacementPolicy.validateContainerPlacement(
+ Mockito.argThat(list -> list.size() == 3),
+ Mockito.anyInt()
+ )).thenAnswer(
+ invocation -> new ContainerPlacementStatusDefault(1, 2, 3));
+
+ int currentDeleteCommandCount = datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
+
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ // TODO the new (non-legacy) RM needs a separate handler for
+ // topology status to make progress in this case by:
+ // 1. Deleting the closed replica to restore proper replica count.
+ // 2. Deleting the unhealthy replica since there are adequate healthy
+ // replicas.
+ // 3. Fixing topology issues left by the previous cleanup tasks.
+ // Current legacy RM implementation will take no action in this case
+ // because deletion would compromise topology.
+ Assertions.assertEquals(currentDeleteCommandCount,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.deleteContainerCommand));
+ Assertions.assertEquals(currentDeleteCommandCount,
+ replicationManager.getMetrics().getNumDeletionCmdsSent());
- //deleteContainerCommand is sent, but the src replica is not deleted now
- Assertions.assertEquals(1, datanodeCommandHandler.getInvocationCount(
- SCMCommandProto.Type.deleteContainerCommand));
-
- //crash happens, restart scm.
- //clear current inflight actions and reload inflightMove from DBStore.
- resetReplicationManager();
- replicationManager.getMoveScheduler()
- .reinitialize(SCMDBDefinition.MOVE.getTable(dbStore));
- Assertions.assertTrue(replicationManager.getMoveScheduler()
- .getInflightMove().containsKey(id));
- kv = replicationManager.getMoveScheduler()
- .getInflightMove().get(id);
- Assertions.assertEquals(kv.getSrc(), dn1.getDatanodeDetails());
- Assertions.assertEquals(kv.getTgt(), dn3);
- serviceManager.notifyStatusChanged();
+ Assertions.assertFalse(datanodeCommandHandler.received(
+ SCMCommandProto.Type.deleteContainerCommand,
+ replicaFive.getDatanodeDetails()));
+ Assertions.assertEquals(0, getInflightCount(InflightType.DELETION));
+ Assertions.assertEquals(0, replicationManager.getMetrics()
+ .getInflightDeletion());
+ assertOverReplicatedCount(1);
+ }
- //after restart and the container is over-replicated now,
- //deleteContainerCommand will be sent again
- Assertions.assertEquals(2, datanodeCommandHandler.getInvocationCount(
- SCMCommandProto.Type.deleteContainerCommand));
- containerStateManager.removeContainerReplica(id, dn1);
+ @Test
+ public void testOverReplicatedAndPolicySatisfied()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
+ final ContainerID id = container.containerID();
+ final UUID originNodeId = UUID.randomUUID();
+ final ContainerReplica replicaOne = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaTwo = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaThree = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaFour = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+
+ containerStateManager.addContainer(container.getProtobuf());
+ containerStateManager.updateContainerReplica(id, replicaOne);
+ containerStateManager.updateContainerReplica(id, replicaTwo);
+ containerStateManager.updateContainerReplica(
+ id, replicaThree);
+ containerStateManager.updateContainerReplica(id, replicaFour);
+
+ Mockito.when(ratisContainerPlacementPolicy.validateContainerPlacement(
+ Mockito.argThat(list -> list.size() == 3),
+ Mockito.anyInt()
+ )).thenAnswer(
+ invocation -> new ContainerPlacementStatusDefault(2, 2, 3));
+
+ final int currentDeleteCommandCount = datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
+
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ Assertions.assertEquals(currentDeleteCommandCount + 1,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.deleteContainerCommand));
+ Assertions.assertEquals(currentDeleteCommandCount + 1,
+ replicationManager.getMetrics().getNumDeletionCmdsSent());
+ Assertions.assertEquals(1, getInflightCount(InflightType.DELETION));
+ Assertions.assertEquals(1, replicationManager.getMetrics()
+ .getInflightDeletion());
+
+ assertOverReplicatedCount(1);
+ }
- //replica in src datanode is deleted now
- containerStateManager.removeContainerReplica(id, dn1);
- replicationManager.processAll();
- eventQueue.processAll(1000);
+ @Test
+ public void testOverReplicatedAndPolicyUnSatisfiedAndDeleted()
+ throws IOException, TimeoutException {
+ final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
+ final ContainerID id = container.containerID();
+ final UUID originNodeId = UUID.randomUUID();
+ final ContainerReplica replicaOne = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaTwo = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaThree = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaFour = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaFive = getReplicas(
+ id, QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+
+ containerStateManager.addContainer(container.getProtobuf());
+ containerStateManager.updateContainerReplica(id, replicaOne);
+ containerStateManager.updateContainerReplica(id, replicaTwo);
+ containerStateManager.updateContainerReplica(
+ id, replicaThree);
+ containerStateManager.updateContainerReplica(id, replicaFour);
+ containerStateManager.updateContainerReplica(id, replicaFive);
+
+ Mockito.when(ratisContainerPlacementPolicy.validateContainerPlacement(
+ Mockito.argThat(list -> list != null && list.size() <= 4),
+ Mockito.anyInt()
+ )).thenAnswer(
+ invocation -> new ContainerPlacementStatusDefault(1, 2, 3));
+
+ int currentDeleteCommandCount = datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
+
+ // On the first run, RM will delete one of the extra closed replicas.
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ Assertions.assertEquals(currentDeleteCommandCount + 1,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.deleteContainerCommand));
+ Assertions.assertEquals(currentDeleteCommandCount + 1,
+ replicationManager.getMetrics().getNumDeletionCmdsSent());
+ Assertions.assertEquals(1, getInflightCount(InflightType.DELETION));
+ Assertions.assertEquals(1, replicationManager.getMetrics()
+ .getInflightDeletion());
+
+ assertAnyDeleteTargets(
+ replicaOne.getDatanodeDetails(),
+ replicaTwo.getDatanodeDetails(),
+ replicaThree.getDatanodeDetails(),
+ replicaFour.getDatanodeDetails()
+ );
+
+ currentDeleteCommandCount = datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
+
+ // One the second run, the container is now properly replicated when
+ // counting in flight deletes. This allows the quasi closed container to
+ // be deleted by the unhealthy container handler.
+ replicationManager.processAll();
+ eventQueue.processAll(1000);
+ Assertions.assertEquals(currentDeleteCommandCount + 1,
+ datanodeCommandHandler.getInvocationCount(
+ SCMCommandProto.Type.deleteContainerCommand));
+ Assertions.assertEquals(currentDeleteCommandCount + 1,
+ replicationManager.getMetrics().getNumDeletionCmdsSent());
+ Assertions.assertEquals(1, getInflightCount(InflightType.DELETION));
+ Assertions.assertEquals(1, replicationManager.getMetrics()
+ .getInflightDeletion());
+
+ assertDeleteTargetsContain(replicaFive.getDatanodeDetails());
+ }
+ }
- //since the move is complete,so after scm crash and restart
- //inflightMove should not contain the container again
- resetReplicationManager();
- replicationManager.getMoveScheduler()
- .reinitialize(SCMDBDefinition.MOVE.getTable(dbStore));
- Assertions.assertFalse(replicationManager.getMoveScheduler()
- .getInflightMove().containsKey(id));
+ void runTestLimit(int replicationLimit, int deletionLimit,
+ int expectedReplicationSkipped, int expectedDeletionSkipped,
+ Callable<Void> testcase) throws Exception {
+ createReplicationManager(replicationLimit, deletionLimit);
- //completeableFuture is not stored in DB, so after scm crash and
- //restart ,completeableFuture is missing
- }
+ final ReplicationManagerMetrics metrics = replicationManager.getMetrics();
+ final long replicationSkipped = metrics.getInflightReplicationSkipped();
+ final long deletionSkipped = metrics.getInflightDeletionSkipped();
- /**
- * make sure RM does not delete replica if placement policy is not satisfied.
- */
- @Test
- public void testMoveNotDeleteSrcIfPolicyNotSatisfied()
- throws IOException, NodeNotFoundException,
- InterruptedException, ExecutionException, TimeoutException {
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
- ContainerID id = container.containerID();
- ContainerReplica dn1 = addReplica(container,
- new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- ContainerReplica dn2 = addReplica(container,
- new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container,
- new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- DatanodeDetails dn4 = addNode(new NodeStatus(IN_SERVICE, HEALTHY));
- CompletableFuture<MoveResult> cf =
- replicationManager.move(id, dn1.getDatanodeDetails(), dn4);
- Assertions.assertTrue(scmLogs.getOutput().contains(
- "receive a move request about container"));
- Thread.sleep(100L);
- Assertions.assertTrue(datanodeCommandHandler.received(
- SCMCommandProto.Type.replicateContainerCommand, dn4));
- Assertions.assertEquals(1, datanodeCommandHandler.getInvocationCount(
- SCMCommandProto.Type.replicateContainerCommand));
-
- //replicate container to dn4
- addReplicaToDn(container, dn4, CLOSED);
- //now, replication succeeds, but replica in dn2 lost,
- //and there are only tree replicas totally, so rm should
- //not delete the replica on dn1
- containerStateManager.removeContainerReplica(id, dn2);
- replicationManager.processAll();
- eventQueue.processAll(1000);
+ testcase.call();
- Assertions.assertFalse(datanodeCommandHandler.received(
- SCMCommandProto.Type.deleteContainerCommand, dn1.getDatanodeDetails()));
+ Assertions.assertEquals(replicationSkipped + expectedReplicationSkipped,
+ metrics.getInflightReplicationSkipped());
+ Assertions.assertEquals(deletionSkipped + expectedDeletionSkipped,
+ metrics.getInflightDeletionSkipped());
- Assertions.assertTrue(cf.isDone() &&
- cf.get() == MoveResult.DELETE_FAIL_POLICY);
+ //reset limits for other tests.
+ createReplicationManager(0, 0);
}
-
/**
- * test src and target datanode become unhealthy when moving.
+ * Checks that the set of datanodes given delete commands exactly matches
+ * targetDNs.
*/
- @Test
- public void testDnBecameUnhealthyWhenMoving() throws IOException,
- NodeNotFoundException, InterruptedException, ExecutionException,
- TimeoutException {
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
- ContainerID id = container.containerID();
- ContainerReplica dn1 = addReplica(container,
- new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container,
- new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container,
- new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- DatanodeDetails dn3 = addNode(new NodeStatus(IN_SERVICE, HEALTHY));
- CompletableFuture<MoveResult> cf =
- replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
- Assertions.assertTrue(scmLogs.getOutput().contains(
- "receive a move request about container"));
-
- nodeManager.setNodeStatus(dn3, new NodeStatus(IN_SERVICE, STALE));
- replicationManager.processAll();
- eventQueue.processAll(1000);
+ private void assertExactDeleteTargets(DatanodeDetails... targetDNs) {
+ List<CommandForDatanode> deleteCommands = datanodeCommandHandler
+ .getReceivedCommands().stream()
+ .filter(c -> c.getCommand().getType() ==
+ SCMCommandProto.Type.deleteContainerCommand)
+ .collect(Collectors.toList());
- Assertions.assertTrue(cf.isDone() && cf.get() ==
- MoveResult.REPLICATION_FAIL_NODE_UNHEALTHY);
+ Assertions.assertEquals(targetDNs.length, deleteCommands.size());
- nodeManager.setNodeStatus(dn3, new NodeStatus(IN_SERVICE, HEALTHY));
- cf = replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
- addReplicaToDn(container, dn3, CLOSED);
- replicationManager.processAll();
- eventQueue.processAll(1000);
- nodeManager.setNodeStatus(dn1.getDatanodeDetails(),
- new NodeStatus(IN_SERVICE, STALE));
- replicationManager.processAll();
- eventQueue.processAll(1000);
+ Set<UUID> targetDNIDs = Arrays.stream(targetDNs)
+ .map(DatanodeDetails::getUuid)
+ .collect(Collectors.toSet());
+ Set<UUID> chosenDNIDs = deleteCommands.stream()
+ .map(CommandForDatanode::getDatanodeId)
+ .collect(Collectors.toSet());
- Assertions.assertTrue(cf.isDone() && cf.get() ==
- MoveResult.DELETION_FAIL_NODE_UNHEALTHY);
+ Assertions.assertEquals(targetDNIDs, chosenDNIDs);
}
/**
- * before Replication Manager generates a completablefuture for a move option,
- * some Prerequisites should be satisfied.
+ * Checks if the set of nodes with deletions scheduled were taken from the
+ * provided set of DNs.
*/
- @Test
- public void testMovePrerequisites() throws IOException, NodeNotFoundException,
- InterruptedException, ExecutionException,
- InvalidStateTransitionException, TimeoutException {
- //all conditions is met
- final ContainerInfo container = createContainer(LifeCycleState.OPEN);
- ContainerID id = container.containerID();
- ContainerReplica dn1 = addReplica(container,
- new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- ContainerReplica dn2 = addReplica(container,
- new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- DatanodeDetails dn3 = addNode(new NodeStatus(IN_SERVICE, HEALTHY));
- ContainerReplica dn4 = addReplica(container,
- new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
-
- CompletableFuture<MoveResult> cf;
- //the above move is executed successfully, so there may be some item in
- //inflightReplication or inflightDeletion. here we stop replication manager
- //to clear these states, which may impact the tests below.
- //we don't need a running replicationManamger now
- replicationManager.stop();
- Thread.sleep(100L);
- cf = replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
- Assertions.assertTrue(cf.isDone() && cf.get() ==
- MoveResult.FAIL_NOT_RUNNING);
- replicationManager.start();
- Thread.sleep(100L);
-
- //container in not in OPEN state
- cf = replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
- Assertions.assertTrue(cf.isDone() && cf.get() ==
- MoveResult.REPLICATION_FAIL_CONTAINER_NOT_CLOSED);
- //open -> closing
- containerStateManager.updateContainerState(id.getProtobuf(),
- LifeCycleEvent.FINALIZE);
- cf = replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
- Assertions.assertTrue(cf.isDone() && cf.get() ==
- MoveResult.REPLICATION_FAIL_CONTAINER_NOT_CLOSED);
- //closing -> quasi_closed
- containerStateManager.updateContainerState(id.getProtobuf(),
- LifeCycleEvent.QUASI_CLOSE);
- cf = replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
- Assertions.assertTrue(cf.isDone() && cf.get() ==
- MoveResult.REPLICATION_FAIL_CONTAINER_NOT_CLOSED);
-
- //quasi_closed -> closed
- containerStateManager.updateContainerState(id.getProtobuf(),
- LifeCycleEvent.FORCE_CLOSE);
- Assertions.assertSame(LifeCycleState.CLOSED,
- containerStateManager.getContainer(id).getState());
-
- //Node is not in healthy state
- for (HddsProtos.NodeState state : HddsProtos.NodeState.values()) {
- if (state != HEALTHY) {
- nodeManager.setNodeStatus(dn3,
- new NodeStatus(IN_SERVICE, state));
- cf = replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
- Assertions.assertTrue(cf.isDone() && cf.get() ==
- MoveResult.REPLICATION_FAIL_NODE_UNHEALTHY);
- cf = replicationManager.move(id, dn3, dn1.getDatanodeDetails());
- Assertions.assertTrue(cf.isDone() && cf.get() ==
- MoveResult.REPLICATION_FAIL_NODE_UNHEALTHY);
- }
- }
- nodeManager.setNodeStatus(dn3, new NodeStatus(IN_SERVICE, HEALTHY));
-
- //Node is not in IN_SERVICE state
- for (HddsProtos.NodeOperationalState state :
- HddsProtos.NodeOperationalState.values()) {
- if (state != IN_SERVICE) {
- nodeManager.setNodeStatus(dn3,
- new NodeStatus(state, HEALTHY));
- cf = replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
- Assertions.assertTrue(cf.isDone() && cf.get() ==
- MoveResult.REPLICATION_FAIL_NODE_NOT_IN_SERVICE);
- cf = replicationManager.move(id, dn3, dn1.getDatanodeDetails());
- Assertions.assertTrue(cf.isDone() && cf.get() ==
- MoveResult.REPLICATION_FAIL_NODE_NOT_IN_SERVICE);
- }
- }
- nodeManager.setNodeStatus(dn3, new NodeStatus(IN_SERVICE, HEALTHY));
-
- //container exists in target datanode
- cf = replicationManager.move(id, dn1.getDatanodeDetails(),
- dn2.getDatanodeDetails());
- Assertions.assertTrue(cf.isDone() && cf.get() ==
- MoveResult.REPLICATION_FAIL_EXIST_IN_TARGET);
-
- //container does not exist in source datanode
- cf = replicationManager.move(id, dn3, dn3);
- Assertions.assertTrue(cf.isDone() && cf.get() ==
- MoveResult.REPLICATION_FAIL_NOT_EXIST_IN_SOURCE);
-
- //make container over relplicated to test the
- // case that container is in inflightDeletion
- ContainerReplica dn5 = addReplica(container,
- new NodeStatus(IN_SERVICE, HEALTHY), State.CLOSED);
- replicationManager.processAll();
- //waiting for inflightDeletion generation
- eventQueue.processAll(1000);
- cf = replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
- Assertions.assertTrue(cf.isDone() && cf.get() ==
- MoveResult.REPLICATION_FAIL_INFLIGHT_DELETION);
- resetReplicationManager();
-
- //make the replica num be 2 to test the case
- //that container is in inflightReplication
- containerStateManager.removeContainerReplica(id, dn5);
- containerStateManager.removeContainerReplica(id, dn4);
- //replication manager should generate inflightReplication
- replicationManager.processAll();
- //waiting for inflightReplication generation
- eventQueue.processAll(1000);
- cf = replicationManager.move(id, dn1.getDatanodeDetails(), dn3);
- Assertions.assertTrue(cf.isDone() && cf.get() ==
- MoveResult.REPLICATION_FAIL_INFLIGHT_REPLICATION);
- }
+ private void assertAnyDeleteTargets(DatanodeDetails... validDeleteDNs) {
+ List<CommandForDatanode> deleteCommands = datanodeCommandHandler
+ .getReceivedCommands().stream()
+ .filter(c -> c.getCommand().getType() ==
+ SCMCommandProto.Type.deleteContainerCommand)
+ .collect(Collectors.toList());
- @Test
- public void testReplicateCommandTimeout()
- throws IOException, TimeoutException {
- long timeout = new ReplicationManagerConfiguration().getEventTimeout();
-
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- assertReplicaScheduled(1);
-
- // Already a pending replica, so nothing scheduled
- assertReplicaScheduled(0);
-
- // Advance the clock past the timeout, and there should be a replica
- // scheduled
- clock.fastForward(timeout + 1000);
- assertReplicaScheduled(1);
- Assertions.assertEquals(1, replicationManager.getMetrics()
- .getNumReplicationCmdsTimeout());
- }
+ Set<UUID> deleteCandidateIDs = Arrays.stream(validDeleteDNs)
+ .map(DatanodeDetails::getUuid)
+ .collect(Collectors.toSet());
+ Set<UUID> chosenDNIDs = deleteCommands.stream()
+ .map(CommandForDatanode::getDatanodeId)
+ .collect(Collectors.toSet());
- @Test
- public void testDeleteCommandTimeout()
- throws IOException, TimeoutException {
- long timeout = new ReplicationManagerConfiguration().getEventTimeout();
-
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- assertDeleteScheduled(1);
-
- // Already a pending replica, so nothing scheduled
- assertReplicaScheduled(0);
-
- // Advance the clock past the timeout, and there should be a replica
- // scheduled
- clock.fastForward(timeout + 1000);
- assertDeleteScheduled(1);
- Assertions.assertEquals(1, replicationManager.getMetrics()
- .getNumDeletionCmdsTimeout());
+ Assertions.assertTrue(deleteCandidateIDs.containsAll(chosenDNIDs));
}
/**
- * A closed empty container with all the replicas also closed and empty
- * should be deleted.
- * A container/ replica should be deemed empty when it has 0 keyCount even
- * if the usedBytes is not 0 (usedBytes should not be used to determine if
- * the container or replica is empty).
+ * Checks if the set of nodes with deletions scheduled contains all of the
+ * provided DNs.
*/
- @Test
- public void testDeleteEmptyContainer() throws Exception {
- runTestDeleteEmptyContainer(3);
- }
-
- Void runTestDeleteEmptyContainer(int expectedDelete) throws Exception {
- // Create container with usedBytes = 1000 and keyCount = 0
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED, 1000,
- 0);
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- // Create a replica with usedBytes != 0 and keyCount = 0
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED, 100, 0);
-
- assertDeleteScheduled(expectedDelete);
- return null;
- }
+ private void assertDeleteTargetsContain(DatanodeDetails... deleteDN) {
+ List<CommandForDatanode> deleteCommands = datanodeCommandHandler
+ .getReceivedCommands().stream()
+ .filter(c -> c.getCommand().getType() ==
+ SCMCommandProto.Type.deleteContainerCommand)
+ .collect(Collectors.toList());
- @Test
- public void testDeletionLimit() throws Exception {
- runTestLimit(0, 2, 0, 1,
- () -> runTestDeleteEmptyContainer(2));
- }
+ Set<UUID> deleteDNIDs = Arrays.stream(deleteDN)
+ .map(DatanodeDetails::getUuid)
+ .collect(Collectors.toSet());
+ Set<UUID> chosenDNIDs = deleteCommands.stream()
+ .map(CommandForDatanode::getDatanodeId)
+ .collect(Collectors.toSet());
- /**
- * A closed empty container with a non-empty replica should not be deleted.
- */
- @Test
- public void testDeleteEmptyContainerNonEmptyReplica() throws Exception {
- final ContainerInfo container = createContainer(LifeCycleState.CLOSED, 0,
- 0);
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
- // Create the 3rd replica with non-zero key count and used bytes
- addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED, 100, 1);
- assertDeleteScheduled(0);
+ Assertions.assertTrue(chosenDNIDs.containsAll(deleteDNIDs));
}
private ContainerInfo createContainer(LifeCycleState containerState)
@@ -2052,6 +2464,12 @@ public class TestLegacyReplicationManager {
private ContainerReplica addReplicaToDn(ContainerInfo container,
DatanodeDetails dn, State replicaState)
throws ContainerNotFoundException {
+ return addReplicaToDn(container, dn, replicaState, 1000L);
+ }
+
+ private ContainerReplica addReplicaToDn(ContainerInfo container,
+ DatanodeDetails dn, State replicaState, long bcsId)
+ throws ContainerNotFoundException {
// Using the same originID for all replica in the container set. If each
// replica has a unique originID, it causes problems in ReplicationManager
// when processing over-replicated containers.
@@ -2059,7 +2477,7 @@ public class TestLegacyReplicationManager {
UUID.nameUUIDFromBytes(Longs.toByteArray(container.getContainerID()));
final ContainerReplica replica = getReplicas(container.containerID(),
replicaState, container.getUsedBytes(), container.getNumberOfKeys(),
- 1000L, originNodeId, dn);
+ bcsId, originNodeId, dn);
containerStateManager
.updateContainerReplica(container.containerID(), replica);
return replica;
@@ -2124,16 +2542,6 @@ public class TestLegacyReplicationManager {
ReplicationManagerReport.HealthState.OVER_REPLICATED));
}
- @AfterEach
- public void teardown() throws Exception {
- containerStateManager.close();
- replicationManager.stop();
- if (dbStore != null) {
- dbStore.close();
- }
- FileUtils.deleteDirectory(testDir);
- }
-
private static class DatanodeCommandHandler implements
EventHandler<CommandForDatanode> {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org