You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by so...@apache.org on 2022/09/28 15:10:08 UTC
[ozone] branch master updated: HDDS-7263. Add a handler for Quasi Closed containers to RM (#3785)
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new c96a342448 HDDS-7263. Add a handler for Quasi Closed containers to RM (#3785)
c96a342448 is described below
commit c96a342448f0c5c03d8b824b471b9f21fcd475d6
Author: Siddhant Sangwan <si...@gmail.com>
AuthorDate: Wed Sep 28 20:40:02 2022 +0530
HDDS-7263. Add a handler for Quasi Closed containers to RM (#3785)
---
.../container/replication/ReplicationManager.java | 2 +
.../health/QuasiClosedContainerHandler.java | 131 ++++++++++++
.../container/replication/ReplicationTestUtil.java | 29 ++-
.../health/TestQuasiClosedContainerHandler.java | 238 +++++++++++++++++++++
4 files changed, 397 insertions(+), 3 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java
index fc8666cf21..85bbaf2c77 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hdds.scm.container.replication.health.ClosingContainerH
import org.apache.hadoop.hdds.scm.container.replication.health.ECReplicationCheckHandler;
import org.apache.hadoop.hdds.scm.container.replication.health.HealthCheck;
import org.apache.hadoop.hdds.scm.container.replication.health.OpenContainerHandler;
+import org.apache.hadoop.hdds.scm.container.replication.health.QuasiClosedContainerHandler;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.ha.SCMService;
@@ -207,6 +208,7 @@ public class ReplicationManager implements SCMService {
containerCheckChain = new OpenContainerHandler(this);
containerCheckChain
.addNext(new ClosingContainerHandler(this))
+ .addNext(new QuasiClosedContainerHandler(this))
.addNext(new ClosedWithMismatchedReplicasHandler(this))
.addNext(ecReplicationCheckHandler);
start();
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/QuasiClosedContainerHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/QuasiClosedContainerHandler.java
new file mode 100644
index 0000000000..449d0776e1
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/QuasiClosedContainerHandler.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.container.replication.health;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport;
+import org.apache.hadoop.hdds.scm.container.replication.ContainerCheckRequest;
+import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Class for handling containers that are in QUASI_CLOSED state. This will
+ * send commands to Datanodes to force close these containers if they satisfy
+ * the requirements to be force closed. Only meant for RATIS containers.
+ */
+public class QuasiClosedContainerHandler extends AbstractCheck {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(QuasiClosedContainerHandler.class);
+
+ private final ReplicationManager replicationManager;
+
+ public QuasiClosedContainerHandler(ReplicationManager replicationManager) {
+ this.replicationManager = replicationManager;
+ }
+
+ /**
+ * If possible, force closes the Ratis container in QUASI_CLOSED state.
+ * Replicas with the highest Sequence ID are selected to be closed.
+ * @param request ContainerCheckRequest object representing the container
+ * @return true if close commands were sent, otherwise false
+ */
+ @Override
+ public boolean handle(ContainerCheckRequest request) {
+ ContainerInfo containerInfo = request.getContainerInfo();
+ if (containerInfo.getReplicationType() !=
+ HddsProtos.ReplicationType.RATIS) {
+ return false;
+ }
+
+ if (containerInfo.getState() != HddsProtos.LifeCycleState.QUASI_CLOSED) {
+ return false;
+ }
+
+ Set<ContainerReplica> replicas = request.getContainerReplicas();
+ if (canForceCloseContainer(containerInfo, replicas)) {
+ forceCloseContainer(containerInfo, replicas);
+ return true;
+ } else {
+ request.getReport().incrementAndSample(
+ ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK,
+ containerInfo.containerID());
+ }
+ return false;
+ }
+
+ /**
+ * Returns true if more than 50% of the container replicas with unique
+ * originNodeId are in QUASI_CLOSED state.
+ *
+ * @param container Container to check
+ * @param replicas Set of ContainerReplicas
+ * @return true if we can force close the container, false otherwise
+ */
+ private boolean canForceCloseContainer(final ContainerInfo container,
+ final Set<ContainerReplica> replicas) {
+ final int replicationFactor =
+ container.getReplicationConfig().getRequiredNodes();
+ final long uniqueQuasiClosedReplicaCount = replicas.stream()
+ .filter(r -> r.getState() == State.QUASI_CLOSED)
+ .map(ContainerReplica::getOriginDatanodeId)
+ .distinct()
+ .count();
+ return uniqueQuasiClosedReplicaCount > (replicationFactor / 2);
+ }
+
+ /**
+ * Force close the container replica(s) with the highest Sequence ID.
+ *
+ * <p>
+ * Note: We should force close the container only if >50% (quorum)
+ * of replicas with unique originNodeId are in QUASI_CLOSED state.
+ * </p>
+ *
+ * @param container ContainerInfo
+ * @param replicas Set of ContainerReplicas
+ */
+ private void forceCloseContainer(final ContainerInfo container,
+ final Set<ContainerReplica> replicas) {
+ final List<ContainerReplica> quasiClosedReplicas = replicas.stream()
+ .filter(r -> r.getState() == State.QUASI_CLOSED)
+ .collect(Collectors.toList());
+
+ final Long sequenceId = quasiClosedReplicas.stream()
+ .map(ContainerReplica::getSequenceId)
+ .max(Long::compare)
+ .orElse(-1L);
+
+ LOG.info("Force closing container {} with BCSID {}, which is in " +
+ "QUASI_CLOSED state.", container.containerID(), sequenceId);
+
+ quasiClosedReplicas.stream()
+ .filter(r -> sequenceId != -1L)
+ .filter(replica -> replica.getSequenceId().equals(sequenceId))
+ .forEach(replica -> replicationManager.sendCloseContainerReplicaCommand(
+ container, replica.getDatanodeDetails(), true));
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationTestUtil.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationTestUtil.java
index 2b3883cdd7..5493b88a1b 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationTestUtil.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationTestUtil.java
@@ -37,6 +37,7 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.UUID;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.CLOSED;
@@ -83,13 +84,34 @@ public final class ReplicationTestUtil {
return replicas;
}
+ public static Set<ContainerReplica> createReplicasWithSameOrigin(
+ ContainerID containerID, ContainerReplicaProto.State replicaState,
+ int... indexes) {
+ Set<ContainerReplica> replicas = new HashSet<>();
+ UUID originNodeId = MockDatanodeDetails.randomDatanodeDetails().getUuid();
+ for (int i : indexes) {
+ replicas.add(createContainerReplica(
+ containerID, i, IN_SERVICE, replicaState,
+ MockDatanodeDetails.randomDatanodeDetails(), originNodeId));
+ }
+ return replicas;
+ }
+
public static ContainerReplica createContainerReplica(ContainerID containerID,
int replicaIndex, HddsProtos.NodeOperationalState opState,
ContainerReplicaProto.State replicaState) {
- ContainerReplica.ContainerReplicaBuilder builder
- = ContainerReplica.newBuilder();
DatanodeDetails datanodeDetails
= MockDatanodeDetails.randomDatanodeDetails();
+ return createContainerReplica(containerID, replicaIndex, opState,
+ replicaState, datanodeDetails, datanodeDetails.getUuid());
+ }
+
+ public static ContainerReplica createContainerReplica(ContainerID containerID,
+ int replicaIndex, HddsProtos.NodeOperationalState opState,
+ ContainerReplicaProto.State replicaState,
+ DatanodeDetails datanodeDetails, UUID originNodeId) {
+ ContainerReplica.ContainerReplicaBuilder builder
+ = ContainerReplica.newBuilder();
datanodeDetails.setPersistedOpState(opState);
builder.setContainerID(containerID);
builder.setReplicaIndex(replicaIndex);
@@ -98,10 +120,11 @@ public final class ReplicationTestUtil {
builder.setContainerState(replicaState);
builder.setDatanodeDetails(datanodeDetails);
builder.setSequenceId(0);
- builder.setOriginNodeId(datanodeDetails.getUuid());
+ builder.setOriginNodeId(originNodeId);
return builder.build();
}
+
public static ContainerInfo createContainerInfo(ReplicationConfig repConfig) {
return createContainerInfo(repConfig, 1, HddsProtos.LifeCycleState.CLOSED);
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestQuasiClosedContainerHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestQuasiClosedContainerHandler.java
new file mode 100644
index 0000000000..b2a7fce088
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestQuasiClosedContainerHandler.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.container.replication.health;
+
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport;
+import org.apache.hadoop.hdds.scm.container.replication.ContainerCheckRequest;
+import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager;
+import org.apache.hadoop.hdds.scm.container.replication.ReplicationTestUtil;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.OPEN;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.QUASI_CLOSED;
+import static org.apache.hadoop.hdds.scm.HddsTestUtils.getContainer;
+import static org.apache.hadoop.hdds.scm.HddsTestUtils.getReplicas;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.times;
+
+/**
+ * Tests for {@link QuasiClosedContainerHandler}. This handler is only meant
+ * to handle Ratis containers.
+ */
+public class TestQuasiClosedContainerHandler {
+ private ReplicationManager replicationManager;
+ private QuasiClosedContainerHandler quasiClosedContainerHandler;
+ private RatisReplicationConfig ratisReplicationConfig;
+
+ @BeforeEach
+ public void setup() {
+ ratisReplicationConfig = RatisReplicationConfig.getInstance(
+ HddsProtos.ReplicationFactor.THREE);
+ replicationManager = Mockito.mock(ReplicationManager.class);
+ quasiClosedContainerHandler =
+ new QuasiClosedContainerHandler(replicationManager);
+ }
+
+ @Test
+ public void testECContainerReturnsFalse() {
+ ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo(
+ new ECReplicationConfig(3, 2), 1, QUASI_CLOSED);
+ Set<ContainerReplica> containerReplicas = ReplicationTestUtil
+ .createReplicas(containerInfo.containerID(),
+ State.QUASI_CLOSED, 1, 2, 3);
+ ContainerCheckRequest request = new ContainerCheckRequest.Builder()
+ .setPendingOps(Collections.EMPTY_LIST)
+ .setReport(new ReplicationManagerReport())
+ .setContainerInfo(containerInfo)
+ .setContainerReplicas(containerReplicas)
+ .build();
+
+ Assertions.assertFalse(quasiClosedContainerHandler.handle(request));
+ Mockito.verify(replicationManager, times(0))
+ .sendCloseContainerReplicaCommand(any(), any(), anyBoolean());
+ }
+
+ @Test
+ public void testOpenContainerReturnsFalse() {
+ ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo(
+ ratisReplicationConfig, 1, OPEN);
+ Set<ContainerReplica> containerReplicas = ReplicationTestUtil
+ .createReplicas(containerInfo.containerID(),
+ State.OPEN, 0, 0, 0);
+ ContainerCheckRequest request = new ContainerCheckRequest.Builder()
+ .setPendingOps(Collections.EMPTY_LIST)
+ .setReport(new ReplicationManagerReport())
+ .setContainerInfo(containerInfo)
+ .setContainerReplicas(containerReplicas)
+ .build();
+
+ Assertions.assertFalse(quasiClosedContainerHandler.handle(request));
+ Mockito.verify(replicationManager, times(0))
+ .sendCloseContainerReplicaCommand(any(), any(), anyBoolean());
+ }
+
+ /**
+ * When a container is QUASI_CLOSED, and it has greater than 50% of its
+ * replicas in QUASI_CLOSED state with unique origin node id,
+ * the handler should send force close commands to the replica(s) with
+ * highest BCSID.
+ */
+ @Test
+ public void testQuasiClosedWithQuorumReturnsTrue() {
+ ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo(
+ ratisReplicationConfig, 1, QUASI_CLOSED);
+ Set<ContainerReplica> containerReplicas = ReplicationTestUtil
+ .createReplicas(containerInfo.containerID(),
+ State.QUASI_CLOSED, 0, 0);
+ ContainerReplica openReplica = ReplicationTestUtil.createContainerReplica(
+ containerInfo.containerID(), 0,
+ HddsProtos.NodeOperationalState.IN_SERVICE, State.OPEN);
+ containerReplicas.add(openReplica);
+ ContainerCheckRequest request = new ContainerCheckRequest.Builder()
+ .setPendingOps(Collections.EMPTY_LIST)
+ .setReport(new ReplicationManagerReport())
+ .setContainerInfo(containerInfo)
+ .setContainerReplicas(containerReplicas)
+ .build();
+
+ Assertions.assertTrue(quasiClosedContainerHandler.handle(request));
+ Mockito.verify(replicationManager, times(2))
+ .sendCloseContainerReplicaCommand(any(), any(), anyBoolean());
+ }
+
+ /**
+ * The replicas are QUASI_CLOSED, but all of them have the same origin node
+ * id. Since a quorum (greater than 50% of replicas with unique origin node
+ * ids in QUASI_CLOSED state) is not formed, the handler should return false.
+ */
+ @Test
+ public void testHealthyQuasiClosedContainerReturnsFalse() {
+ ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo(
+ ratisReplicationConfig, 1, QUASI_CLOSED);
+ Set<ContainerReplica> containerReplicas = ReplicationTestUtil
+ .createReplicasWithSameOrigin(containerInfo.containerID(),
+ State.QUASI_CLOSED, 0, 0, 0);
+ ContainerCheckRequest request = new ContainerCheckRequest.Builder()
+ .setPendingOps(Collections.EMPTY_LIST)
+ .setReport(new ReplicationManagerReport())
+ .setContainerInfo(containerInfo)
+ .setContainerReplicas(containerReplicas)
+ .build();
+
+ Assertions.assertFalse(quasiClosedContainerHandler.handle(request));
+ Mockito.verify(replicationManager, times(0))
+ .sendCloseContainerReplicaCommand(any(), any(), anyBoolean());
+ Assertions.assertEquals(1, request.getReport().getStat(
+ ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
+ }
+
+ /**
+ * Only one replica is in QUASI_CLOSED state. This fails the condition of
+ * having greater than 50% of replicas with unique origin nodes in
+ * QUASI_CLOSED state. The handler should return false.
+ */
+ @Test
+ public void testQuasiClosedWithTwoOpenReplicasReturnsFalse() {
+ ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo(
+ ratisReplicationConfig, 1, QUASI_CLOSED);
+ Set<ContainerReplica> containerReplicas = ReplicationTestUtil
+ .createReplicasWithSameOrigin(containerInfo.containerID(),
+ State.OPEN, 0, 0);
+ ContainerReplica quasiClosed = ReplicationTestUtil.createContainerReplica(
+ containerInfo.containerID(), 0,
+ HddsProtos.NodeOperationalState.IN_SERVICE, State.QUASI_CLOSED);
+ containerReplicas.add(quasiClosed);
+ ContainerCheckRequest request = new ContainerCheckRequest.Builder()
+ .setPendingOps(Collections.EMPTY_LIST)
+ .setReport(new ReplicationManagerReport())
+ .setContainerInfo(containerInfo)
+ .setContainerReplicas(containerReplicas)
+ .build();
+
+ Assertions.assertFalse(quasiClosedContainerHandler.handle(request));
+ Mockito.verify(replicationManager, times(0))
+ .sendCloseContainerReplicaCommand(any(), any(), anyBoolean());
+ Assertions.assertEquals(1, request.getReport().getStat(
+ ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK));
+ }
+
+ /**
+ * If it's possible to force close replicas then only replicas with the
+ * highest Sequence ID (also known as BCSID) should be closed.
+ */
+ @Test
+ public void testReplicasWithHighestBCSIDAreClosed() {
+ final ContainerInfo containerInfo =
+ getContainer(HddsProtos.LifeCycleState.QUASI_CLOSED);
+ containerInfo.setUsedBytes(99);
+ final ContainerID id = containerInfo.containerID();
+
+ // create replicas with unique origin DNs
+ DatanodeDetails dnOne = randomDatanodeDetails();
+ DatanodeDetails dnTwo = randomDatanodeDetails();
+ DatanodeDetails dnThree = randomDatanodeDetails();
+
+ // 1001 is the highest sequence id
+ final ContainerReplica replicaOne = getReplicas(
+ id, State.QUASI_CLOSED, 1000L, dnOne.getUuid(), dnOne);
+ final ContainerReplica replicaTwo = getReplicas(
+ id, State.QUASI_CLOSED, 1001L, dnTwo.getUuid(), dnTwo);
+ final ContainerReplica replicaThree = getReplicas(
+ id, State.QUASI_CLOSED, 1001L, dnThree.getUuid(), dnThree);
+ Set<ContainerReplica> containerReplicas = new HashSet<>();
+ containerReplicas.add(replicaOne);
+ containerReplicas.add(replicaTwo);
+ containerReplicas.add(replicaThree);
+
+ ContainerCheckRequest request = new ContainerCheckRequest.Builder()
+ .setPendingOps(Collections.EMPTY_LIST)
+ .setReport(new ReplicationManagerReport())
+ .setContainerInfo(containerInfo)
+ .setContainerReplicas(containerReplicas)
+ .build();
+
+ Assertions.assertTrue(quasiClosedContainerHandler.handle(request));
+ // verify close command was sent for replicas with sequence ID 1001, that
+ // is dnTwo and dnThree
+ Mockito.verify(replicationManager, times(1))
+ .sendCloseContainerReplicaCommand(eq(containerInfo), eq(dnTwo),
+ anyBoolean());
+ Mockito.verify(replicationManager, times(1))
+ .sendCloseContainerReplicaCommand(eq(containerInfo), eq(dnThree),
+ anyBoolean());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org