You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by el...@apache.org on 2020/04/06 12:04:33 UTC
[hadoop-ozone] branch master updated: HDDS-3241. Invalid container
reported to SCM should be deleted
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new d3b81a4 HDDS-3241. Invalid container reported to SCM should be deleted
d3b81a4 is described below
commit d3b81a4108ed13c0fda3fcff3c68a377d1f988f0
Author: Yiqun Lin <yq...@apache.org>
AuthorDate: Mon Apr 6 13:52:41 2020 +0200
HDDS-3241. Invalid container reported to SCM should be deleted
Closes #702
---
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 24 ++++
.../hdds/scm/container/ContainerReportHandler.java | 57 +++++++-
.../hdds/scm/server/StorageContainerManager.java | 2 +-
.../scm/container/TestUnknownContainerReport.java | 145 +++++++++++++++++++++
4 files changed, 220 insertions(+), 8 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java
index c98546b..73701ea 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java
@@ -45,6 +45,19 @@ public class ScmConfig {
)
private String keytab;
+ @Config(key = "unknown-container.action",
+ type = ConfigType.STRING,
+ defaultValue = "WARN",
+ tags = { ConfigTag.SCM, ConfigTag.MANAGEMENT },
+ description =
+ "The action taken by SCM to process unknown "
+ + "containers that reported by Datanodes. The default "
+ + "action is just logging container not found warning, "
+ + "another available action is DELETE action. "
+ + "These unknown containers will be deleted under this "
+ + "action way."
+ )
+ private String action;
public void setKerberosPrincipal(String kerberosPrincipal) {
this.principal = kerberosPrincipal;
@@ -55,6 +68,10 @@ public class ScmConfig {
this.keytab = kerberosKeytab;
}
+ public void setUnknownContainerAction(String unknownContainerAction) {
+ this.action = unknownContainerAction;
+ }
+
public String getKerberosPrincipal() {
return this.principal;
}
@@ -63,6 +80,10 @@ public class ScmConfig {
return this.keytab;
}
+ public String getUnknownContainerAction() {
+ return this.action;
+ }
+
/**
* Configuration strings class.
* required for SCMSecurityProtocol where the KerberosInfo references
@@ -77,4 +98,7 @@ public class ScmConfig {
public static final String HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY =
"hdds.scm.kerberos.keytab.file";
}
+
+ public static final String HDDS_SCM_UNKNOWN_CONTAINER_ACTION =
+ "hdds.scm.unknown-container.action";
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
index 2227df6..1b0b81f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
@@ -18,11 +18,13 @@
package org.apache.hadoop.hdds.scm.container;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.scm.ScmConfig;
import org.apache.hadoop.hdds.scm.block.PendingDeleteStatusList;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.node.NodeManager;
@@ -31,6 +33,8 @@ import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
.ContainerReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
+import org.apache.hadoop.ozone.protocol.commands.DeleteContainerCommand;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,6 +55,14 @@ public class ContainerReportHandler extends AbstractContainerReportHandler
private final NodeManager nodeManager;
private final ContainerManager containerManager;
+ private final String unknownContainerHandleAction;
+
+ /**
+ * The action taken by ContainerReportHandler to handle
+ * unknown containers.
+ */
+ static final String UNKNOWN_CONTAINER_ACTION_WARN = "WARN";
+ static final String UNKNOWN_CONTAINER_ACTION_DELETE = "DELETE";
/**
* Constructs ContainerReportHandler instance with the
@@ -58,12 +70,26 @@ public class ContainerReportHandler extends AbstractContainerReportHandler
*
* @param nodeManager NodeManager instance
* @param containerManager ContainerManager instance
+ * @param conf OzoneConfiguration instance
*/
public ContainerReportHandler(final NodeManager nodeManager,
- final ContainerManager containerManager) {
+ final ContainerManager containerManager,
+ OzoneConfiguration conf) {
super(containerManager, LOG);
this.nodeManager = nodeManager;
this.containerManager = containerManager;
+
+ if (conf != null) {
+ ScmConfig scmConfig = conf.getObject(ScmConfig.class);
+ unknownContainerHandleAction = scmConfig.getUnknownContainerAction();
+ } else {
+ unknownContainerHandleAction = UNKNOWN_CONTAINER_ACTION_WARN;
+ }
+ }
+
+ public ContainerReportHandler(final NodeManager nodeManager,
+ final ContainerManager containerManager) {
+ this(nodeManager, containerManager, null);
}
/**
@@ -94,7 +120,7 @@ public class ContainerReportHandler extends AbstractContainerReportHandler
final Set<ContainerID> missingReplicas = new HashSet<>(containersInSCM);
missingReplicas.removeAll(containersInDn);
- processContainerReplicas(datanodeDetails, replicas);
+ processContainerReplicas(datanodeDetails, replicas, publisher);
processMissingReplicas(datanodeDetails, missingReplicas);
updateDeleteTransaction(datanodeDetails, replicas, publisher);
@@ -114,20 +140,37 @@ public class ContainerReportHandler extends AbstractContainerReportHandler
}
/**
- * Processes the ContainerReport.
+ * Processes the ContainerReport, unknown container reported
+ * that will be deleted by SCM.
*
* @param datanodeDetails Datanode from which this report was received
* @param replicas list of ContainerReplicaProto
+ * @param publisher EventPublisher reference
*/
private void processContainerReplicas(final DatanodeDetails datanodeDetails,
- final List<ContainerReplicaProto> replicas) {
+ final List<ContainerReplicaProto> replicas,
+ final EventPublisher publisher) {
for (ContainerReplicaProto replicaProto : replicas) {
try {
processContainerReplica(datanodeDetails, replicaProto);
} catch (ContainerNotFoundException e) {
- LOG.error("Received container report for an unknown container" +
- " {} from datanode {}.", replicaProto.getContainerID(),
- datanodeDetails, e);
+ if(unknownContainerHandleAction.equals(
+ UNKNOWN_CONTAINER_ACTION_WARN)) {
+ LOG.error("Received container report for an unknown container" +
+ " {} from datanode {}.", replicaProto.getContainerID(),
+ datanodeDetails, e);
+ } else if (unknownContainerHandleAction.equals(
+ UNKNOWN_CONTAINER_ACTION_DELETE)) {
+ final ContainerID containerId = ContainerID
+ .valueof(replicaProto.getContainerID());
+ final DeleteContainerCommand deleteCommand =
+ new DeleteContainerCommand(containerId.getId(), true);
+ final CommandForDatanode datanodeCommand = new CommandForDatanode<>(
+ datanodeDetails.getUuid(), deleteCommand);
+ publisher.fireEvent(SCMEvents.DATANODE_COMMAND, datanodeCommand);
+ LOG.info("Sending delete container command for unknown container {}"
+ + " to datanode {}", containerId.getId(), datanodeDetails);
+ }
} catch (IOException e) {
LOG.error("Exception while processing container report for container" +
" {} from datanode {}.", replicaProto.getContainerID(),
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 9dcb8f2..383e7ce 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -303,7 +303,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
new PendingDeleteHandler(scmBlockManager.getSCMBlockDeletingService());
ContainerReportHandler containerReportHandler =
- new ContainerReportHandler(scmNodeManager, containerManager);
+ new ContainerReportHandler(scmNodeManager, containerManager, conf);
IncrementalContainerReportHandler incrementalContainerReportHandler =
new IncrementalContainerReportHandler(
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestUnknownContainerReport.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestUnknownContainerReport.java
new file mode 100644
index 0000000..9c11caa
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestUnknownContainerReport.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.container;
+
+import static org.apache.hadoop.hdds.scm.TestUtils.getContainer;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.scm.ScmConfig;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.server
+ .SCMDatanodeHeartbeatDispatcher.ContainerReportFromDatanode;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+/**
+ * Test container deletion behaviour of unknown containers
+ * that reported by Datanodes.
+ */
+public class TestUnknownContainerReport {
+
+ private NodeManager nodeManager;
+ private ContainerManager containerManager;
+ private ContainerStateManager containerStateManager;
+ private EventPublisher publisher;
+
+ @Before
+ public void setup() throws IOException {
+ final Configuration conf = new OzoneConfiguration();
+ this.nodeManager = new MockNodeManager(true, 10);
+ this.containerManager = Mockito.mock(ContainerManager.class);
+ this.containerStateManager = new ContainerStateManager(conf);
+ this.publisher = Mockito.mock(EventPublisher.class);
+
+ Mockito.when(containerManager.getContainer(Mockito.any(ContainerID.class)))
+ .thenThrow(new ContainerNotFoundException());
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ containerStateManager.close();
+ }
+
+ @Test
+ public void testUnknownContainerNotDeleted() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ sendContainerReport(conf);
+
+ // By default, unknown containers won't be taken delete action by SCM
+ verify(publisher, times(0)).fireEvent(
+ Mockito.eq(SCMEvents.DATANODE_COMMAND),
+ Mockito.any(CommandForDatanode.class));
+ }
+
+ @Test
+ public void testUnknownContainerDeleted() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(
+ ScmConfig.HDDS_SCM_UNKNOWN_CONTAINER_ACTION,
+ ContainerReportHandler.UNKNOWN_CONTAINER_ACTION_DELETE);
+
+ sendContainerReport(conf);
+ verify(publisher, times(1)).fireEvent(
+ Mockito.eq(SCMEvents.DATANODE_COMMAND),
+ Mockito.any(CommandForDatanode.class));
+ }
+
+ /**
+ * Trigger datanode to send unknown container report to SCM.
+ * @param conf OzoneConfiguration instance to initialize
+ * ContainerReportHandler
+ */
+ private void sendContainerReport(OzoneConfiguration conf) {
+ ContainerReportHandler reportHandler = new ContainerReportHandler(
+ nodeManager, containerManager, conf);
+
+ ContainerInfo container = getContainer(LifeCycleState.CLOSED);
+ Iterator<DatanodeDetails> nodeIterator = nodeManager
+ .getNodes(NodeState.HEALTHY).iterator();
+ DatanodeDetails datanode = nodeIterator.next();
+
+ ContainerReportsProto containerReport = getContainerReportsProto(
+ container.containerID(), ContainerReplicaProto.State.CLOSED,
+ datanode.getUuidString());
+ ContainerReportFromDatanode containerReportFromDatanode =
+ new ContainerReportFromDatanode(datanode, containerReport);
+ reportHandler.onMessage(containerReportFromDatanode, publisher);
+ }
+
+ private static ContainerReportsProto getContainerReportsProto(
+ final ContainerID containerId, final ContainerReplicaProto.State state,
+ final String originNodeId) {
+ final ContainerReportsProto.Builder crBuilder =
+ ContainerReportsProto.newBuilder();
+ final ContainerReplicaProto replicaProto =
+ ContainerReplicaProto.newBuilder()
+ .setContainerID(containerId.getId())
+ .setState(state)
+ .setOriginNodeId(originNodeId)
+ .setFinalhash("e16cc9d6024365750ed8dbd194ea46d2")
+ .setSize(5368709120L)
+ .setUsed(2000000000L)
+ .setKeyCount(100000000L)
+ .setReadCount(100000000L)
+ .setWriteCount(100000000L)
+ .setReadBytes(2000000000L)
+ .setWriteBytes(2000000000L)
+ .setBlockCommitSequenceId(10000L)
+ .setDeleteTransactionId(0)
+ .build();
+ return crBuilder.addReports(replicaProto).build();
+ }
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org