You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by na...@apache.org on 2020/02/01 18:46:24 UTC
[hadoop-ozone] branch master updated: HDDS-2850. Handle Create
container use case in Recon. (#503)
This is an automated email from the ASF dual-hosted git repository.
nanda pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 7a3dcf9 HDDS-2850. Handle Create container use case in Recon. (#503)
7a3dcf9 is described below
commit 7a3dcf962d87c0929e5eff6fc9317eb28752481e
Author: avijayanhwx <14...@users.noreply.github.com>
AuthorDate: Sat Feb 1 10:46:16 2020 -0800
HDDS-2850. Handle Create container use case in Recon. (#503)
---
.../container/AbstractContainerReportHandler.java | 2 +-
.../hdds/scm/container/ContainerManager.java | 2 +
.../hdds/scm/container/ContainerStateManager.java | 15 ++-
.../IncrementalContainerReportHandler.java | 3 +
.../hdds/scm/container/SCMContainerManager.java | 22 +++-
.../ozone/recon/scm/ReconContainerManager.java | 28 +++++
.../ReconIncrementalContainerReportHandler.java | 70 +++++++-----
.../scm/ReconStorageContainerManagerFacade.java | 3 +-
.../recon/spi/StorageContainerServiceProvider.java | 10 ++
.../impl/StorageContainerServiceProviderImpl.java | 6 +
.../scm/AbstractReconContainerManagerTest.java | 84 ++++++++++++++
.../ozone/recon/scm/TestReconContainerManager.java | 81 ++++++++++++++
...TestReconIncrementalContainerReportHandler.java | 121 +++++++++++++++++++++
13 files changed, 410 insertions(+), 37 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java
index 59be36b..56a51d7 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java
@@ -63,7 +63,7 @@ public class AbstractContainerReportHandler {
*
* @throws IOException In case of any Exception while processing the report
*/
- void processContainerReplica(final DatanodeDetails datanodeDetails,
+ protected void processContainerReplica(final DatanodeDetails datanodeDetails,
final ContainerReplicaProto replicaProto)
throws IOException {
final ContainerID containerId = ContainerID
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
index f9488e2..973026d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
@@ -74,6 +74,8 @@ public interface ContainerManager extends Closeable {
ContainerInfo getContainer(ContainerID containerID)
throws ContainerNotFoundException;
+ boolean exists(ContainerID containerID);
+
/**
* Returns containers under certain conditions.
* Search container IDs from start ID(exclusive),
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
index cefc185..526e4b3 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
@@ -301,15 +301,22 @@ public class ContainerStateManager {
.setReplicationFactor(pipeline.getFactor())
.setReplicationType(pipeline.getType())
.build();
+ addContainerInfo(containerID, containerInfo, pipelineManager, pipeline);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("New container allocated: {}", containerInfo);
+ }
+ return containerInfo;
+ }
+
+ public void addContainerInfo(long containerID,
+ ContainerInfo containerInfo,
+ PipelineManager pipelineManager,
+ Pipeline pipeline) throws IOException {
Preconditions.checkNotNull(containerInfo);
containers.addContainer(containerInfo);
pipelineManager.addContainerToPipeline(pipeline.getId(),
ContainerID.valueof(containerID));
containerStateCount.incrementAndGet(containerInfo.getState());
- if (LOG.isTraceEnabled()) {
- LOG.trace("New container allocated: {}", containerInfo);
- }
- return containerInfo;
}
/**
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/IncrementalContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/IncrementalContainerReportHandler.java
index b581000..c3fd94f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/IncrementalContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/IncrementalContainerReportHandler.java
@@ -90,4 +90,7 @@ public class IncrementalContainerReportHandler extends
}
+ protected NodeManager getNodeManager() {
+ return this.nodeManager;
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
index e08dd8c..9624ce0 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
@@ -194,6 +194,18 @@ public class SCMContainerManager implements ContainerManager {
return containerStateManager.getContainer(containerID);
}
+ @Override
+ public boolean exists(ContainerID containerID) {
+ lock.lock();
+ try {
+ return (containerStateManager.getContainer(containerID) != null);
+ } catch (ContainerNotFoundException e) {
+ return false;
+ } finally {
+ lock.unlock();
+ }
+ }
+
/**
* {@inheritDoc}
*/
@@ -441,7 +453,7 @@ public class SCMContainerManager implements ContainerManager {
* @param containerInfo
* @throws IOException
*/
- private void addContainerToDB(ContainerInfo containerInfo)
+ protected void addContainerToDB(ContainerInfo containerInfo)
throws IOException {
try {
final byte[] containerIDBytes = Longs.toByteArray(
@@ -583,4 +595,12 @@ public class SCMContainerManager implements ContainerManager {
File metaDir = ServerUtils.getScmDbDir(conf);
return new File(metaDir, SCM_CONTAINER_DB);
}
+
+ protected PipelineManager getPipelineManager() {
+ return pipelineManager;
+ }
+
+ public Lock getLock() {
+ return lock;
+ }
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
index af2ae60..a751da8 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
@@ -24,7 +24,10 @@ import java.io.File;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.ozone.recon.ReconUtils;
@@ -55,4 +58,29 @@ public class ReconContainerManager extends SCMContainerManager {
File metaDir = ReconUtils.getReconScmDbDir(conf);
return new File(metaDir, RECON_SCM_CONTAINER_DB);
}
+
+ /**
+ * Adds a new container to Recon's container manager.
+ * @param containerId id
+ * @param containerWithPipeline containerInfo with pipeline info
+ * @throws IOException on Error.
+ */
+ public void addNewContainer(long containerId,
+ ContainerWithPipeline containerWithPipeline)
+ throws IOException {
+ ContainerInfo containerInfo = containerWithPipeline.getContainerInfo();
+ getLock().lock();
+ try {
+ getContainerStateManager().addContainerInfo(containerId, containerInfo,
+ getPipelineManager(), containerWithPipeline.getPipeline());
+ addContainerToDB(containerInfo);
+ } catch (IOException ex) {
+ getPipelineManager().removeContainerFromPipeline(
+ containerInfo.getPipelineID(),
+ new ContainerID(containerInfo.getContainerID()));
+ throw ex;
+ } finally {
+ getLock().unlock();
+ }
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/IncrementalContainerReportHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconIncrementalContainerReportHandler.java
similarity index 55%
copy from hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/IncrementalContainerReportHandler.java
copy to hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconIncrementalContainerReportHandler.java
index b581000..d50156e 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/IncrementalContainerReportHandler.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconIncrementalContainerReportHandler.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,39 +16,41 @@
* limitations under the License.
*/
-package org.apache.hadoop.hdds.scm.container;
+package org.apache.hadoop.ozone.recon.scm;
import java.io.IOException;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos
- .ContainerReplicaProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerManager;
+import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
+import org.apache.hadoop.hdds.scm.container.IncrementalContainerReportHandler;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
- .IncrementalContainerReportFromDatanode;
-import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.IncrementalContainerReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Handles incremental container reports from datanode.
+ * Recon ICR handler.
*/
-public class IncrementalContainerReportHandler extends
- AbstractContainerReportHandler
- implements EventHandler<IncrementalContainerReportFromDatanode> {
+public class ReconIncrementalContainerReportHandler
+ extends IncrementalContainerReportHandler {
private static final Logger LOG = LoggerFactory.getLogger(
- IncrementalContainerReportHandler.class);
+ ReconIncrementalContainerReportHandler.class);
- private final NodeManager nodeManager;
+ private StorageContainerServiceProvider scmClient;
- public IncrementalContainerReportHandler(
- final NodeManager nodeManager,
- final ContainerManager containerManager) {
- super(containerManager, LOG);
- this.nodeManager = nodeManager;
+ public ReconIncrementalContainerReportHandler(NodeManager nodeManager,
+ ContainerManager containerManager,
+ StorageContainerServiceProvider scmClient) {
+ super(nodeManager, containerManager);
+ this.scmClient = scmClient;
}
@Override
@@ -56,9 +58,11 @@ public class IncrementalContainerReportHandler extends
final EventPublisher publisher) {
if (LOG.isDebugEnabled()) {
LOG.debug("Processing incremental container report from data node {}",
- report.getDatanodeDetails().getUuid());
+ report.getDatanodeDetails());
}
+ ReconContainerManager containerManager =
+ (ReconContainerManager) getContainerManager();
boolean success = true;
for (ContainerReplicaProto replicaProto :
report.getReport().getReportList()) {
@@ -66,7 +70,20 @@ public class IncrementalContainerReportHandler extends
final DatanodeDetails dd = report.getDatanodeDetails();
final ContainerID id = ContainerID.valueof(
replicaProto.getContainerID());
- nodeManager.addContainer(dd, id);
+ if (!getContainerManager().exists(id)) {
+ LOG.info("New container {} got from {}.", id,
+ report.getDatanodeDetails());
+ try {
+ ContainerWithPipeline containerWithPipeline =
+ scmClient.getContainerWithPipeline(id.getId());
+ containerManager.addNewContainer(id.getId(), containerWithPipeline);
+ } catch (IOException ioEx) {
+ LOG.error("Exception while getting new container info from SCM",
+ ioEx);
+ return;
+ }
+ }
+ getNodeManager().addContainer(dd, id);
processContainerReplica(dd, replicaProto);
} catch (ContainerNotFoundException e) {
success = false;
@@ -81,13 +98,6 @@ public class IncrementalContainerReportHandler extends
replicaProto.getContainerID());
}
}
-
- if (success) {
- getContainerManager().notifyContainerReportProcessing(false, true);
- } else {
- getContainerManager().notifyContainerReportProcessing(false, false);
- }
-
+ getContainerManager().notifyContainerReportProcessing(false, success);
}
-
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
index db57e99..cf5756f 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
@@ -114,7 +114,8 @@ public class ReconStorageContainerManagerFacade
ContainerReportHandler containerReportHandler =
new ContainerReportHandler(scmNodeManager, containerManager);
IncrementalContainerReportHandler icrHandler =
- new IncrementalContainerReportHandler(scmNodeManager, containerManager);
+ new ReconIncrementalContainerReportHandler(scmNodeManager,
+ containerManager, scmServiceProvider);
CloseContainerEventHandler closeContainerHandler =
new CloseContainerEventHandler(pipelineManager, containerManager);
ContainerActionsHandler actionsHandler = new ContainerActionsHandler();
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/StorageContainerServiceProvider.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/StorageContainerServiceProvider.java
index 23db579..925f46e 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/StorageContainerServiceProvider.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/StorageContainerServiceProvider.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
/**
@@ -43,4 +44,13 @@ public interface StorageContainerServiceProvider {
* @throws IOException in case of exception
*/
Pipeline getPipeline(HddsProtos.PipelineID pipelineID) throws IOException;
+
+ /**
+ * Requests SCM for a container given ID.
+ * @param containerId containerId
+ * @return ContainerInfo + Pipeline info
+ * @throws IOException in case of any exception.
+ */
+ ContainerWithPipeline getContainerWithPipeline(long containerId)
+ throws IOException;
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java
index 77cdb4b..6ee1318 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java
@@ -24,6 +24,7 @@ import java.util.List;
import javax.inject.Inject;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
@@ -54,4 +55,9 @@ public class StorageContainerServiceProviderImpl
return scmClient.getPipeline(pipelineID);
}
+ @Override
+ public ContainerWithPipeline getContainerWithPipeline(long containerId)
+ throws IOException {
+ return scmClient.getContainerWithPipeline(containerId);
+ }
}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java
new file mode 100644
index 0000000..d7745c4
--- /dev/null
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.scm;
+
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.net.NetworkTopology;
+import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.SCMNodeManager;
+import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Abstract class for Recon Container Manager related tests.
+ */
+public class AbstractReconContainerManagerTest {
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ private OzoneConfiguration conf;
+ private SCMStorageConfig scmStorageConfig;
+ private ReconPipelineManager pipelineManager;
+ private ReconContainerManager containerManager;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new OzoneConfiguration();
+ conf.set(OZONE_METADATA_DIRS,
+ temporaryFolder.newFolder().getAbsolutePath());
+ conf.set(OZONE_SCM_NAMES, "localhost");
+ scmStorageConfig = new ReconStorageConfig(conf);
+ NetworkTopology clusterMap = new NetworkTopologyImpl(conf);
+ EventQueue eventQueue = new EventQueue();
+ NodeManager nodeManager =
+ new SCMNodeManager(conf, scmStorageConfig, eventQueue, clusterMap);
+ pipelineManager = new ReconPipelineManager(conf, nodeManager, eventQueue);
+ containerManager = new ReconContainerManager(conf, pipelineManager);
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ containerManager.close();
+ pipelineManager.close();
+ }
+
+ protected OzoneConfiguration getConf() {
+ return conf;
+ }
+
+ protected ReconPipelineManager getPipelineManager() {
+ return pipelineManager;
+ }
+
+ protected ReconContainerManager getContainerManager() {
+ return containerManager;
+ }
+
+}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java
new file mode 100644
index 0000000..742a258
--- /dev/null
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.scm;
+
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.OPEN;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.STAND_ALONE;
+import static org.apache.hadoop.ozone.recon.AbstractOMMetadataManagerTest.getRandomPipeline;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.NavigableSet;
+
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.junit.Test;
+
+/**
+ * Test Recon Container Manager.
+ */
+public class TestReconContainerManager
+ extends AbstractReconContainerManagerTest {
+
+ @Test
+ public void testAddNewContainer() throws IOException {
+ ContainerID containerID = new ContainerID(100L);
+ Pipeline pipeline = getRandomPipeline();
+ ReconPipelineManager pipelineManager = getPipelineManager();
+ pipelineManager.addPipeline(pipeline);
+ ContainerInfo containerInfo =
+ new ContainerInfo.Builder()
+ .setContainerID(containerID.getId())
+ .setNumberOfKeys(10)
+ .setPipelineID(pipeline.getId())
+ .setReplicationFactor(ONE)
+ .setOwner("test")
+ .setState(OPEN)
+ .setReplicationType(STAND_ALONE)
+ .build();
+ ContainerWithPipeline containerWithPipeline =
+ new ContainerWithPipeline(containerInfo, pipeline);
+
+ ReconContainerManager containerManager = getContainerManager();
+ assertFalse(containerManager.exists(containerID));
+
+ containerManager.addNewContainer(
+ containerID.getId(), containerWithPipeline);
+
+ assertTrue(containerManager.exists(containerID));
+
+ List<ContainerInfo> containers = containerManager.getContainers(OPEN);
+ assertEquals(1, containers.size());
+ assertEquals(containerInfo, containers.get(0));
+ NavigableSet<ContainerID> containersInPipeline =
+ pipelineManager.getContainersInPipeline(pipeline.getId());
+ assertEquals(1, containersInPipeline.size());
+ assertEquals(containerID, containersInPipeline.first());
+ }
+
+}
\ No newline at end of file
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconIncrementalContainerReportHandler.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconIncrementalContainerReportHandler.java
new file mode 100644
index 0000000..768de8f
--- /dev/null
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconIncrementalContainerReportHandler.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.scm;
+
+import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.OPEN;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.STAND_ALONE;
+import static org.apache.hadoop.ozone.recon.AbstractOMMetadataManagerTest.getRandomPipeline;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.IncrementalContainerReportFromDatanode;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
+import org.junit.Test;
+
+/**
+ * Test Recon ICR handler.
+ */
+public class TestReconIncrementalContainerReportHandler
+ extends AbstractReconContainerManagerTest {
+
+ @Test
+ public void testProcessICR() throws IOException, NodeNotFoundException {
+
+ Pipeline pipeline = getRandomPipeline();
+ ReconPipelineManager pipelineManager = getPipelineManager();
+ pipelineManager.addPipeline(pipeline);
+
+ ContainerID containerID = new ContainerID(100L);
+ ContainerInfo containerInfo =
+ new ContainerInfo.Builder()
+ .setContainerID(containerID.getId())
+ .setNumberOfKeys(10)
+ .setPipelineID(pipeline.getId())
+ .setReplicationFactor(ONE)
+ .setOwner("test")
+ .setState(OPEN)
+ .setReplicationType(STAND_ALONE)
+ .build();
+ ContainerWithPipeline containerWithPipeline =
+ new ContainerWithPipeline(containerInfo, pipeline);
+
+ StorageContainerServiceProvider scmServiceProviderMock = mock(
+ StorageContainerServiceProvider.class);
+ when(scmServiceProviderMock.getContainerWithPipeline(100L))
+ .thenReturn(containerWithPipeline);
+
+ DatanodeDetails datanodeDetails = randomDatanodeDetails();
+ IncrementalContainerReportFromDatanode reportMock =
+ mock(IncrementalContainerReportFromDatanode.class);
+ when(reportMock.getDatanodeDetails()).thenReturn(datanodeDetails);
+ IncrementalContainerReportProto containerReport =
+ getIncrementalContainerReportProto(containerID,
+ State.OPEN,
+ datanodeDetails.getUuidString());
+ when(reportMock.getReport()).thenReturn(containerReport);
+
+ NodeManager nodeManagerMock = mock(NodeManager.class);
+
+ ReconContainerManager containerManager = getContainerManager();
+ ReconIncrementalContainerReportHandler recconIcr =
+ new ReconIncrementalContainerReportHandler(nodeManagerMock,
+ containerManager, scmServiceProviderMock);
+ EventPublisher eventPublisherMock = mock(EventPublisher.class);
+
+ recconIcr.onMessage(reportMock, eventPublisherMock);
+ verify(nodeManagerMock, times(1))
+ .addContainer(datanodeDetails, containerID);
+ assertTrue(containerManager.exists(containerID));
+ assertEquals(1, containerManager.getContainerReplicas(containerID).size());
+ }
+
+ private static IncrementalContainerReportProto
+ getIncrementalContainerReportProto(final ContainerID containerId,
+ final State state,
+ final String originNodeId) {
+ final IncrementalContainerReportProto.Builder crBuilder =
+ IncrementalContainerReportProto.newBuilder();
+ final ContainerReplicaProto replicaProto =
+ ContainerReplicaProto.newBuilder()
+ .setContainerID(containerId.getId())
+ .setState(state)
+ .setOriginNodeId(originNodeId)
+ .build();
+ return crBuilder.addReport(replicaProto).build();
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org