You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2018/10/05 12:56:06 UTC
[11/51] [partial] hadoop git commit: HADOOP-15791. Remove Ozone
related sources from the 3.2 branch. Contributed by Elek, Marton.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
deleted file mode 100644
index f9a881e..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
+++ /dev/null
@@ -1,380 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.XceiverClientManager;
-import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.container.common.SCMTestUtils;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.test.LambdaTestUtils;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.NavigableSet;
-import java.util.Random;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Tests for Container Mapping.
- */
-public class TestContainerMapping {
- private static ContainerMapping mapping;
- private static MockNodeManager nodeManager;
- private static File testDir;
- private static XceiverClientManager xceiverClientManager;
- private static String containerOwner = "OZONE";
- private static Random random;
-
- private static final long TIMEOUT = 10000;
-
- @Rule
- public ExpectedException thrown = ExpectedException.none();
- @BeforeClass
- public static void setUp() throws Exception {
- Configuration conf = SCMTestUtils.getConf();
-
- testDir = GenericTestUtils
- .getTestDir(TestContainerMapping.class.getSimpleName());
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS,
- testDir.getAbsolutePath());
- conf.setTimeDuration(
- ScmConfigKeys.OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT,
- TIMEOUT,
- TimeUnit.MILLISECONDS);
- boolean folderExisted = testDir.exists() || testDir.mkdirs();
- if (!folderExisted) {
- throw new IOException("Unable to create test directory path");
- }
- nodeManager = new MockNodeManager(true, 10);
- mapping = new ContainerMapping(conf, nodeManager, 128,
- new EventQueue());
- xceiverClientManager = new XceiverClientManager(conf);
- random = new Random();
- }
-
- @AfterClass
- public static void cleanup() throws IOException {
- if(mapping != null) {
- mapping.close();
- }
- FileUtil.fullyDelete(testDir);
- }
-
- @Before
- public void clearChillMode() {
- nodeManager.setChillmode(false);
- }
-
- @Test
- public void testallocateContainer() throws Exception {
- ContainerWithPipeline containerInfo = mapping.allocateContainer(
- xceiverClientManager.getType(),
- xceiverClientManager.getFactor(),
- containerOwner);
- Assert.assertNotNull(containerInfo);
- }
-
- @Test
- public void testallocateContainerDistributesAllocation() throws Exception {
- /* This is a lame test, we should really be testing something like
- z-score or make sure that we don't have 3sigma kind of events. Too lazy
- to write all that code. This test very lamely tests if we have more than
- 5 separate nodes from the list of 10 datanodes that got allocated a
- container.
- */
- Set<UUID> pipelineList = new TreeSet<>();
- for (int x = 0; x < 30; x++) {
- ContainerWithPipeline containerInfo = mapping.allocateContainer(
- xceiverClientManager.getType(),
- xceiverClientManager.getFactor(),
- containerOwner);
-
- Assert.assertNotNull(containerInfo);
- Assert.assertNotNull(containerInfo.getPipeline());
- pipelineList.add(containerInfo.getPipeline().getLeader()
- .getUuid());
- }
- Assert.assertTrue(pipelineList.size() > 5);
- }
-
- @Test
- public void testGetContainer() throws IOException {
- ContainerWithPipeline containerInfo = mapping.allocateContainer(
- xceiverClientManager.getType(),
- xceiverClientManager.getFactor(),
- containerOwner);
- Pipeline pipeline = containerInfo.getPipeline();
- Assert.assertNotNull(pipeline);
- Pipeline newPipeline = containerInfo.getPipeline();
- Assert.assertEquals(pipeline.getLeader().getUuid(),
- newPipeline.getLeader().getUuid());
- }
-
- @Test
- public void testGetContainerWithPipeline() throws Exception {
- ContainerWithPipeline containerWithPipeline = mapping.allocateContainer(
- xceiverClientManager.getType(),
- xceiverClientManager.getFactor(),
- containerOwner);
- ContainerInfo contInfo = containerWithPipeline.getContainerInfo();
- // Add dummy replicas for container.
- DatanodeDetails dn1 = DatanodeDetails.newBuilder()
- .setHostName("host1")
- .setIpAddress("1.1.1.1")
- .setUuid(UUID.randomUUID().toString()).build();
- DatanodeDetails dn2 = DatanodeDetails.newBuilder()
- .setHostName("host2")
- .setIpAddress("2.2.2.2")
- .setUuid(UUID.randomUUID().toString()).build();
- mapping
- .updateContainerState(contInfo.getContainerID(), LifeCycleEvent.CREATE);
- mapping.updateContainerState(contInfo.getContainerID(),
- LifeCycleEvent.CREATED);
- mapping.updateContainerState(contInfo.getContainerID(),
- LifeCycleEvent.FINALIZE);
- mapping
- .updateContainerState(contInfo.getContainerID(), LifeCycleEvent.CLOSE);
- ContainerInfo finalContInfo = contInfo;
- LambdaTestUtils.intercept(SCMException.class, "No entry exist for "
- + "containerId:", () -> mapping.getContainerWithPipeline(
- finalContInfo.getContainerID()));
-
- mapping.getStateManager().getContainerStateMap()
- .addContainerReplica(contInfo.containerID(), dn1, dn2);
-
- contInfo = mapping.getContainer(contInfo.getContainerID());
- Assert.assertEquals(contInfo.getState(), LifeCycleState.CLOSED);
- Pipeline pipeline = containerWithPipeline.getPipeline();
- mapping.getPipelineSelector().finalizePipeline(pipeline);
-
- ContainerWithPipeline containerWithPipeline2 = mapping
- .getContainerWithPipeline(contInfo.getContainerID());
- pipeline = containerWithPipeline2.getPipeline();
- Assert.assertNotEquals(containerWithPipeline, containerWithPipeline2);
- Assert.assertNotNull("Pipeline should not be null", pipeline);
- Assert.assertTrue(pipeline.getDatanodeHosts().contains(dn1.getHostName()));
- Assert.assertTrue(pipeline.getDatanodeHosts().contains(dn2.getHostName()));
- }
-
- @Test
- public void testgetNoneExistentContainer() throws IOException {
- thrown.expectMessage("Specified key does not exist.");
- mapping.getContainer(random.nextLong());
- }
-
- @Test
- public void testContainerCreationLeaseTimeout() throws IOException,
- InterruptedException {
- nodeManager.setChillmode(false);
- ContainerWithPipeline containerInfo = mapping.allocateContainer(
- xceiverClientManager.getType(),
- xceiverClientManager.getFactor(),
- containerOwner);
- mapping.updateContainerState(containerInfo.getContainerInfo()
- .getContainerID(), HddsProtos.LifeCycleEvent.CREATE);
- Thread.sleep(TIMEOUT + 1000);
-
- NavigableSet<ContainerID> deleteContainers = mapping.getStateManager()
- .getMatchingContainerIDs(
- "OZONE",
- xceiverClientManager.getType(),
- xceiverClientManager.getFactor(),
- HddsProtos.LifeCycleState.DELETING);
- Assert.assertTrue(deleteContainers
- .contains(containerInfo.getContainerInfo().containerID()));
-
- thrown.expect(IOException.class);
- thrown.expectMessage("Lease Exception");
- mapping
- .updateContainerState(containerInfo.getContainerInfo().getContainerID(),
- HddsProtos.LifeCycleEvent.CREATED);
- }
-
- @Test
- public void testFullContainerReport() throws Exception {
- ContainerInfo info = createContainer();
- DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
- List<StorageContainerDatanodeProtocolProtos.ContainerInfo> reports =
- new ArrayList<>();
- StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder =
- StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder();
- ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea46d2")
- .setSize(5368709120L)
- .setUsed(2000000000L)
- .setKeyCount(100000000L)
- .setReadCount(100000000L)
- .setWriteCount(100000000L)
- .setReadBytes(2000000000L)
- .setWriteBytes(2000000000L)
- .setContainerID(info.getContainerID())
- .setDeleteTransactionId(0);
-
- reports.add(ciBuilder.build());
-
- ContainerReportsProto.Builder crBuilder = ContainerReportsProto
- .newBuilder();
- crBuilder.addAllReports(reports);
-
- mapping.processContainerReports(datanodeDetails, crBuilder.build(), false);
-
- ContainerInfo updatedContainer =
- mapping.getContainer(info.getContainerID());
- Assert.assertEquals(100000000L,
- updatedContainer.getNumberOfKeys());
- Assert.assertEquals(2000000000L, updatedContainer.getUsedBytes());
-
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo c : reports) {
- LambdaTestUtils.intercept(SCMException.class, "No entry "
- + "exist for containerId:", () -> mapping.getStateManager()
- .getContainerReplicas(ContainerID.valueof(c.getContainerID())));
- }
-
- mapping.processContainerReports(TestUtils.randomDatanodeDetails(),
- crBuilder.build(), true);
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo c : reports) {
- Assert.assertTrue(mapping.getStateManager().getContainerReplicas(
- ContainerID.valueof(c.getContainerID())).size() > 0);
- }
- }
-
- @Test
- public void testListContainerAfterReport() throws Exception {
- ContainerInfo info1 = createContainer();
- ContainerInfo info2 = createContainer();
- DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
- List<StorageContainerDatanodeProtocolProtos.ContainerInfo> reports =
- new ArrayList<>();
- StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder =
- StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder();
- long cID1 = info1.getContainerID();
- long cID2 = info2.getContainerID();
- ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea46d2")
- .setSize(1000000000L)
- .setUsed(987654321L)
- .setKeyCount(100000000L)
- .setReadBytes(1000000000L)
- .setWriteBytes(1000000000L)
- .setContainerID(cID1);
- reports.add(ciBuilder.build());
-
- ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea54a9")
- .setSize(1000000000L)
- .setUsed(123456789L)
- .setKeyCount(200000000L)
- .setReadBytes(3000000000L)
- .setWriteBytes(4000000000L)
- .setContainerID(cID2);
- reports.add(ciBuilder.build());
-
- ContainerReportsProto.Builder crBuilder = ContainerReportsProto
- .newBuilder();
- crBuilder.addAllReports(reports);
-
- mapping.processContainerReports(datanodeDetails, crBuilder.build(), false);
-
- List<ContainerInfo> list = mapping.listContainer(0, 50);
- Assert.assertEquals(2, list.stream().filter(
- x -> x.getContainerID() == cID1 || x.getContainerID() == cID2).count());
- Assert.assertEquals(300000000L, list.stream().filter(
- x -> x.getContainerID() == cID1 || x.getContainerID() == cID2)
- .mapToLong(x -> x.getNumberOfKeys()).sum());
- Assert.assertEquals(1111111110L, list.stream().filter(
- x -> x.getContainerID() == cID1 || x.getContainerID() == cID2)
- .mapToLong(x -> x.getUsedBytes()).sum());
- }
-
- @Test
- public void testCloseContainer() throws IOException {
- ContainerInfo info = createContainer();
- mapping.updateContainerState(info.getContainerID(),
- HddsProtos.LifeCycleEvent.FINALIZE);
- NavigableSet<ContainerID> pendingCloseContainers = mapping.getStateManager()
- .getMatchingContainerIDs(
- containerOwner,
- xceiverClientManager.getType(),
- xceiverClientManager.getFactor(),
- HddsProtos.LifeCycleState.CLOSING);
- Assert.assertTrue(pendingCloseContainers.contains(info.containerID()));
- mapping.updateContainerState(info.getContainerID(),
- HddsProtos.LifeCycleEvent.CLOSE);
- NavigableSet<ContainerID> closeContainers = mapping.getStateManager()
- .getMatchingContainerIDs(
- containerOwner,
- xceiverClientManager.getType(),
- xceiverClientManager.getFactor(),
- HddsProtos.LifeCycleState.CLOSED);
- Assert.assertTrue(closeContainers.contains(info.containerID()));
- }
-
- /**
- * Creates a container with the given name in ContainerMapping.
- * @throws IOException
- */
- private ContainerInfo createContainer()
- throws IOException {
- nodeManager.setChillmode(false);
- ContainerWithPipeline containerWithPipeline = mapping.allocateContainer(
- xceiverClientManager.getType(),
- xceiverClientManager.getFactor(),
- containerOwner);
- ContainerInfo containerInfo = containerWithPipeline.getContainerInfo();
- mapping.updateContainerState(containerInfo.getContainerID(),
- HddsProtos.LifeCycleEvent.CREATE);
- mapping.updateContainerState(containerInfo.getContainerID(),
- HddsProtos.LifeCycleEvent.CREATED);
- return containerInfo;
- }
-
- @Test
- public void testFlushAllContainers() throws IOException {
- ContainerInfo info = createContainer();
- List<ContainerInfo> containers = mapping.getStateManager()
- .getAllContainers();
- Assert.assertTrue(containers.size() > 0);
- mapping.flushContainerInfo();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
deleted file mode 100644
index f79ae1e..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo
- .Builder;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
-import org.apache.hadoop.hdds.scm.container.replication
- .ReplicationActivityStatus;
-import org.apache.hadoop.hdds.scm.container.replication.ReplicationRequest;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
- .ContainerReportFromDatanode;
-import org.apache.hadoop.hdds.server.events.Event;
-import org.apache.hadoop.hdds.server.events.EventPublisher;
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import static org.mockito.Matchers.anyLong;
-import org.mockito.Mockito;
-import static org.mockito.Mockito.when;
-import org.mockito.stubbing.Answer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Test the behaviour of the ContainerReportHandler.
- */
-public class TestContainerReportHandler implements EventPublisher {
-
- private List<Object> publishedEvents = new ArrayList<>();
- private final NodeManager nodeManager = new MockNodeManager(true, 1);
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestContainerReportHandler.class);
-
- @Before
- public void resetEventCollector() {
- publishedEvents.clear();
- }
-
- @Test
- public void test() throws IOException {
- //GIVEN
- OzoneConfiguration conf = new OzoneConfiguration();
- Mapping mapping = Mockito.mock(Mapping.class);
- PipelineSelector selector = Mockito.mock(PipelineSelector.class);
-
- when(mapping.getContainer(anyLong()))
- .thenAnswer(
- (Answer<ContainerInfo>) invocation ->
- new Builder()
- .setReplicationFactor(ReplicationFactor.THREE)
- .setContainerID((Long) invocation.getArguments()[0])
- .setState(LifeCycleState.CLOSED)
- .build()
- );
-
- ContainerStateManager containerStateManager =
- new ContainerStateManager(conf, mapping, selector);
-
- when(mapping.getStateManager()).thenReturn(containerStateManager);
-
- ReplicationActivityStatus replicationActivityStatus =
- new ReplicationActivityStatus();
-
- ContainerReportHandler reportHandler =
- new ContainerReportHandler(mapping, nodeManager,
- replicationActivityStatus);
-
- DatanodeDetails dn1 = TestUtils.randomDatanodeDetails();
- DatanodeDetails dn2 = TestUtils.randomDatanodeDetails();
- DatanodeDetails dn3 = TestUtils.randomDatanodeDetails();
- DatanodeDetails dn4 = TestUtils.randomDatanodeDetails();
- nodeManager.addDatanodeInContainerMap(dn1.getUuid(), new HashSet<>());
- nodeManager.addDatanodeInContainerMap(dn2.getUuid(), new HashSet<>());
- nodeManager.addDatanodeInContainerMap(dn3.getUuid(), new HashSet<>());
- nodeManager.addDatanodeInContainerMap(dn4.getUuid(), new HashSet<>());
- PipelineSelector pipelineSelector = Mockito.mock(PipelineSelector.class);
-
- Pipeline pipeline = new Pipeline("leader", LifeCycleState.CLOSED,
- ReplicationType.STAND_ALONE, ReplicationFactor.THREE,
- PipelineID.randomId());
-
- when(pipelineSelector.getReplicationPipeline(ReplicationType.STAND_ALONE,
- ReplicationFactor.THREE)).thenReturn(pipeline);
-
- ContainerInfo cont1 = containerStateManager
- .allocateContainer(pipelineSelector, ReplicationType.STAND_ALONE,
- ReplicationFactor.THREE, "root").getContainerInfo();
- ContainerInfo cont2 = containerStateManager
- .allocateContainer(pipelineSelector, ReplicationType.STAND_ALONE,
- ReplicationFactor.THREE, "root").getContainerInfo();
- // Open Container
- ContainerInfo cont3 = containerStateManager
- .allocateContainer(pipelineSelector, ReplicationType.STAND_ALONE,
- ReplicationFactor.THREE, "root").getContainerInfo();
-
- long c1 = cont1.getContainerID();
- long c2 = cont2.getContainerID();
- long c3 = cont3.getContainerID();
-
- // Close remaining containers
- TestUtils.closeContainer(containerStateManager, cont1);
- TestUtils.closeContainer(containerStateManager, cont2);
-
- //when
-
- //initial reports before replication is enabled. 2 containers w 3 replicas.
- reportHandler.onMessage(
- new ContainerReportFromDatanode(dn1,
- createContainerReport(new long[] {c1, c2, c3})), this);
-
- reportHandler.onMessage(
- new ContainerReportFromDatanode(dn2,
- createContainerReport(new long[] {c1, c2, c3})), this);
-
- reportHandler.onMessage(
- new ContainerReportFromDatanode(dn3,
- createContainerReport(new long[] {c1, c2})), this);
-
- reportHandler.onMessage(
- new ContainerReportFromDatanode(dn4,
- createContainerReport(new long[] {})), this);
-
- Assert.assertEquals(0, publishedEvents.size());
-
- replicationActivityStatus.enableReplication();
-
- //no problem here
- reportHandler.onMessage(
- new ContainerReportFromDatanode(dn1,
- createContainerReport(new long[] {c1, c2})), this);
-
- Assert.assertEquals(0, publishedEvents.size());
-
- //container is missing from d2
- reportHandler.onMessage(
- new ContainerReportFromDatanode(dn2,
- createContainerReport(new long[] {c1})), this);
-
- Assert.assertEquals(1, publishedEvents.size());
- ReplicationRequest replicationRequest =
- (ReplicationRequest) publishedEvents.get(0);
-
- Assert.assertEquals(c2, replicationRequest.getContainerId());
- Assert.assertEquals(3, replicationRequest.getExpecReplicationCount());
- Assert.assertEquals(2, replicationRequest.getReplicationCount());
-
- //container was replicated to dn4
- reportHandler.onMessage(
- new ContainerReportFromDatanode(dn4,
- createContainerReport(new long[] {c2})), this);
-
- //no more event, everything is perfect
- Assert.assertEquals(1, publishedEvents.size());
-
- //c2 was found at dn2 (it was missing before, magic)
- reportHandler.onMessage(
- new ContainerReportFromDatanode(dn2,
- createContainerReport(new long[] {c1, c2})), this);
-
- //c2 is over replicated (dn1,dn2,dn3,dn4)
- Assert.assertEquals(2, publishedEvents.size());
-
- replicationRequest =
- (ReplicationRequest) publishedEvents.get(1);
-
- Assert.assertEquals(c2, replicationRequest.getContainerId());
- Assert.assertEquals(3, replicationRequest.getExpecReplicationCount());
- Assert.assertEquals(4, replicationRequest.getReplicationCount());
-
- }
-
- private ContainerReportsProto createContainerReport(long[] containerIds) {
-
- ContainerReportsProto.Builder crBuilder =
- ContainerReportsProto.newBuilder();
-
- for (long containerId : containerIds) {
- org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder
- ciBuilder = org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder();
- ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea46d2")
- .setSize(5368709120L)
- .setUsed(2000000000L)
- .setKeyCount(100000000L)
- .setReadCount(100000000L)
- .setWriteCount(100000000L)
- .setReadBytes(2000000000L)
- .setWriteBytes(2000000000L)
- .setContainerID(containerId)
- .setDeleteTransactionId(0);
-
- crBuilder.addReports(ciBuilder.build());
- }
-
- return crBuilder.build();
- }
-
- @Override
- public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void fireEvent(
- EVENT_TYPE event, PAYLOAD payload) {
- LOG.info("Event is published: {}", payload);
- publishedEvents.add(payload);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java
deleted file mode 100644
index b857740..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.container;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.hdds.scm.container.replication.ReplicationRequest;
-
-import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-/**
- * Testing ContainerStatemanager.
- */
-public class TestContainerStateManager {
-
- private ContainerStateManager containerStateManager;
-
- @Before
- public void init() throws IOException {
- OzoneConfiguration conf = new OzoneConfiguration();
- Mapping mapping = Mockito.mock(Mapping.class);
- PipelineSelector selector = Mockito.mock(PipelineSelector.class);
- containerStateManager = new ContainerStateManager(conf, mapping, selector);
-
- }
-
- @Test
- public void checkReplicationStateOK() throws IOException {
- //GIVEN
- ContainerInfo c1 = TestUtils.allocateContainer(containerStateManager);
-
- DatanodeDetails d1 = TestUtils.randomDatanodeDetails();
- DatanodeDetails d2 = TestUtils.randomDatanodeDetails();
- DatanodeDetails d3 = TestUtils.randomDatanodeDetails();
-
- addReplica(c1, d1);
- addReplica(c1, d2);
- addReplica(c1, d3);
-
- //WHEN
- ReplicationRequest replicationRequest = containerStateManager
- .checkReplicationState(new ContainerID(c1.getContainerID()));
-
- //THEN
- Assert.assertNull(replicationRequest);
- }
-
- @Test
- public void checkReplicationStateMissingReplica() throws IOException {
- //GIVEN
-
- ContainerInfo c1 = TestUtils.allocateContainer(containerStateManager);
-
- DatanodeDetails d1 = TestUtils.randomDatanodeDetails();
- DatanodeDetails d2 = TestUtils.randomDatanodeDetails();
-
- addReplica(c1, d1);
- addReplica(c1, d2);
-
- //WHEN
- ReplicationRequest replicationRequest = containerStateManager
- .checkReplicationState(new ContainerID(c1.getContainerID()));
-
- Assert
- .assertEquals(c1.getContainerID(), replicationRequest.getContainerId());
- Assert.assertEquals(2, replicationRequest.getReplicationCount());
- Assert.assertEquals(3, replicationRequest.getExpecReplicationCount());
- }
-
- private void addReplica(ContainerInfo c1, DatanodeDetails d1) {
- containerStateManager
- .addContainerReplica(new ContainerID(c1.getContainerID()), d1);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/package-info.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/package-info.java
deleted file mode 100644
index 2f35719..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-/**
- * Make CheckStyle happy.
- */
-package org.apache.hadoop.hdds.scm.container.closer;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java
deleted file mode 100644
index f93aea6..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-/**
- * Make CheckStyle Happy.
- */
-package org.apache.hadoop.hdds.scm.container;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java
deleted file mode 100644
index 764daff..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.placement.algorithms;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-
-import org.junit.Assert;
-import org.junit.Test;
-import static org.mockito.Matchers.anyObject;
-import org.mockito.Mockito;
-import static org.mockito.Mockito.when;
-
-public class TestSCMContainerPlacementCapacity {
- @Test
- public void chooseDatanodes() throws SCMException {
- //given
- Configuration conf = new OzoneConfiguration();
-
- List<DatanodeDetails> datanodes = new ArrayList<>();
- for (int i = 0; i < 7; i++) {
- datanodes.add(TestUtils.randomDatanodeDetails());
- }
-
- NodeManager mockNodeManager = Mockito.mock(NodeManager.class);
- when(mockNodeManager.getNodes(NodeState.HEALTHY))
- .thenReturn(new ArrayList<>(datanodes));
-
- when(mockNodeManager.getNodeStat(anyObject()))
- .thenReturn(new SCMNodeMetric(100L, 0L, 100L));
- when(mockNodeManager.getNodeStat(datanodes.get(2)))
- .thenReturn(new SCMNodeMetric(100L, 90L, 10L));
- when(mockNodeManager.getNodeStat(datanodes.get(3)))
- .thenReturn(new SCMNodeMetric(100L, 80L, 20L));
- when(mockNodeManager.getNodeStat(datanodes.get(4)))
- .thenReturn(new SCMNodeMetric(100L, 70L, 30L));
-
- SCMContainerPlacementCapacity scmContainerPlacementRandom =
- new SCMContainerPlacementCapacity(mockNodeManager, conf);
-
- List<DatanodeDetails> existingNodes = new ArrayList<>();
- existingNodes.add(datanodes.get(0));
- existingNodes.add(datanodes.get(1));
-
- Map<DatanodeDetails, Integer> selectedCount = new HashMap<>();
- for (DatanodeDetails datanode : datanodes) {
- selectedCount.put(datanode, 0);
- }
-
- for (int i = 0; i < 1000; i++) {
-
- //when
- List<DatanodeDetails> datanodeDetails =
- scmContainerPlacementRandom.chooseDatanodes(existingNodes, 1, 15);
-
- //then
- Assert.assertEquals(1, datanodeDetails.size());
- DatanodeDetails datanode0Details = datanodeDetails.get(0);
-
- Assert.assertNotEquals(
- "Datanode 0 should not been selected: excluded by parameter",
- datanodes.get(0), datanode0Details);
- Assert.assertNotEquals(
- "Datanode 1 should not been selected: excluded by parameter",
- datanodes.get(1), datanode0Details);
- Assert.assertNotEquals(
- "Datanode 2 should not been selected: not enough space there",
- datanodes.get(2), datanode0Details);
-
- selectedCount
- .put(datanode0Details, selectedCount.get(datanode0Details) + 1);
-
- }
-
- //datanode 4 has less space. Should be selected less times.
- Assert.assertTrue(selectedCount.get(datanodes.get(3)) > selectedCount
- .get(datanodes.get(6)));
- Assert.assertTrue(selectedCount.get(datanodes.get(4)) > selectedCount
- .get(datanodes.get(6)));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java
deleted file mode 100644
index b652b6b..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.placement.algorithms;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-
-import org.junit.Assert;
-import org.junit.Test;
-import static org.mockito.Matchers.anyObject;
-import org.mockito.Mockito;
-import static org.mockito.Mockito.when;
-
-public class TestSCMContainerPlacementRandom {
-
- @Test
- public void chooseDatanodes() throws SCMException {
- //given
- Configuration conf = new OzoneConfiguration();
-
- List<DatanodeDetails> datanodes = new ArrayList<>();
- for (int i = 0; i < 5; i++) {
- datanodes.add(TestUtils.randomDatanodeDetails());
- }
-
- NodeManager mockNodeManager = Mockito.mock(NodeManager.class);
- when(mockNodeManager.getNodes(NodeState.HEALTHY))
- .thenReturn(new ArrayList<>(datanodes));
-
- when(mockNodeManager.getNodeStat(anyObject()))
- .thenReturn(new SCMNodeMetric(100L, 0L, 100L));
- when(mockNodeManager.getNodeStat(datanodes.get(2)))
- .thenReturn(new SCMNodeMetric(100L, 90L, 10L));
-
- SCMContainerPlacementRandom scmContainerPlacementRandom =
- new SCMContainerPlacementRandom(mockNodeManager, conf);
-
- List<DatanodeDetails> existingNodes = new ArrayList<>();
- existingNodes.add(datanodes.get(0));
- existingNodes.add(datanodes.get(1));
-
- for (int i = 0; i < 100; i++) {
- //when
- List<DatanodeDetails> datanodeDetails =
- scmContainerPlacementRandom.chooseDatanodes(existingNodes, 1, 15);
-
- //then
- Assert.assertEquals(1, datanodeDetails.size());
- DatanodeDetails datanode0Details = datanodeDetails.get(0);
-
- Assert.assertNotEquals(
- "Datanode 0 should not been selected: excluded by parameter",
- datanodes.get(0), datanode0Details);
- Assert.assertNotEquals(
- "Datanode 1 should not been selected: excluded by parameter",
- datanodes.get(1), datanode0Details);
- Assert.assertNotEquals(
- "Datanode 2 should not been selected: not enough space there",
- datanodes.get(2), datanode0Details);
-
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationActivityStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationActivityStatus.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationActivityStatus.java
deleted file mode 100644
index a4615fc..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationActivityStatus.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.container.replication;
-
-import static org.junit.Assert.*;
-
-import java.util.concurrent.TimeoutException;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * Tests for ReplicationActivityStatus.
- */
-public class TestReplicationActivityStatus {
-
- private static EventQueue eventQueue;
- private static ReplicationActivityStatus replicationActivityStatus;
-
- @BeforeClass
- public static void setup() {
- eventQueue = new EventQueue();
- replicationActivityStatus = new ReplicationActivityStatus();
- eventQueue.addHandler(SCMEvents.START_REPLICATION,
- replicationActivityStatus.getReplicationStatusListener());
- eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS,
- replicationActivityStatus.getChillModeStatusListener());
- }
-
- @Test
- public void testReplicationStatusForChillMode()
- throws TimeoutException, InterruptedException {
- assertFalse(replicationActivityStatus.isReplicationEnabled());
- // In chill mode replication process should be stopped.
- eventQueue.fireEvent(SCMEvents.CHILL_MODE_STATUS, true);
- assertFalse(replicationActivityStatus.isReplicationEnabled());
-
- // Replication should be enabled when chill mode if off.
- eventQueue.fireEvent(SCMEvents.CHILL_MODE_STATUS, false);
- GenericTestUtils.waitFor(() -> {
- return replicationActivityStatus.isReplicationEnabled();
- }, 10, 1000*5);
- assertTrue(replicationActivityStatus.isReplicationEnabled());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java
deleted file mode 100644
index 06beb7c..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java
+++ /dev/null
@@ -1,238 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Objects;
-
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ReplicateContainerCommandProto;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.container.ContainerStateManager;
-import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
-import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager.ReplicationRequestToRepeat;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.ozone.lease.LeaseManager;
-import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
-
-import com.google.common.base.Preconditions;
-import static org.apache.hadoop.hdds.scm.events.SCMEvents.TRACK_REPLICATE_COMMAND;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import static org.mockito.Matchers.anyObject;
-import org.mockito.Mockito;
-import static org.mockito.Mockito.when;
-
-/**
- * Test behaviour of the TestReplication.
- */
-public class TestReplicationManager {
-
- private EventQueue queue;
-
- private List<ReplicationRequestToRepeat> trackReplicationEvents;
-
- private List<CommandForDatanode<ReplicateContainerCommandProto>> copyEvents;
-
- private ContainerStateManager containerStateManager;
-
- private ContainerPlacementPolicy containerPlacementPolicy;
- private List<DatanodeDetails> listOfDatanodeDetails;
- private LeaseManager<Long> leaseManager;
- private ReplicationManager replicationManager;
-
- @Before
- public void initReplicationManager() throws IOException {
-
- listOfDatanodeDetails = new ArrayList<>();
- listOfDatanodeDetails.add(TestUtils.randomDatanodeDetails());
- listOfDatanodeDetails.add(TestUtils.randomDatanodeDetails());
- listOfDatanodeDetails.add(TestUtils.randomDatanodeDetails());
- listOfDatanodeDetails.add(TestUtils.randomDatanodeDetails());
- listOfDatanodeDetails.add(TestUtils.randomDatanodeDetails());
-
- containerPlacementPolicy =
- (excludedNodes, nodesRequired, sizeRequired) -> listOfDatanodeDetails
- .subList(2, 2 + nodesRequired);
-
- containerStateManager = Mockito.mock(ContainerStateManager.class);
-
- ContainerInfo containerInfo = new ContainerInfo.Builder()
- .setState(LifeCycleState.CLOSED)
- .build();
-
- when(containerStateManager.getContainer(anyObject()))
- .thenReturn(containerInfo);
-
- when(containerStateManager.getContainerReplicas(new ContainerID(1L)))
- .thenReturn(new HashSet<>(Arrays.asList(
- listOfDatanodeDetails.get(0),
- listOfDatanodeDetails.get(1)
- )));
-
-
- when(containerStateManager.getContainerReplicas(new ContainerID(3L)))
- .thenReturn(new HashSet<>());
-
- queue = new EventQueue();
-
- trackReplicationEvents = new ArrayList<>();
- queue.addHandler(TRACK_REPLICATE_COMMAND,
- (event, publisher) -> trackReplicationEvents.add(event));
-
- copyEvents = new ArrayList<>();
- queue.addHandler(SCMEvents.DATANODE_COMMAND,
- (event, publisher) -> copyEvents.add(event));
-
- leaseManager = new LeaseManager<>("Test", 100000L);
-
- replicationManager = new ReplicationManager(containerPlacementPolicy,
- containerStateManager, queue, leaseManager);
-
-
-
- }
-
- /**
- * Container should be replicated but no source replicas.
- */
- @Test()
- public void testNoExistingReplicas() throws InterruptedException {
- try {
- leaseManager.start();
- replicationManager.start();
-
- //WHEN
- queue.fireEvent(SCMEvents.REPLICATE_CONTAINER,
- new ReplicationRequest(3L, (short) 2, System.currentTimeMillis(),
- (short) 3));
-
- Thread.sleep(500L);
- queue.processAll(1000L);
-
- //THEN
- Assert.assertEquals(0, trackReplicationEvents.size());
- Assert.assertEquals(0, copyEvents.size());
-
- } finally {
- if (leaseManager != null) {
- leaseManager.shutdown();
- }
- }
- }
-
- @Test
- public void testEventSending() throws InterruptedException, IOException {
-
- //GIVEN
- try {
- leaseManager.start();
-
- replicationManager.start();
-
- //WHEN
- queue.fireEvent(SCMEvents.REPLICATE_CONTAINER,
- new ReplicationRequest(1L, (short) 2, System.currentTimeMillis(),
- (short) 3));
-
- Thread.sleep(500L);
- queue.processAll(1000L);
-
- //THEN
- Assert.assertEquals(1, trackReplicationEvents.size());
- Assert.assertEquals(1, copyEvents.size());
- } finally {
- if (leaseManager != null) {
- leaseManager.shutdown();
- }
- }
- }
-
- @Test
- public void testCommandWatcher() throws InterruptedException, IOException {
- LeaseManager<Long> rapidLeaseManager =
- new LeaseManager<>("Test", 1000L);
-
- replicationManager = new ReplicationManager(containerPlacementPolicy,
- containerStateManager, queue, rapidLeaseManager);
-
- try {
- rapidLeaseManager.start();
- replicationManager.start();
-
- queue.fireEvent(SCMEvents.REPLICATE_CONTAINER,
- new ReplicationRequest(1L, (short) 2, System.currentTimeMillis(),
- (short) 3));
-
- Thread.sleep(500L);
-
- queue.processAll(1000L);
-
- Assert.assertEquals(1, trackReplicationEvents.size());
- Assert.assertEquals(1, copyEvents.size());
-
- Assert.assertEquals(trackReplicationEvents.get(0).getId(),
- copyEvents.get(0).getCommand().getId());
-
- //event is timed out
- Thread.sleep(1500);
-
- queue.processAll(1000L);
-
- //original copy command + retry
- Assert.assertEquals(2, trackReplicationEvents.size());
- Assert.assertEquals(2, copyEvents.size());
-
- } finally {
- if (rapidLeaseManager != null) {
- rapidLeaseManager.shutdown();
- }
- }
- }
-
- public static Pipeline createPipeline(Iterable<DatanodeDetails> ids)
- throws IOException {
- Objects.requireNonNull(ids, "ids == null");
- final Iterator<DatanodeDetails> i = ids.iterator();
- Preconditions.checkArgument(i.hasNext());
- final DatanodeDetails leader = i.next();
- final Pipeline pipeline =
- new Pipeline(leader.getUuidString(), LifeCycleState.OPEN,
- ReplicationType.STAND_ALONE, ReplicationFactor.ONE,
- PipelineID.randomId());
- pipeline.addMember(leader);
- while (i.hasNext()) {
- pipeline.addMember(i.next());
- }
- return pipeline;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationQueue.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationQueue.java
deleted file mode 100644
index 9dd4fe3..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationQueue.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-
-import java.util.Random;
-import java.util.UUID;
-import org.apache.hadoop.util.Time;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Test class for ReplicationQueue.
- */
-public class TestReplicationQueue {
-
- private ReplicationQueue replicationQueue;
- private Random random;
-
- @Before
- public void setUp() {
- replicationQueue = new ReplicationQueue();
- random = new Random();
- }
-
- @Test
- public void testDuplicateAddOp() throws InterruptedException {
- long contId = random.nextLong();
- String nodeId = UUID.randomUUID().toString();
- ReplicationRequest obj1, obj2, obj3;
- long time = Time.monotonicNow();
- obj1 = new ReplicationRequest(contId, (short) 2, time, (short) 3);
- obj2 = new ReplicationRequest(contId, (short) 2, time + 1, (short) 3);
- obj3 = new ReplicationRequest(contId, (short) 1, time+2, (short) 3);
-
- replicationQueue.add(obj1);
- replicationQueue.add(obj2);
- replicationQueue.add(obj3);
- Assert.assertEquals("Should add only 1 msg as second one is duplicate",
- 1, replicationQueue.size());
- ReplicationRequest temp = replicationQueue.take();
- Assert.assertEquals(temp, obj3);
- }
-
- @Test
- public void testPollOp() throws InterruptedException {
- long contId = random.nextLong();
- String nodeId = UUID.randomUUID().toString();
- ReplicationRequest msg1, msg2, msg3, msg4, msg5;
- msg1 = new ReplicationRequest(contId, (short) 1, Time.monotonicNow(),
- (short) 3);
- long time = Time.monotonicNow();
- msg2 = new ReplicationRequest(contId + 1, (short) 4, time, (short) 3);
- msg3 = new ReplicationRequest(contId + 2, (short) 0, time, (short) 3);
- msg4 = new ReplicationRequest(contId, (short) 2, time, (short) 3);
- // Replication message for same container but different nodeId
- msg5 = new ReplicationRequest(contId + 1, (short) 2, time, (short) 3);
-
- replicationQueue.add(msg1);
- replicationQueue.add(msg2);
- replicationQueue.add(msg3);
- replicationQueue.add(msg4);
- replicationQueue.add(msg5);
- Assert.assertEquals("Should have 3 objects",
- 3, replicationQueue.size());
-
- // Since Priority queue orders messages according to replication count,
- // message with lowest replication should be first
- ReplicationRequest temp;
- temp = replicationQueue.take();
- Assert.assertEquals("Should have 2 objects",
- 2, replicationQueue.size());
- Assert.assertEquals(temp, msg3);
-
- temp = replicationQueue.take();
- Assert.assertEquals("Should have 1 objects",
- 1, replicationQueue.size());
- Assert.assertEquals(temp, msg5);
-
- // Message 2 should be ordered before message 5 as both have same
- // replication number but message 2 has earlier timestamp.
- temp = replicationQueue.take();
- Assert.assertEquals("Should have 0 objects",
- replicationQueue.size(), 0);
- Assert.assertEquals(temp, msg4);
- }
-
- @Test
- public void testRemoveOp() {
- long contId = random.nextLong();
- String nodeId = UUID.randomUUID().toString();
- ReplicationRequest obj1, obj2, obj3;
- obj1 = new ReplicationRequest(contId, (short) 1, Time.monotonicNow(),
- (short) 3);
- obj2 = new ReplicationRequest(contId + 1, (short) 2, Time.monotonicNow(),
- (short) 3);
- obj3 = new ReplicationRequest(contId + 2, (short) 3, Time.monotonicNow(),
- (short) 3);
-
- replicationQueue.add(obj1);
- replicationQueue.add(obj2);
- replicationQueue.add(obj3);
- Assert.assertEquals("Should have 3 objects",
- 3, replicationQueue.size());
-
- replicationQueue.remove(obj3);
- Assert.assertEquals("Should have 2 objects",
- 2, replicationQueue.size());
-
- replicationQueue.remove(obj2);
- Assert.assertEquals("Should have 1 objects",
- 1, replicationQueue.size());
-
- replicationQueue.remove(obj1);
- Assert.assertEquals("Should have 0 objects",
- 0, replicationQueue.size());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
deleted file mode 100644
index 1423c99..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-/**
- * SCM Testing and Mocking Utils.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-// Test classes for Replication functionality.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerAttribute.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerAttribute.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerAttribute.java
deleted file mode 100644
index 63cc9bf..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerAttribute.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.container.states;
-
-import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Test ContainerAttribute management.
- */
-public class TestContainerAttribute {
-
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
- @Test
- public void testInsert() throws SCMException {
- ContainerAttribute<Integer> containerAttribute = new ContainerAttribute<>();
- ContainerID id = new ContainerID(42);
- containerAttribute.insert(1, id);
- Assert.assertEquals(1,
- containerAttribute.getCollection(1).size());
- Assert.assertTrue(containerAttribute.getCollection(1).contains(id));
-
- // Insert again and verify that it overwrites an existing value.
- ContainerID newId =
- new ContainerID(42);
- containerAttribute.insert(1, newId);
- Assert.assertEquals(1,
- containerAttribute.getCollection(1).size());
- Assert.assertTrue(containerAttribute.getCollection(1).contains(newId));
- }
-
- @Test
- public void testHasKey() throws SCMException {
- ContainerAttribute<Integer> containerAttribute = new ContainerAttribute<>();
-
- for (int x = 1; x < 42; x++) {
- containerAttribute.insert(1, new ContainerID(x));
- }
- Assert.assertTrue(containerAttribute.hasKey(1));
- for (int x = 1; x < 42; x++) {
- Assert.assertTrue(containerAttribute.hasContainerID(1, x));
- }
-
- Assert.assertFalse(containerAttribute.hasContainerID(1,
- new ContainerID(42)));
- }
-
- @Test
- public void testClearSet() throws SCMException {
- List<String> keyslist = Arrays.asList("Key1", "Key2", "Key3");
- ContainerAttribute<String> containerAttribute = new ContainerAttribute<>();
- for (String k : keyslist) {
- for (int x = 1; x < 101; x++) {
- containerAttribute.insert(k, new ContainerID(x));
- }
- }
- for (String k : keyslist) {
- Assert.assertEquals(100,
- containerAttribute.getCollection(k).size());
- }
- containerAttribute.clearSet("Key1");
- Assert.assertEquals(0,
- containerAttribute.getCollection("Key1").size());
- }
-
- @Test
- public void testRemove() throws SCMException {
-
- List<String> keyslist = Arrays.asList("Key1", "Key2", "Key3");
- ContainerAttribute<String> containerAttribute = new ContainerAttribute<>();
-
- for (String k : keyslist) {
- for (int x = 1; x < 101; x++) {
- containerAttribute.insert(k, new ContainerID(x));
- }
- }
- for (int x = 1; x < 101; x += 2) {
- containerAttribute.remove("Key1", new ContainerID(x));
- }
-
- for (int x = 1; x < 101; x += 2) {
- Assert.assertFalse(containerAttribute.hasContainerID("Key1",
- new ContainerID(x)));
- }
-
- Assert.assertEquals(100,
- containerAttribute.getCollection("Key2").size());
-
- Assert.assertEquals(100,
- containerAttribute.getCollection("Key3").size());
-
- Assert.assertEquals(50,
- containerAttribute.getCollection("Key1").size());
- }
-
- @Test
- public void tesUpdate() throws SCMException {
- String key1 = "Key1";
- String key2 = "Key2";
- String key3 = "Key3";
-
- ContainerAttribute<String> containerAttribute = new ContainerAttribute<>();
- ContainerID id = new ContainerID(42);
-
- containerAttribute.insert(key1, id);
- Assert.assertTrue(containerAttribute.hasContainerID(key1, id));
- Assert.assertFalse(containerAttribute.hasContainerID(key2, id));
-
- // This should move the id from key1 bucket to key2 bucket.
- containerAttribute.update(key1, key2, id);
- Assert.assertFalse(containerAttribute.hasContainerID(key1, id));
- Assert.assertTrue(containerAttribute.hasContainerID(key2, id));
-
- // This should fail since we cannot find this id in the key3 bucket.
- thrown.expect(SCMException.class);
- containerAttribute.update(key3, key1, id);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/package-info.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/package-info.java
deleted file mode 100644
index 795dfc1..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-/**
- * Make CheckStyle Happy.
- */
-package org.apache.hadoop.hdds.scm.container.states;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java
deleted file mode 100644
index 6f8534d..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.node;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.XceiverClientManager;
-import org.apache.hadoop.hdds.scm.container.ContainerMapping;
-import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
- .ContainerPlacementPolicy;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
- .SCMContainerPlacementCapacity;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.test.PathUtils;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.mockito.Mockito;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_DB_CACHE_SIZE_MB;
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState
- .HEALTHY;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test for different container placement policy.
- */
-public class TestContainerPlacement {
- @Rule
- public ExpectedException thrown = ExpectedException.none();
- private static XceiverClientManager xceiverClientManager =
- new XceiverClientManager(new OzoneConfiguration());
-
- /**
- * Returns a new copy of Configuration.
- *
- * @return Config
- */
- OzoneConfiguration getConf() {
- return new OzoneConfiguration();
- }
-
- /**
- * Creates a NodeManager.
- *
- * @param config - Config for the node manager.
- * @return SCNNodeManager
- * @throws IOException
- */
-
- SCMNodeManager createNodeManager(OzoneConfiguration config)
- throws IOException {
- EventQueue eventQueue = new EventQueue();
- eventQueue.addHandler(SCMEvents.NEW_NODE,
- Mockito.mock(NewNodeHandler.class));
- eventQueue.addHandler(SCMEvents.STALE_NODE,
- Mockito.mock(StaleNodeHandler.class));
- eventQueue.addHandler(SCMEvents.DEAD_NODE,
- Mockito.mock(DeadNodeHandler.class));
- SCMNodeManager nodeManager = new SCMNodeManager(config,
- UUID.randomUUID().toString(), null, eventQueue);
- assertFalse("Node manager should be in chill mode",
- nodeManager.isOutOfChillMode());
- return nodeManager;
- }
-
- ContainerMapping createContainerManager(Configuration config,
- NodeManager scmNodeManager) throws IOException {
- EventQueue eventQueue = new EventQueue();
- final int cacheSize = config.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
- OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
- return new ContainerMapping(config, scmNodeManager, cacheSize, eventQueue);
-
- }
-
- /**
- * Test capacity based container placement policy with node reports.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws TimeoutException
- */
- @Test
- @Ignore
- public void testContainerPlacementCapacity() throws IOException,
- InterruptedException, TimeoutException {
- OzoneConfiguration conf = getConf();
- final int nodeCount = 4;
- final long capacity = 10L * OzoneConsts.GB;
- final long used = 2L * OzoneConsts.GB;
- final long remaining = capacity - used;
-
- final File testDir = PathUtils.getTestDir(
- TestContainerPlacement.class);
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS,
- testDir.getAbsolutePath());
- conf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
-
- SCMNodeManager nodeManager = createNodeManager(conf);
- ContainerMapping containerManager =
- createContainerManager(conf, nodeManager);
- List<DatanodeDetails> datanodes =
- TestUtils.getListOfRegisteredDatanodeDetails(nodeManager, nodeCount);
- try {
- for (DatanodeDetails datanodeDetails : datanodes) {
- nodeManager.processHeartbeat(datanodeDetails);
- }
-
- //TODO: wait for heartbeat to be processed
- Thread.sleep(4 * 1000);
- assertEquals(nodeCount, nodeManager.getNodeCount(HEALTHY));
- assertEquals(capacity * nodeCount,
- (long) nodeManager.getStats().getCapacity().get());
- assertEquals(used * nodeCount,
- (long) nodeManager.getStats().getScmUsed().get());
- assertEquals(remaining * nodeCount,
- (long) nodeManager.getStats().getRemaining().get());
-
- assertTrue(nodeManager.isOutOfChillMode());
-
- ContainerWithPipeline containerWithPipeline = containerManager
- .allocateContainer(
- xceiverClientManager.getType(),
- xceiverClientManager.getFactor(), "OZONE");
- assertEquals(xceiverClientManager.getFactor().getNumber(),
- containerWithPipeline.getPipeline().getMachines().size());
- } finally {
- IOUtils.closeQuietly(containerManager);
- IOUtils.closeQuietly(nodeManager);
- FileUtil.fullyDelete(testDir);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
deleted file mode 100644
index 7bba032..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.node;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.container.ContainerStateManager;
-import org.apache.hadoop.hdds.scm.container.Mapping;
-import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
-import org.apache.hadoop.hdds.scm.container.replication.ReplicationRequest;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeReportFromDatanode;
-import org.apache.hadoop.hdds.server.events.EventPublisher;
-
-import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentCaptor;
-import static org.mockito.Matchers.eq;
-import org.mockito.Mockito;
-
-/**
- * Test DeadNodeHandler.
- */
-public class TestDeadNodeHandler {
-
- private List<ReplicationRequest> sentEvents = new ArrayList<>();
- private SCMNodeManager nodeManager;
- private ContainerStateManager containerStateManager;
- private NodeReportHandler nodeReportHandler;
- private DeadNodeHandler deadNodeHandler;
- private EventPublisher publisher;
- private EventQueue eventQueue;
-
- @Before
- public void setup() throws IOException {
- OzoneConfiguration conf = new OzoneConfiguration();
- containerStateManager = new ContainerStateManager(conf,
- Mockito.mock(Mapping.class),
- Mockito.mock(PipelineSelector.class));
- eventQueue = new EventQueue();
- nodeManager = new SCMNodeManager(conf, "cluster1", null, eventQueue);
- deadNodeHandler = new DeadNodeHandler(nodeManager,
- containerStateManager);
- eventQueue.addHandler(SCMEvents.DEAD_NODE, deadNodeHandler);
- publisher = Mockito.mock(EventPublisher.class);
- nodeReportHandler = new NodeReportHandler(nodeManager);
- }
-
- @Test
- public void testOnMessage() throws IOException {
- //GIVEN
- DatanodeDetails datanode1 = TestUtils.randomDatanodeDetails();
- DatanodeDetails datanode2 = TestUtils.randomDatanodeDetails();
-
- ContainerInfo container1 =
- TestUtils.allocateContainer(containerStateManager);
- ContainerInfo container2 =
- TestUtils.allocateContainer(containerStateManager);
- ContainerInfo container3 =
- TestUtils.allocateContainer(containerStateManager);
-
- registerReplicas(datanode1, container1, container2);
- registerReplicas(datanode2, container1, container3);
-
- registerReplicas(containerStateManager, container1, datanode1, datanode2);
- registerReplicas(containerStateManager, container2, datanode1);
- registerReplicas(containerStateManager, container3, datanode2);
-
- TestUtils.closeContainer(containerStateManager, container1);
-
- deadNodeHandler.onMessage(datanode1, publisher);
-
- Set<DatanodeDetails> container1Replicas =
- containerStateManager.getContainerStateMap()
- .getContainerReplicas(new ContainerID(container1.getContainerID()));
- Assert.assertEquals(1, container1Replicas.size());
- Assert.assertEquals(datanode2, container1Replicas.iterator().next());
-
- Set<DatanodeDetails> container2Replicas =
- containerStateManager.getContainerStateMap()
- .getContainerReplicas(new ContainerID(container2.getContainerID()));
- Assert.assertEquals(0, container2Replicas.size());
-
- Set<DatanodeDetails> container3Replicas =
- containerStateManager.getContainerStateMap()
- .getContainerReplicas(new ContainerID(container3.getContainerID()));
- Assert.assertEquals(1, container3Replicas.size());
- Assert.assertEquals(datanode2, container3Replicas.iterator().next());
-
- ArgumentCaptor<ReplicationRequest> replicationRequestParameter =
- ArgumentCaptor.forClass(ReplicationRequest.class);
-
- Mockito.verify(publisher)
- .fireEvent(eq(SCMEvents.REPLICATE_CONTAINER),
- replicationRequestParameter.capture());
-
- Assert
- .assertEquals(container1.getContainerID(),
- replicationRequestParameter.getValue().getContainerId());
- Assert
- .assertEquals(1,
- replicationRequestParameter.getValue().getReplicationCount());
- Assert
- .assertEquals(3,
- replicationRequestParameter.getValue().getExpecReplicationCount());
- }
-
- @Test
- public void testStatisticsUpdate() throws Exception {
- //GIVEN
- DatanodeDetails datanode1 = TestUtils.randomDatanodeDetails();
- DatanodeDetails datanode2 = TestUtils.randomDatanodeDetails();
- String storagePath1 = GenericTestUtils.getRandomizedTempPath()
- .concat("/" + datanode1.getUuidString());
- String storagePath2 = GenericTestUtils.getRandomizedTempPath()
- .concat("/" + datanode2.getUuidString());
-
- StorageReportProto storageOne = TestUtils.createStorageReport(
- datanode1.getUuid(), storagePath1, 100, 10, 90, null);
- StorageReportProto storageTwo = TestUtils.createStorageReport(
- datanode2.getUuid(), storagePath2, 200, 20, 180, null);
- nodeReportHandler.onMessage(getNodeReport(datanode1, storageOne),
- Mockito.mock(EventPublisher.class));
- nodeReportHandler.onMessage(getNodeReport(datanode2, storageTwo),
- Mockito.mock(EventPublisher.class));
-
- ContainerInfo container1 =
- TestUtils.allocateContainer(containerStateManager);
- registerReplicas(datanode1, container1);
-
- SCMNodeStat stat = nodeManager.getStats();
- Assert.assertTrue(stat.getCapacity().get() == 300);
- Assert.assertTrue(stat.getRemaining().get() == 270);
- Assert.assertTrue(stat.getScmUsed().get() == 30);
-
- SCMNodeMetric nodeStat = nodeManager.getNodeStat(datanode1);
- Assert.assertTrue(nodeStat.get().getCapacity().get() == 100);
- Assert.assertTrue(nodeStat.get().getRemaining().get() == 90);
- Assert.assertTrue(nodeStat.get().getScmUsed().get() == 10);
-
- //WHEN datanode1 is dead.
- eventQueue.fireEvent(SCMEvents.DEAD_NODE, datanode1);
- Thread.sleep(100);
-
- //THEN statistics in SCM should changed.
- stat = nodeManager.getStats();
- Assert.assertTrue(stat.getCapacity().get() == 200);
- Assert.assertTrue(stat.getRemaining().get() == 180);
- Assert.assertTrue(stat.getScmUsed().get() == 20);
-
- nodeStat = nodeManager.getNodeStat(datanode1);
- Assert.assertTrue(nodeStat.get().getCapacity().get() == 0);
- Assert.assertTrue(nodeStat.get().getRemaining().get() == 0);
- Assert.assertTrue(nodeStat.get().getScmUsed().get() == 0);
- }
-
- @Test
- public void testOnMessageReplicaFailure() throws Exception {
- DatanodeDetails dn1 = TestUtils.randomDatanodeDetails();
- GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
- .captureLogs(DeadNodeHandler.getLogger());
- String storagePath1 = GenericTestUtils.getRandomizedTempPath()
- .concat("/" + dn1.getUuidString());
-
- StorageReportProto storageOne = TestUtils.createStorageReport(
- dn1.getUuid(), storagePath1, 100, 10, 90, null);
- nodeReportHandler.onMessage(getNodeReport(dn1, storageOne),
- Mockito.mock(EventPublisher.class));
-
- ContainerInfo container1 =
- TestUtils.allocateContainer(containerStateManager);
- registerReplicas(dn1, container1);
-
- deadNodeHandler.onMessage(dn1, eventQueue);
- Assert.assertTrue(logCapturer.getOutput().contains(
- "DataNode " + dn1.getUuid() + " doesn't have replica for container "
- + container1.getContainerID()));
- }
-
- private void registerReplicas(ContainerStateManager csm,
- ContainerInfo container, DatanodeDetails... datanodes) {
- csm.getContainerStateMap()
- .addContainerReplica(new ContainerID(container.getContainerID()),
- datanodes);
- }
-
- private void registerReplicas(DatanodeDetails datanode,
- ContainerInfo... containers)
- throws SCMException {
- nodeManager
- .addDatanodeInContainerMap(datanode.getUuid(),
- Arrays.stream(containers)
- .map(container -> new ContainerID(container.getContainerID()))
- .collect(Collectors.toSet()));
- }
-
- private NodeReportFromDatanode getNodeReport(DatanodeDetails dn,
- StorageReportProto... reports) {
- NodeReportProto nodeReportProto = TestUtils.createNodeReport(reports);
- return new NodeReportFromDatanode(dn, nodeReportProto);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org