You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ms...@apache.org on 2018/07/24 15:54:02 UTC
hadoop git commit: HDDS-272. TestBlockDeletingService is failing with
DiskOutOfSpaceException. Contributed by Lokesh Jain.
Repository: hadoop
Updated Branches:
refs/heads/trunk ff7c2eda3 -> 773d312f7
HDDS-272. TestBlockDeletingService is failing with DiskOutOfSpaceException. Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/773d312f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/773d312f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/773d312f
Branch: refs/heads/trunk
Commit: 773d312f7412d5050c106ed3a1cd0d1934bfa2e0
Parents: ff7c2ed
Author: Mukul Kumar Singh <ms...@apache.org>
Authored: Tue Jul 24 21:23:20 2018 +0530
Committer: Mukul Kumar Singh <ms...@apache.org>
Committed: Tue Jul 24 21:23:20 2018 +0530
----------------------------------------------------------------------
.../container/keyvalue/KeyValueHandler.java | 2 +-
.../background/BlockDeletingService.java | 9 +-
.../testutils/BlockDeletingServiceTestImpl.java | 3 +-
.../common/TestBlockDeletingService.java | 90 ++++++++++++--------
4 files changed, 60 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773d312f/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 9aa3df7..d3a1ca4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -136,7 +136,7 @@ public class KeyValueHandler extends Handler {
TimeUnit.MILLISECONDS);
this.blockDeletingService =
new BlockDeletingService(containerSet, svcInterval, serviceTimeout,
- config);
+ TimeUnit.MILLISECONDS, config);
blockDeletingService.start();
// TODO: Add supoort for different volumeChoosingPolicies.
volumeChoosingPolicy = new RoundRobinVolumeChoosingPolicy();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773d312f/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
index a3e36f4..4a572ca 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
@@ -88,11 +88,10 @@ public class BlockDeletingService extends BackgroundService{
// Core pool size for container tasks
private final static int BLOCK_DELETING_SERVICE_CORE_POOL_SIZE = 10;
- public BlockDeletingService(ContainerSet containerSet,
- long serviceInterval, long serviceTimeout, Configuration conf) {
- super("BlockDeletingService", serviceInterval,
- TimeUnit.MILLISECONDS, BLOCK_DELETING_SERVICE_CORE_POOL_SIZE,
- serviceTimeout);
+ public BlockDeletingService(ContainerSet containerSet, long serviceInterval,
+ long serviceTimeout, TimeUnit timeUnit, Configuration conf) {
+ super("BlockDeletingService", serviceInterval, timeUnit,
+ BLOCK_DELETING_SERVICE_CORE_POOL_SIZE, serviceTimeout);
this.containerSet = containerSet;
containerDeletionPolicy = ReflectionUtils.newInstance(conf.getClass(
ScmConfigKeys.OZONE_SCM_KEY_VALUE_CONTAINER_DELETION_CHOOSING_POLICY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773d312f/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
index a87f655..115b5e2 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
@@ -44,7 +44,8 @@ public class BlockDeletingServiceTestImpl
public BlockDeletingServiceTestImpl(ContainerSet containerSet,
int serviceInterval, Configuration conf) {
- super(containerSet, serviceInterval, SERVICE_TIMEOUT_IN_MILLISECONDS, conf);
+ super(containerSet, serviceInterval, SERVICE_TIMEOUT_IN_MILLISECONDS,
+ TimeUnit.MILLISECONDS, conf);
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773d312f/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
index 1ddd39a..a6e53c2 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
@@ -45,17 +45,17 @@ import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
import org.apache.hadoop.utils.BackgroundService;
import org.apache.hadoop.utils.MetadataKeyFilters;
import org.apache.hadoop.utils.MetadataStore;
+import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Test;
import org.junit.BeforeClass;
-import org.junit.Before;
-import org.junit.After;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@@ -72,35 +72,28 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys
/**
* Tests to test block deleting service.
*/
-// TODO: Fix BlockDeletingService to work with new StorageLayer
public class TestBlockDeletingService {
private static final Logger LOG =
LoggerFactory.getLogger(TestBlockDeletingService.class);
private static File testRoot;
- private static File containersDir;
- private static File chunksDir;
+ private static String scmId;
+ private static String clusterID;
@BeforeClass
- public static void init() {
+ public static void init() throws IOException {
testRoot = GenericTestUtils
.getTestDir(TestBlockDeletingService.class.getSimpleName());
- chunksDir = new File(testRoot, "chunks");
- containersDir = new File(testRoot, "containers");
- }
-
- @Before
- public void setup() throws IOException {
- if (chunksDir.exists()) {
- FileUtils.deleteDirectory(chunksDir);
+ if (testRoot.exists()) {
+ FileUtils.cleanDirectory(testRoot);
}
+ scmId = UUID.randomUUID().toString();
+ clusterID = UUID.randomUUID().toString();
}
- @After
- public void cleanup() throws IOException {
- FileUtils.deleteDirectory(chunksDir);
- FileUtils.deleteDirectory(containersDir);
+ @AfterClass
+ public static void cleanup() throws IOException {
FileUtils.deleteDirectory(testRoot);
}
@@ -111,14 +104,15 @@ public class TestBlockDeletingService {
*/
private void createToDeleteBlocks(ContainerSet containerSet,
Configuration conf, int numOfContainers, int numOfBlocksPerContainer,
- int numOfChunksPerBlock, File chunkDir) throws IOException {
+ int numOfChunksPerBlock) throws IOException {
for (int x = 0; x < numOfContainers; x++) {
+ conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, testRoot.getAbsolutePath());
long containerID = ContainerTestHelper.getTestContainerID();
KeyValueContainerData data = new KeyValueContainerData(containerID,
ContainerTestHelper.CONTAINER_MAX_SIZE_GB);
Container container = new KeyValueContainer(data, conf);
- container.create(new VolumeSet(UUID.randomUUID().toString(), conf),
- new RoundRobinVolumeChoosingPolicy(), UUID.randomUUID().toString());
+ container.create(new VolumeSet(scmId, clusterID, conf),
+ new RoundRobinVolumeChoosingPolicy(), scmId);
containerSet.addContainer(container);
data = (KeyValueContainerData) containerSet.getContainer(
containerID).getContainerData();
@@ -133,7 +127,7 @@ public class TestBlockDeletingService {
for (int k = 0; k<numOfChunksPerBlock; k++) {
// offset doesn't matter here
String chunkName = blockID.getLocalID() + "_chunk_" + k;
- File chunk = new File(chunkDir, chunkName);
+ File chunk = new File(data.getChunksPath(), chunkName);
FileUtils.writeStringToFile(chunk, "a chunk",
Charset.defaultCharset());
LOG.info("Creating file {}", chunk.getAbsolutePath());
@@ -193,7 +187,7 @@ public class TestBlockDeletingService {
conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10);
conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 2);
ContainerSet containerSet = new ContainerSet();
- createToDeleteBlocks(containerSet, conf, 1, 3, 1, chunksDir);
+ createToDeleteBlocks(containerSet, conf, 1, 3, 1);
BlockDeletingServiceTestImpl svc =
new BlockDeletingServiceTestImpl(containerSet, 1000, conf);
@@ -208,12 +202,13 @@ public class TestBlockDeletingService {
MetadataStore meta = KeyUtils.getDB(
(KeyValueContainerData) containerData.get(0), conf);
Map<Long, Container> containerMap = containerSet.getContainerMap();
+ long transactionId = containerMap.get(containerData.get(0).getContainerID())
+ .getContainerData().getDeleteTransactionId();
// Number of deleted blocks in container should be equal to 0 before
// block delete
- // TODO : Implement deleteTransactionID in ContainerData.
-// Assert.assertEquals(0, transactionId);
+ Assert.assertEquals(0, transactionId);
// Ensure there are 3 blocks under deletion and 0 deleted blocks
Assert.assertEquals(3, getUnderDeletionBlocksCount(meta));
@@ -247,7 +242,7 @@ public class TestBlockDeletingService {
conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 10);
ContainerSet containerSet = new ContainerSet();
// Create 1 container with 100 blocks
- createToDeleteBlocks(containerSet, conf, 1, 100, 1, chunksDir);
+ createToDeleteBlocks(containerSet, conf, 1, 100, 1);
BlockDeletingServiceTestImpl service =
new BlockDeletingServiceTestImpl(containerSet, 1000, conf);
@@ -275,12 +270,13 @@ public class TestBlockDeletingService {
conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10);
conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 2);
ContainerSet containerSet = new ContainerSet();
- createToDeleteBlocks(containerSet, conf, 1, 3, 1, chunksDir);
+ createToDeleteBlocks(containerSet, conf, 1, 3, 1);
// set timeout value as 1ns to trigger timeout behavior
long timeout = 1;
- BlockDeletingService svc =
- new BlockDeletingService(containerSet, 1000, timeout, conf);
+ BlockDeletingService svc = new BlockDeletingService(containerSet,
+ TimeUnit.MILLISECONDS.toNanos(1000), timeout, TimeUnit.NANOSECONDS,
+ conf);
svc.start();
LogCapturer log = LogCapturer.captureLogs(BackgroundService.LOG);
@@ -299,8 +295,10 @@ public class TestBlockDeletingService {
// test for normal case that doesn't have timeout limitation
timeout = 0;
- createToDeleteBlocks(containerSet, conf, 1, 3, 1, chunksDir);
- svc = new BlockDeletingService(containerSet, 1000, timeout, conf);
+ createToDeleteBlocks(containerSet, conf, 1, 3, 1);
+ svc = new BlockDeletingService(containerSet,
+ TimeUnit.MILLISECONDS.toNanos(1000), timeout, TimeUnit.MILLISECONDS,
+ conf);
svc.start();
// get container meta data
@@ -347,7 +345,7 @@ public class TestBlockDeletingService {
conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 1);
conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 1);
ContainerSet containerSet = new ContainerSet();
- createToDeleteBlocks(containerSet, conf, 2, 1, 10, chunksDir);
+ createToDeleteBlocks(containerSet, conf, 2, 1, 10);
BlockDeletingServiceTestImpl service =
new BlockDeletingServiceTestImpl(containerSet, 1000, conf);
@@ -357,7 +355,12 @@ public class TestBlockDeletingService {
GenericTestUtils.waitFor(() -> service.isStarted(), 100, 3000);
// 1st interval processes 1 container 1 block and 10 chunks
deleteAndWait(service, 1);
- Assert.assertEquals(10, chunksDir.listFiles().length);
+ Assert.assertEquals(10, getNumberOfChunksInContainers(containerSet));
+ deleteAndWait(service, 2);
+ deleteAndWait(service, 3);
+ deleteAndWait(service, 4);
+ deleteAndWait(service, 5);
+ Assert.assertEquals(0, getNumberOfChunksInContainers(containerSet));
} finally {
service.shutdown();
}
@@ -383,10 +386,10 @@ public class TestBlockDeletingService {
conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10);
conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 2);
ContainerSet containerSet = new ContainerSet();
- createToDeleteBlocks(containerSet, conf, 5, 3, 1, chunksDir);
+ createToDeleteBlocks(containerSet, conf, 5, 3, 1);
// Make sure chunks are created
- Assert.assertEquals(15, chunksDir.listFiles().length);
+ Assert.assertEquals(15, getNumberOfChunksInContainers(containerSet));
BlockDeletingServiceTestImpl service =
new BlockDeletingServiceTestImpl(containerSet, 1000, conf);
@@ -399,13 +402,26 @@ public class TestBlockDeletingService {
// number of containers = 5
// each interval will at most runDeletingTasks 5 * 2 = 10 blocks
deleteAndWait(service, 1);
- Assert.assertEquals(5, chunksDir.listFiles().length);
+ Assert.assertEquals(5, getNumberOfChunksInContainers(containerSet));
// There is only 5 blocks left to runDeletingTasks
deleteAndWait(service, 2);
- Assert.assertEquals(0, chunksDir.listFiles().length);
+ Assert.assertEquals(0, getNumberOfChunksInContainers(containerSet));
} finally {
service.shutdown();
}
}
+
+ private int getNumberOfChunksInContainers(ContainerSet containerSet) {
+ Iterator<Container> containerIterator = containerSet.getContainerIterator();
+ int numChunks = 0;
+ while (containerIterator.hasNext()) {
+ Container container = containerIterator.next();
+ File chunkDir = FileUtils.getFile(
+ ((KeyValueContainerData) container.getContainerData())
+ .getChunksPath());
+ numChunks += chunkDir.listFiles().length;
+ }
+ return numChunks;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org