You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2019/10/04 04:34:52 UTC
[hadoop] branch trunk updated: HDFS-14889. Ability to check if a
block has a replica on provided storage. Contributed by Ashvin Agrawal.
(#1573)"
This is an automated email from the ASF dual-hosted git repository.
virajith pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 844b766 HDFS-14889. Ability to check if a block has a replica on provided storage. Contributed by Ashvin Agrawal. (#1573)"
844b766 is described below
commit 844b766da535894b792892b38de6bc2500eca57f
Author: Virajith Jalaparti <vi...@apache.org>
AuthorDate: Thu Oct 3 21:31:23 2019 -0700
HDFS-14889. Ability to check if a block has a replica on provided storage. Contributed by Ashvin Agrawal. (#1573)"
---
.../hdfs/server/blockmanagement/BlockInfo.java | 6 ++++
.../blockmanagement/BlockInfoContiguous.java | 14 +++++++++
.../server/blockmanagement/BlockInfoStriped.java | 9 ++++++
.../hdfs/server/blockmanagement/TestBlockInfo.java | 36 ++++++++++++++++++++++
4 files changed, 65 insertions(+)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
index d160f61..dc6cf32 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
@@ -182,6 +182,12 @@ public abstract class BlockInfo extends Block
abstract boolean hasNoStorage();
/**
+ * Checks whether this block has a Provided replica.
+ * @return true if this block has a replica on Provided storage.
+ */
+ abstract boolean isProvided();
+
+ /**
* Find specified DatanodeStorageInfo.
* @return DatanodeStorageInfo or null if not found.
*/
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
index 149efc9..7378e6f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockType;
@@ -81,6 +82,19 @@ public class BlockInfoContiguous extends BlockInfo {
}
@Override
+ boolean isProvided() {
+ int len = getCapacity();
+ for (int idx = 0; idx < len; idx++) {
+ DatanodeStorageInfo storage = getStorageInfo(idx);
+ if (storage != null
+ && storage.getStorageType().equals(StorageType.PROVIDED)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
public int numNodes() {
assert this.storages != null : "BlockInfo is not initialized";
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
index 8bc63c1..16265de 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
@@ -245,6 +245,15 @@ public class BlockInfoStriped extends BlockInfo {
}
/**
+ * Striped blocks on Provided Storage is not supported. All blocks on
+ * Provided storage are assumed to be "contiguous".
+ */
+ @Override
+ boolean isProvided() {
+ return false;
+ }
+
+ /**
* This class contains datanode storage information and block index in the
* block group.
*/
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
index fa0dd70..3c5c5d9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
@@ -19,7 +19,10 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import static org.apache.hadoop.hdfs.server.namenode.INodeId.INVALID_INODE_ID;
import static org.hamcrest.core.Is.is;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import org.apache.hadoop.fs.StorageType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -65,6 +68,39 @@ public class TestBlockInfo {
}
@Test
+ public void testAddProvidedStorage() throws Exception {
+ // block with only provided storage
+ BlockInfo blockInfo = new BlockInfoContiguous((short) 3);
+ DatanodeStorageInfo providedStorage = mock(DatanodeStorageInfo.class);
+ when(providedStorage.getStorageType()).thenReturn(StorageType.PROVIDED);
+ boolean added = blockInfo.addStorage(providedStorage, blockInfo);
+ Assert.assertTrue(added);
+ Assert.assertEquals(providedStorage, blockInfo.getStorageInfo(0));
+ Assert.assertTrue(blockInfo.isProvided());
+ }
+
+ @Test
+ public void testAddTwoStorageTypes() throws Exception {
+ // block with only disk storage
+ BlockInfo blockInfo = new BlockInfoContiguous((short) 3);
+ DatanodeStorageInfo diskStorage = mock(DatanodeStorageInfo.class);
+ DatanodeDescriptor mockDN = mock(DatanodeDescriptor.class);
+ when(diskStorage.getDatanodeDescriptor()).thenReturn(mockDN);
+ when(diskStorage.getStorageType()).thenReturn(StorageType.DISK);
+ boolean added = blockInfo.addStorage(diskStorage, blockInfo);
+ Assert.assertTrue(added);
+ Assert.assertEquals(diskStorage, blockInfo.getStorageInfo(0));
+ Assert.assertFalse(blockInfo.isProvided());
+
+ // now add provided storage
+ DatanodeStorageInfo providedStorage = mock(DatanodeStorageInfo.class);
+ when(providedStorage.getStorageType()).thenReturn(StorageType.PROVIDED);
+ added = blockInfo.addStorage(providedStorage, blockInfo);
+ Assert.assertTrue(added);
+ Assert.assertTrue(blockInfo.isProvided());
+ }
+
+ @Test
public void testReplaceStorage() throws Exception {
// Create two dummy storages.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org