You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2017/12/15 18:38:55 UTC
[09/50] [abbrv] hadoop git commit: HDFS-11703. [READ] Tests for
ProvidedStorageMap
HDFS-11703. [READ] Tests for ProvidedStorageMap
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0793a78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0793a78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0793a78
Branch: refs/heads/HDFS-9806
Commit: e0793a78212aea18ffb6ec55ea287982cfe7cb8b
Parents: 08f6b2d
Author: Virajith Jalaparti <vi...@apache.org>
Authored: Thu May 4 13:14:41 2017 -0700
Committer: Virajith Jalaparti <vi...@apache.org>
Committed: Fri Dec 15 10:18:26 2017 -0800
----------------------------------------------------------------------
.../blockmanagement/ProvidedStorageMap.java | 6 +
.../blockmanagement/TestProvidedStorageMap.java | 153 +++++++++++++++++++
2 files changed, 159 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0793a78/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ProvidedStorageMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ProvidedStorageMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ProvidedStorageMap.java
index 518b7e9..0faf16d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ProvidedStorageMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ProvidedStorageMap.java
@@ -28,6 +28,7 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentSkipListMap;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -121,6 +122,11 @@ public class ProvidedStorageMap {
return dn.getStorageInfo(s.getStorageID());
}
+ @VisibleForTesting
+ public DatanodeStorageInfo getProvidedStorageInfo() {
+ return providedStorageInfo;
+ }
+
public LocatedBlockBuilder newLocatedBlocks(int maxValue) {
if (!providedEnabled) {
return new LocatedBlockBuilder(maxValue);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0793a78/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestProvidedStorageMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestProvidedStorageMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestProvidedStorageMap.java
new file mode 100644
index 0000000..50e2fed
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestProvidedStorageMap.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.util.RwLock;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * This class tests the {@link ProvidedStorageMap}.
+ */
+public class TestProvidedStorageMap {
+
+ private Configuration conf;
+ private BlockManager bm;
+ private RwLock nameSystemLock;
+ private String providedStorageID;
+
+ static class TestBlockProvider extends BlockProvider
+ implements Configurable {
+
+ @Override
+ public void setConf(Configuration conf) {
+ }
+
+ @Override
+ public Configuration getConf() {
+ return null;
+ }
+
+ @Override
+ public Iterator<Block> iterator() {
+ return new Iterator<Block>() {
+ @Override
+ public boolean hasNext() {
+ return false;
+ }
+ @Override
+ public Block next() {
+ return null;
+ }
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+ }
+
+ @Before
+ public void setup() {
+ providedStorageID = DFSConfigKeys.DFS_PROVIDER_STORAGEUUID_DEFAULT;
+ conf = new HdfsConfiguration();
+ conf.set(DFSConfigKeys.DFS_PROVIDER_STORAGEUUID,
+ providedStorageID);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_PROVIDED_ENABLED, true);
+ conf.setClass(DFSConfigKeys.DFS_NAMENODE_BLOCK_PROVIDER_CLASS,
+ TestBlockProvider.class, BlockProvider.class);
+
+ bm = mock(BlockManager.class);
+ nameSystemLock = mock(RwLock.class);
+ }
+
+ private DatanodeDescriptor createDatanodeDescriptor(int port) {
+ return DFSTestUtil.getDatanodeDescriptor("127.0.0.1", port, "defaultRack",
+ "localhost");
+ }
+
+ @Test
+ public void testProvidedStorageMap() throws IOException {
+ ProvidedStorageMap providedMap = new ProvidedStorageMap(
+ nameSystemLock, bm, conf);
+ DatanodeStorageInfo providedMapStorage =
+ providedMap.getProvidedStorageInfo();
+ //the provided storage cannot be null
+ assertNotNull(providedMapStorage);
+
+ //create a datanode
+ DatanodeDescriptor dn1 = createDatanodeDescriptor(5000);
+
+ //associate two storages to the datanode
+ DatanodeStorage dn1ProvidedStorage = new DatanodeStorage(
+ providedStorageID,
+ DatanodeStorage.State.NORMAL,
+ StorageType.PROVIDED);
+ DatanodeStorage dn1DiskStorage = new DatanodeStorage(
+ "sid-1", DatanodeStorage.State.NORMAL, StorageType.DISK);
+
+ when(nameSystemLock.hasWriteLock()).thenReturn(true);
+ DatanodeStorageInfo dns1Provided = providedMap.getStorage(dn1,
+ dn1ProvidedStorage);
+ DatanodeStorageInfo dns1Disk = providedMap.getStorage(dn1,
+ dn1DiskStorage);
+
+ assertTrue("The provided storages should be equal",
+ dns1Provided == providedMapStorage);
+ assertTrue("Disk storage has not yet been registered with block manager",
+ dns1Disk == null);
+ //add the disk storage to the datanode.
+ DatanodeStorageInfo dnsDisk = new DatanodeStorageInfo(dn1, dn1DiskStorage);
+ dn1.injectStorage(dnsDisk);
+ assertTrue("Disk storage must match the injected storage info",
+ dnsDisk == providedMap.getStorage(dn1, dn1DiskStorage));
+
+ //create a 2nd datanode
+ DatanodeDescriptor dn2 = createDatanodeDescriptor(5010);
+ //associate a provided storage with the datanode
+ DatanodeStorage dn2ProvidedStorage = new DatanodeStorage(
+ providedStorageID,
+ DatanodeStorage.State.NORMAL,
+ StorageType.PROVIDED);
+
+ DatanodeStorageInfo dns2Provided = providedMap.getStorage(
+ dn2, dn2ProvidedStorage);
+ assertTrue("The provided storages should be equal",
+ dns2Provided == providedMapStorage);
+ assertTrue("The DatanodeDescriptor should contain the provided storage",
+ dn2.getStorageInfo(providedStorageID) == providedMapStorage);
+
+
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org