You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ki...@apache.org on 2014/09/16 16:05:57 UTC
git commit: HDFS-6965. NN continues to issue block locations for DNs
with full disks. Contributed by Rushabh Shah.
Repository: hadoop
Updated Branches:
refs/heads/trunk 7e08c0f23 -> 0c26412be
HDFS-6965. NN continues to issue block locations for DNs with full
disks. Contributed by Rushabh Shah.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c26412b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c26412b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c26412b
Branch: refs/heads/trunk
Commit: 0c26412be4b3ec40130b7200506c957f0402ecbc
Parents: 7e08c0f
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue Sep 16 09:04:54 2014 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Tue Sep 16 09:04:54 2014 -0500
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../BlockPlacementPolicyDefault.java | 2 +-
.../blockmanagement/TestBlockManager.java | 55 ++++++++++++++++++++
3 files changed, 59 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c26412b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f596265..819f636 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -653,6 +653,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7032. Add WebHDFS support for reading and writing to encryption zones.
(clamb via wang)
+ HDFS-6965. NN continues to issue block locations for DNs with full disks.
+ (Rushabh Shah via kihwal)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c26412b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index e2026c1..f77d4ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -635,7 +635,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
final long requiredSize = blockSize * HdfsConstants.MIN_BLOCKS_FOR_WRITE;
final long scheduledSize = blockSize * node.getBlocksScheduled();
- if (requiredSize > node.getRemaining() - scheduledSize) {
+ if (requiredSize > storage.getRemaining() - scheduledSize) {
logNodeIsNotChosen(storage, "the node does not have enough space ");
return false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c26412b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 41af237..1a8262f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -35,17 +35,24 @@ import java.util.List;
import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
@@ -599,5 +606,53 @@ public class TestBlockManager {
new BlockListAsLongs(null, null));
assertEquals(1, ds.getBlockReportCount());
}
+
+
+ /**
+ * Tests that a namenode doesn't choose a datanode with full disks to
+ * store blocks.
+ * @throws Exception
+ */
+ @Test
+ public void testStorageWithRemainingCapacity() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ FileSystem fs = FileSystem.get(conf);
+ Path file1 = null;
+ try {
+ cluster.waitActive();
+ final FSNamesystem namesystem = cluster.getNamesystem();
+ final String poolId = namesystem.getBlockPoolId();
+ final DatanodeRegistration nodeReg =
+ DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().
+ get(0), poolId);
+ final DatanodeDescriptor dd = NameNodeAdapter.getDatanode(namesystem,
+ nodeReg);
+ // By default, MiniDFSCluster will create 1 datanode with 2 storages.
+ // Assigning 64k for remaining storage capacity and will
+ //create a file with 100k.
+ for(DatanodeStorageInfo storage: dd.getStorageInfos()) {
+ storage.setUtilizationForTesting(65536, 0, 65536, 0);
+ }
+ //sum of the remaining capacity of both the storages
+ dd.setRemaining(131072);
+ file1 = new Path("testRemainingStorage.dat");
+ try {
+ DFSTestUtil.createFile(fs, file1, 102400, 102400, 102400, (short)1,
+ 0x1BAD5EED);
+ }
+ catch (RemoteException re) {
+ GenericTestUtils.assertExceptionContains("nodes instead of "
+ + "minReplication", re);
+ }
+ }
+ finally {
+ // Clean up
+ assertTrue(fs.exists(file1));
+ fs.delete(file1, true);
+ assertTrue(!fs.exists(file1));
+ cluster.shutdown();
+ }
+ }
}