You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/05/14 17:44:27 UTC
[24/50] [abbrv] hadoop git commit: HDFS-13542.
TestBlockManager#testNeededReplicationWhileAppending fails due to improper
cluster shutdown in TestBlockManager#testBlockManagerMachinesArray on
Windows. Contributed by Anbang Hu.
HDFS-13542. TestBlockManager#testNeededReplicationWhileAppending fails due to improper cluster shutdown in TestBlockManager#testBlockManagerMachinesArray on Windows. Contributed by Anbang Hu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f809c2b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f809c2b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f809c2b
Branch: refs/heads/HDDS-4
Commit: 6f809c2b94e24645b8410de221493d2a869984d9
Parents: c3077c4
Author: Inigo Goiri <in...@apache.org>
Authored: Fri May 11 09:47:57 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Mon May 14 10:31:09 2018 -0700
----------------------------------------------------------------------
.../blockmanagement/TestBlockManager.java | 155 ++++++++++---------
1 file changed, 85 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f809c2b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 5219a44..58ca2e3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -452,8 +452,8 @@ public class TestBlockManager {
String src = "/test-file";
Path file = new Path(src);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
- cluster.waitActive();
try {
+ cluster.waitActive();
BlockManager bm = cluster.getNamesystem().getBlockManager();
FileSystem fs = cluster.getFileSystem();
NamenodeProtocols namenode = cluster.getNameNodeRpc();
@@ -492,7 +492,9 @@ public class TestBlockManager {
IOUtils.closeStream(out);
}
} finally {
- cluster.shutdown();
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
}
@@ -1043,7 +1045,9 @@ public class TestBlockManager {
assertTrue(fs.exists(file1));
fs.delete(file1, true);
assertTrue(!fs.exists(file1));
- cluster.shutdown();
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
}
@@ -1143,7 +1147,9 @@ public class TestBlockManager {
assertEquals(0, bm.getBlockOpQueueLength());
assertTrue(doneLatch.await(1, TimeUnit.SECONDS));
} finally {
- cluster.shutdown();
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
}
@@ -1218,7 +1224,9 @@ public class TestBlockManager {
long batched = MetricsAsserts.getLongCounter("BlockOpsBatched", rb);
assertTrue(batched > 0);
} finally {
- cluster.shutdown();
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
}
@@ -1227,76 +1235,83 @@ public class TestBlockManager {
final Configuration conf = new HdfsConfiguration();
final MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
- cluster.waitActive();
- BlockManager blockManager = cluster.getNamesystem().getBlockManager();
- FileSystem fs = cluster.getFileSystem();
- final Path filePath = new Path("/tmp.txt");
- final long fileLen = 1L;
- DFSTestUtil.createFile(fs, filePath, fileLen, (short) 3, 1L);
- DFSTestUtil.waitForReplication((DistributedFileSystem)fs,
- filePath, (short) 3, 60000);
- ArrayList<DataNode> datanodes = cluster.getDataNodes();
- assertEquals(datanodes.size(), 4);
- FSNamesystem ns = cluster.getNamesystem();
- // get the block
- final String bpid = cluster.getNamesystem().getBlockPoolId();
- File storageDir = cluster.getInstanceStorageDir(0, 0);
- File dataDir = MiniDFSCluster.getFinalizedDir(storageDir, bpid);
- assertTrue("Data directory does not exist", dataDir.exists());
- BlockInfo blockInfo = blockManager.blocksMap.getBlocks().iterator().next();
- ExtendedBlock blk = new ExtendedBlock(bpid, blockInfo.getBlockId(),
- blockInfo.getNumBytes(), blockInfo.getGenerationStamp());
- DatanodeDescriptor failedStorageDataNode =
- blockManager.getStoredBlock(blockInfo).getDatanode(0);
- DatanodeDescriptor corruptStorageDataNode =
- blockManager.getStoredBlock(blockInfo).getDatanode(1);
-
- ArrayList<StorageReport> reports = new ArrayList<StorageReport>();
- for(int i=0; i<failedStorageDataNode.getStorageInfos().length; i++) {
- DatanodeStorageInfo storageInfo = failedStorageDataNode
- .getStorageInfos()[i];
- DatanodeStorage dns = new DatanodeStorage(
- failedStorageDataNode.getStorageInfos()[i].getStorageID(),
- DatanodeStorage.State.FAILED,
- failedStorageDataNode.getStorageInfos()[i].getStorageType());
- while(storageInfo.getBlockIterator().hasNext()) {
- BlockInfo blockInfo1 = storageInfo.getBlockIterator().next();
- if(blockInfo1.equals(blockInfo)) {
- StorageReport report = new StorageReport(
- dns, true, storageInfo.getCapacity(),
- storageInfo.getDfsUsed(), storageInfo.getRemaining(),
- storageInfo.getBlockPoolUsed(), 0L);
- reports.add(report);
- break;
+ try {
+ cluster.waitActive();
+ BlockManager blockManager = cluster.getNamesystem().getBlockManager();
+ FileSystem fs = cluster.getFileSystem();
+ final Path filePath = new Path("/tmp.txt");
+ final long fileLen = 1L;
+ DFSTestUtil.createFile(fs, filePath, fileLen, (short) 3, 1L);
+ DFSTestUtil.waitForReplication((DistributedFileSystem)fs,
+ filePath, (short) 3, 60000);
+ ArrayList<DataNode> datanodes = cluster.getDataNodes();
+ assertEquals(datanodes.size(), 4);
+ FSNamesystem ns = cluster.getNamesystem();
+ // get the block
+ final String bpid = cluster.getNamesystem().getBlockPoolId();
+ File storageDir = cluster.getInstanceStorageDir(0, 0);
+ File dataDir = MiniDFSCluster.getFinalizedDir(storageDir, bpid);
+ assertTrue("Data directory does not exist", dataDir.exists());
+ BlockInfo blockInfo =
+ blockManager.blocksMap.getBlocks().iterator().next();
+ ExtendedBlock blk = new ExtendedBlock(bpid, blockInfo.getBlockId(),
+ blockInfo.getNumBytes(), blockInfo.getGenerationStamp());
+ DatanodeDescriptor failedStorageDataNode =
+ blockManager.getStoredBlock(blockInfo).getDatanode(0);
+ DatanodeDescriptor corruptStorageDataNode =
+ blockManager.getStoredBlock(blockInfo).getDatanode(1);
+
+ ArrayList<StorageReport> reports = new ArrayList<StorageReport>();
+ for(int i=0; i<failedStorageDataNode.getStorageInfos().length; i++) {
+ DatanodeStorageInfo storageInfo = failedStorageDataNode
+ .getStorageInfos()[i];
+ DatanodeStorage dns = new DatanodeStorage(
+ failedStorageDataNode.getStorageInfos()[i].getStorageID(),
+ DatanodeStorage.State.FAILED,
+ failedStorageDataNode.getStorageInfos()[i].getStorageType());
+ while(storageInfo.getBlockIterator().hasNext()) {
+ BlockInfo blockInfo1 = storageInfo.getBlockIterator().next();
+ if(blockInfo1.equals(blockInfo)) {
+ StorageReport report = new StorageReport(
+ dns, true, storageInfo.getCapacity(),
+ storageInfo.getDfsUsed(), storageInfo.getRemaining(),
+ storageInfo.getBlockPoolUsed(), 0L);
+ reports.add(report);
+ break;
+ }
}
}
- }
- failedStorageDataNode.updateHeartbeat(reports.toArray(StorageReport
- .EMPTY_ARRAY), 0L, 0L, 0, 0, null);
- ns.writeLock();
- DatanodeStorageInfo corruptStorageInfo= null;
- for(int i=0; i<corruptStorageDataNode.getStorageInfos().length; i++) {
- corruptStorageInfo = corruptStorageDataNode.getStorageInfos()[i];
- while(corruptStorageInfo.getBlockIterator().hasNext()) {
- BlockInfo blockInfo1 = corruptStorageInfo.getBlockIterator().next();
- if (blockInfo1.equals(blockInfo)) {
- break;
+ failedStorageDataNode.updateHeartbeat(reports.toArray(StorageReport
+ .EMPTY_ARRAY), 0L, 0L, 0, 0, null);
+ ns.writeLock();
+ DatanodeStorageInfo corruptStorageInfo= null;
+ for(int i=0; i<corruptStorageDataNode.getStorageInfos().length; i++) {
+ corruptStorageInfo = corruptStorageDataNode.getStorageInfos()[i];
+ while(corruptStorageInfo.getBlockIterator().hasNext()) {
+ BlockInfo blockInfo1 = corruptStorageInfo.getBlockIterator().next();
+ if (blockInfo1.equals(blockInfo)) {
+ break;
+ }
}
}
+ blockManager.findAndMarkBlockAsCorrupt(blk, corruptStorageDataNode,
+ corruptStorageInfo.getStorageID(),
+ CorruptReplicasMap.Reason.ANY.toString());
+ ns.writeUnlock();
+ BlockInfo[] blockInfos = new BlockInfo[] {blockInfo};
+ ns.readLock();
+ LocatedBlocks locatedBlocks =
+ blockManager.createLocatedBlocks(blockInfos, 3L, false, 0L, 3L,
+ false, false, null, null);
+ assertTrue("Located Blocks should exclude corrupt" +
+ "replicas and failed storages",
+ locatedBlocks.getLocatedBlocks().size() == 1);
+ ns.readUnlock();
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
- blockManager.findAndMarkBlockAsCorrupt(blk, corruptStorageDataNode,
- corruptStorageInfo.getStorageID(),
- CorruptReplicasMap.Reason.ANY.toString());
- ns.writeUnlock();
- BlockInfo[] blockInfos = new BlockInfo[] {blockInfo};
- ns.readLock();
- LocatedBlocks locatedBlocks =
- blockManager.createLocatedBlocks(blockInfos, 3L, false, 0L, 3L,
- false, false, null, null);
- assertTrue("Located Blocks should exclude corrupt" +
- "replicas and failed storages",
- locatedBlocks.getLocatedBlocks().size() == 1);
- ns.readUnlock();
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org