You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2017/10/24 18:32:02 UTC
[28/50] [abbrv] hadoop git commit: HDFS-12249. dfsadmin -metaSave to
output maintenance mode blocks. Contributed by Wellington Chevreuil.
HDFS-12249. dfsadmin -metaSave to output maintenance mode blocks. Contributed by Wellington Chevreuil.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f2a6912
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f2a6912
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f2a6912
Branch: refs/heads/HDFS-9806
Commit: 0f2a69127aa40b7dccdf74ff63e2205a10ae8998
Parents: 9e77dc2
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Mon Oct 23 15:23:52 2017 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Mon Oct 23 15:24:34 2017 -0700
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 12 +-
.../blockmanagement/TestBlockManager.java | 111 +++++++++++++++++++
2 files changed, 119 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f2a6912/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 8b0f094..6cd67f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -833,10 +833,12 @@ public class BlockManager implements BlockStatsMXBean {
// l: == live:, d: == decommissioned c: == corrupt e: == excess
out.print(block + ((usableReplicas > 0)? "" : " MISSING") +
" (replicas:" +
- " l: " + numReplicas.liveReplicas() +
- " d: " + numReplicas.decommissionedAndDecommissioning() +
- " c: " + numReplicas.corruptReplicas() +
- " e: " + numReplicas.excessReplicas() + ") ");
+ " live: " + numReplicas.liveReplicas() +
+ " decommissioning and decommissioned: " +
+ numReplicas.decommissionedAndDecommissioning() +
+ " corrupt: " + numReplicas.corruptReplicas() +
+ " in excess: " + numReplicas.excessReplicas() +
+ " maintenance mode: " + numReplicas.maintenanceReplicas() + ") ");
Collection<DatanodeDescriptor> corruptNodes =
corruptReplicas.getNodes(block);
@@ -849,6 +851,8 @@ public class BlockManager implements BlockStatsMXBean {
} else if (node.isDecommissioned() ||
node.isDecommissionInProgress()) {
state = "(decommissioned)";
+ } else if (node.isMaintenance() || node.isInMaintenance()){
+ state = "(maintenance)";
}
if (storage.areBlockContentsStale()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f2a6912/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 10289ed..4d01d80 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -1511,4 +1511,115 @@ public class TestBlockManager {
return blockInfo;
}
+ private BlockInfo makeBlockReplicasMaintenance(long blockId,
+ List<DatanodeDescriptor> nodesList) throws IOException {
+ long inodeId = ++mockINodeId;
+ final INodeFile bc = TestINodeFile.createINodeFile(inodeId);
+
+ BlockInfo blockInfo = blockOnNodes(blockId, nodesList);
+ blockInfo.setReplication((short) 3);
+ blockInfo.setBlockCollectionId(inodeId);
+
+ Mockito.doReturn(bc).when(fsn).getBlockCollection(inodeId);
+ bm.blocksMap.addBlockCollection(blockInfo, bc);
+ nodesList.get(0).setInMaintenance();
+ BlockCollection mockedBc = Mockito.mock(BlockCollection.class);
+ Mockito.when(mockedBc.getBlocks()).thenReturn(new BlockInfo[]{blockInfo});
+ bm.checkRedundancy(mockedBc);
+ return blockInfo;
+ }
+
+ @Test
+ public void testMetaSaveInMaintenanceReplicas() throws Exception {
+ List<DatanodeStorageInfo> origStorages = getStorages(0, 1);
+ List<DatanodeDescriptor> origNodes = getNodes(origStorages);
+ BlockInfo block = makeBlockReplicasMaintenance(0, origNodes);
+ File file = new File("test.log");
+ PrintWriter out = new PrintWriter(file);
+ bm.metaSave(out);
+ out.flush();
+ FileInputStream fstream = new FileInputStream(file);
+ DataInputStream in = new DataInputStream(fstream);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+ StringBuffer buffer = new StringBuffer();
+ String line;
+ try {
+ while ((line = reader.readLine()) != null) {
+ buffer.append(line);
+ System.out.println(line);
+ }
+ String output = buffer.toString();
+ assertTrue("Metasave output should not have reported " +
+ "missing blocks.",
+ output.contains("Metasave: Blocks currently missing: 0"));
+ assertTrue("There should be 1 block waiting for reconstruction",
+ output.contains("Metasave: Blocks waiting for reconstruction: 1"));
+ String blockNameGS = block.getBlockName() + "_" +
+ block.getGenerationStamp();
+ assertTrue("Block " + blockNameGS +
+ " should be list as maintenance.",
+ output.contains(blockNameGS + " (replicas: live: 1 decommissioning " +
+ "and decommissioned: 0 corrupt: 0 in excess: " +
+ "0 maintenance mode: 1)"));
+ } finally {
+ reader.close();
+ file.delete();
+ }
+ }
+
+ private BlockInfo makeBlockReplicasDecommission(long blockId,
+ List<DatanodeDescriptor> nodesList) throws IOException {
+ long inodeId = ++mockINodeId;
+ final INodeFile bc = TestINodeFile.createINodeFile(inodeId);
+
+ BlockInfo blockInfo = blockOnNodes(blockId, nodesList);
+ blockInfo.setReplication((short) 3);
+ blockInfo.setBlockCollectionId(inodeId);
+
+ Mockito.doReturn(bc).when(fsn).getBlockCollection(inodeId);
+ bm.blocksMap.addBlockCollection(blockInfo, bc);
+ nodesList.get(0).startDecommission();
+ BlockCollection mockedBc = Mockito.mock(BlockCollection.class);
+ Mockito.when(mockedBc.getBlocks()).thenReturn(new BlockInfo[]{blockInfo});
+ bm.checkRedundancy(mockedBc);
+ return blockInfo;
+ }
+
+ @Test
+ public void testMetaSaveDecommissioningReplicas() throws Exception {
+ List<DatanodeStorageInfo> origStorages = getStorages(0, 1);
+ List<DatanodeDescriptor> origNodes = getNodes(origStorages);
+ BlockInfo block = makeBlockReplicasDecommission(0, origNodes);
+ File file = new File("test.log");
+ PrintWriter out = new PrintWriter(file);
+ bm.metaSave(out);
+ out.flush();
+ FileInputStream fstream = new FileInputStream(file);
+ DataInputStream in = new DataInputStream(fstream);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+ StringBuffer buffer = new StringBuffer();
+ String line;
+ try {
+ while ((line = reader.readLine()) != null) {
+ buffer.append(line);
+ }
+ String output = buffer.toString();
+ assertTrue("Metasave output should not have reported " +
+ "missing blocks.",
+ output.contains("Metasave: Blocks currently missing: 0"));
+ assertTrue("There should be 1 block waiting for reconstruction",
+ output.contains("Metasave: Blocks waiting for reconstruction: 1"));
+ String blockNameGS = block.getBlockName() + "_" +
+ block.getGenerationStamp();
+ assertTrue("Block " + blockNameGS +
+ " should be list as maintenance.",
+ output.contains(blockNameGS + " (replicas: live: 1 decommissioning " +
+ "and decommissioned: 1 corrupt: 0 in excess: " +
+ "0 maintenance mode: 0)"));
+ } finally {
+ reader.close();
+ file.delete();
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org