You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zh...@apache.org on 2017/06/07 15:59:43 UTC
hadoop git commit: HDFS-11743. Revert the incompatible fsck reporting
output in HDFS-7933 from branch-2.7.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 d2ec34d55 -> a91a1031e
HDFS-11743. Revert the incompatible fsck reporting output in HDFS-7933 from branch-2.7.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a91a1031
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a91a1031
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a91a1031
Branch: refs/heads/branch-2.7
Commit: a91a1031e35c5f9f0bfa10b2c46ddc5307dacfe8
Parents: d2ec34d
Author: Zhe Zhang <zh...@apache.org>
Authored: Mon Jun 5 22:07:55 2017 -0700
Committer: Zhe Zhang <zh...@apache.org>
Committed: Wed Jun 7 08:59:28 2017 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/namenode/NamenodeFsck.java | 38 ++----
.../hadoop/hdfs/TestClientReportBadBlock.java | 2 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 118 -------------------
4 files changed, 13 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a91a1031/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 2bac745..559a32e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -349,6 +349,9 @@ Release 2.7.4 - UNRELEASED
HDFS-11708. Positional read will fail if replicas moved to different DNs
after stream is opened. (vinayakumarb)
+ HDFS-11743. Revert the incompatible fsck reporting output in HDFS-7933
+ from branch-2.7. (zhz)
+
Release 2.7.3 - 2016-08-25
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a91a1031/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 62aa12a..654d0ae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -245,10 +245,8 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
out.println("No. of live Replica: " + numberReplicas.liveReplicas());
out.println("No. of excess Replica: " + numberReplicas.excessReplicas());
out.println("No. of stale Replica: " + numberReplicas.replicasOnStaleNodes());
- out.println("No. of decommissioned Replica: "
+ out.println("No. of decommission Replica: "
+ numberReplicas.decommissioned());
- out.println("No. of decommissioning Replica: "
- + numberReplicas.decommissioning());
out.println("No. of corrupted Replica: " + numberReplicas.corruptReplicas());
//record datanodes that have corrupted block replica
Collection<DatanodeDescriptor> corruptionRecord = null;
@@ -511,16 +509,10 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
NumberReplicas numberReplicas =
namenode.getNamesystem().getBlockManager().countNodes(block.getLocalBlock());
int liveReplicas = numberReplicas.liveReplicas();
- int decommissionedReplicas = numberReplicas.decommissioned();;
- int decommissioningReplicas = numberReplicas.decommissioning();
- res.decommissionedReplicas += decommissionedReplicas;
- res.decommissioningReplicas += decommissioningReplicas;
- int totalReplicas = liveReplicas + decommissionedReplicas +
- decommissioningReplicas;
- res.totalReplicas += totalReplicas;
+ res.totalReplicas += liveReplicas;
short targetFileReplication = file.getReplication();
res.numExpectedReplicas += targetFileReplication;
- if(totalReplicas < res.minReplication){
+ if(liveReplicas < res.minReplication){
res.numUnderMinReplicatedBlocks++;
}
if (liveReplicas > targetFileReplication) {
@@ -540,10 +532,10 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
out.print("\n" + path + ": CORRUPT blockpool " + block.getBlockPoolId() +
" block " + block.getBlockName()+"\n");
}
- if (totalReplicas >= res.minReplication)
+ if (liveReplicas >= res.minReplication)
res.numMinReplicatedBlocks++;
- if (totalReplicas < targetFileReplication && totalReplicas > 0) {
- res.missingReplicas += (targetFileReplication - totalReplicas);
+ if (liveReplicas < targetFileReplication && liveReplicas > 0) {
+ res.missingReplicas += (targetFileReplication - liveReplicas);
res.numUnderReplicatedBlocks += 1;
underReplicatedPerFile++;
if (!showFiles) {
@@ -552,9 +544,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
out.println(" Under replicated " + block +
". Target Replicas is " +
targetFileReplication + " but found " +
- liveReplicas + " live replica(s), " +
- decommissionedReplicas + " decommissioned replica(s) and " +
- decommissioningReplicas + " decommissioning replica(s).");
+ liveReplicas + " replica(s).");
}
// count mis replicated blocks
@@ -572,7 +562,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
block + ". " + blockPlacementStatus.getErrorDescription());
}
report.append(i + ". " + blkName + " len=" + block.getNumBytes());
- if (totalReplicas == 0) {
+ if (liveReplicas == 0) {
report.append(" MISSING!");
res.addMissing(block.toString(), block.getNumBytes());
missing++;
@@ -872,8 +862,6 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
long corruptBlocks = 0L;
long excessiveReplicas = 0L;
long missingReplicas = 0L;
- long decommissionedReplicas = 0L;
- long decommissioningReplicas = 0L;
long numUnderMinReplicatedBlocks=0L;
long numOverReplicatedBlocks = 0L;
long numUnderReplicatedBlocks = 0L;
@@ -945,7 +933,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
res.append(" (Total open file blocks (not validated): ").append(
totalOpenFilesBlocks).append(")");
}
- if (corruptFiles > 0 || numUnderMinReplicatedBlocks > 0) {
+ if (corruptFiles > 0 || numUnderMinReplicatedBlocks>0) {
res.append("\n ********************************");
if(numUnderMinReplicatedBlocks>0){
res.append("\n UNDER MIN REPL'D BLOCKS:\t").append(numUnderMinReplicatedBlocks);
@@ -1008,14 +996,6 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
((float) (missingReplicas * 100) / (float) numExpectedReplicas)).append(
" %)");
}
- if (decommissionedReplicas > 0) {
- res.append("\n DecommissionedReplicas:\t").append(
- decommissionedReplicas);
- }
- if (decommissioningReplicas > 0) {
- res.append("\n DecommissioningReplicas:\t").append(
- decommissioningReplicas);
- }
return res.toString();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a91a1031/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientReportBadBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientReportBadBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientReportBadBlock.java
index 669dcec..3bc986f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientReportBadBlock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientReportBadBlock.java
@@ -191,7 +191,7 @@ public class TestClientReportBadBlock {
verifyFirstBlockCorrupted(filePath, false);
int expectedReplicaCount = repl-corruptBlocReplicas;
verifyCorruptedBlockCount(filePath, expectedReplicaCount);
- verifyFsckHealth("Target Replicas is 3 but found 1 live replica");
+ verifyFsckHealth("Target Replicas is 3 but found 1 replica");
testFsckListCorruptFilesBlocks(filePath, 0);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a91a1031/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index 1fe17f6..bd48023 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -1471,122 +1471,4 @@ public class TestFsck {
}
}
}
-
- /**
- * Test for blocks on decommissioning hosts are not shown as missing
- */
- @Test
- public void testFsckWithDecommissionedReplicas() throws Exception {
-
- final short REPL_FACTOR = 1;
- short NUM_DN = 2;
- final long blockSize = 512;
- final long fileSize = 1024;
- boolean checkDecommissionInProgress = false;
- String [] racks = {"/rack1", "/rack2"};
- String [] hosts = {"host1", "host2"};
-
- Configuration conf = new Configuration();
- conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
- conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
-
- MiniDFSCluster cluster;
- DistributedFileSystem dfs ;
- cluster =
- new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DN).hosts(hosts)
- .racks(racks).build();
-
- assertNotNull("Failed Cluster Creation", cluster);
- cluster.waitClusterUp();
- dfs = cluster.getFileSystem();
- assertNotNull("Failed to get FileSystem", dfs);
-
- DFSTestUtil util = new DFSTestUtil.Builder().
- setName(getClass().getSimpleName()).setNumFiles(1).build();
-
- //create files
- final String testFile = new String("/testfile");
- final Path path = new Path(testFile);
- util.createFile(dfs, path, fileSize, REPL_FACTOR, 1000L);
- util.waitReplication(dfs, path, REPL_FACTOR);
- try {
- // make sure datanode that has replica is fine before decommission
- String outStr = runFsck(conf, 0, true, testFile);
- System.out.println(outStr);
- assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
-
- // decommission datanode
- ExtendedBlock eb = util.getFirstBlock(dfs, path);
- DatanodeDescriptor dn = cluster.getNameNode().getNamesystem()
- .getBlockManager().getBlockCollection(eb.getLocalBlock())
- .getBlocks()[0].getDatanode(0);
- cluster.getNameNode().getNamesystem().getBlockManager()
- .getDatanodeManager().getDecomManager().startDecommission(dn);
- String dnName = dn.getXferAddr();
-
- // wait for decommission start
- DatanodeInfo datanodeInfo = null;
- int count = 0;
- do {
- Thread.sleep(2000);
- for (DatanodeInfo info : dfs.getDataNodeStats()) {
- if (dnName.equals(info.getXferAddr())) {
- datanodeInfo = info;
- }
- }
- // check the replica status should be healthy(0)
- // instead of corruption (1) during decommissioning
- if(!checkDecommissionInProgress && datanodeInfo != null
- && datanodeInfo.isDecommissionInProgress()) {
- String fsckOut = runFsck(conf, 0, true, testFile);
- checkDecommissionInProgress = true;
- }
- } while (datanodeInfo != null && !datanodeInfo.isDecommissioned());
-
- // check the replica status should be healthy(0) after decommission
- // is done
- String fsckOut = runFsck(conf, 0, true, testFile);
- } finally {
- if (cluster != null) {
- cluster.shutdown();
- }
- }
- }
-
- @Test(timeout = 300000)
- public void testFsckCorruptWhenOneReplicaIsCorrupt()
- throws Exception {
- Configuration conf = new HdfsConfiguration();
- final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
- .nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(2)
- .build();
- try {
- cluster.waitActive();
- FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf);
- cluster.transitionToActive(0);
- String filePath = "/appendTest";
- Path fileName = new Path(filePath);
- DFSTestUtil.createFile(fs, fileName, 512, (short) 2, 0);
- DFSTestUtil.waitReplication(fs, fileName, (short) 2);
- assertTrue("File not created", fs.exists(fileName));
- cluster.getDataNodes().get(1).shutdown();
- DFSTestUtil.appendFile(fs, fileName, "appendCorruptBlock");
- cluster.restartDataNode(1, true);
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override
- public Boolean get() {
- return (
- cluster.getNameNode(0).getNamesystem().getCorruptReplicaBlocks()
- > 0);
- }
- }, 100, 5000);
-
- DFSTestUtil.appendFile(fs, fileName, "appendCorruptBlock");
- runFsck(cluster.getConfiguration(0), 0, true, "/");
- }finally {
- if(cluster!=null){
- cluster.shutdown();
- }
- }
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org