You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2015/08/21 01:31:49 UTC
hadoop git commit: HDFS-8809. HDFS fsck reports under construction
blocks as CORRUPT. Contributed by Jing Zhao.
Repository: hadoop
Updated Branches:
refs/heads/trunk 7642f64c2 -> c8bca6271
HDFS-8809. HDFS fsck reports under construction blocks as CORRUPT. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c8bca627
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c8bca627
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c8bca627
Branch: refs/heads/trunk
Commit: c8bca62718203a1dad9b70d164bdf10cc71b40cd
Parents: 7642f64
Author: Jing Zhao <ji...@apache.org>
Authored: Thu Aug 20 16:31:24 2015 -0700
Committer: Jing Zhao <ji...@apache.org>
Committed: Thu Aug 20 16:31:24 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../apache/hadoop/hdfs/server/namenode/NamenodeFsck.java | 10 ++++++++++
.../org/apache/hadoop/hdfs/server/namenode/TestFsck.java | 2 ++
3 files changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8bca627/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index dcc5d58..d9d176b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1194,6 +1194,8 @@ Release 2.8.0 - UNRELEASED
HDFS-8922. Link the native_mini_dfs test library with libdl, since IBM Java
requires it (Ayappan via Colin P. McCabe)
+ HDFS-8809. HDFS fsck reports under construction blocks as "CORRUPT". (jing9)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8bca627/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 2f989d1..c7892b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -528,6 +528,9 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
LocatedBlocks blocks) throws IOException {
String path = file.getFullName(parent);
boolean isOpen = blocks.isUnderConstruction();
+ if (isOpen && !showOpenFiles) {
+ return;
+ }
int missing = 0;
int corrupt = 0;
long missize = 0;
@@ -536,8 +539,15 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
int misReplicatedPerFile = 0;
StringBuilder report = new StringBuilder();
int blockNumber = 0;
+ final LocatedBlock lastBlock = blocks.getLastLocatedBlock();
for (LocatedBlock lBlk : blocks.getLocatedBlocks()) {
ExtendedBlock block = lBlk.getBlock();
+ if (!blocks.isLastBlockComplete() && lastBlock != null &&
+ lastBlock.getBlock().equals(block)) {
+ // this is the last block and this is not complete. ignore it since
+ // it is under construction
+ continue;
+ }
BlockManager bm = namenode.getNamesystem().getBlockManager();
final BlockInfo storedBlock = bm.getStoredBlock(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8bca627/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index 8818f17..2226947 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSInputStream;
+import org.apache.hadoop.hdfs.DFSOutputStream;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -612,6 +613,7 @@ public class TestFsck {
out.write(randomString.getBytes());
writeCount++;
}
+ ((DFSOutputStream) out.getWrappedStream()).hflush();
// We expect the filesystem to be HEALTHY and show one open file
outStr = runFsck(conf, 0, true, topDir);
System.out.println(outStr);