You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2016/04/07 23:58:29 UTC
[26/50] [abbrv] hadoop git commit: HDFS-6520. hdfs fsck passes
invalid length value when creating BlockReader (Xiao Chen via cmccabe)
HDFS-6520. hdfs fsck passes invalid length value when creating BlockReader (Xiao Chen via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/188f6528
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/188f6528
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/188f6528
Branch: refs/heads/YARN-3368
Commit: 188f65287d5b2f26a8862c88198f83ac59035016
Parents: 221b3a8
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Wed Apr 6 11:28:34 2016 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Wed Apr 6 11:28:34 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdfs/BlockReaderFactory.java | 4 +-
.../hdfs/server/namenode/NamenodeFsck.java | 2 +-
.../blockmanagement/TestBlockTokenWithDFS.java | 2 +-
.../datanode/TestDataNodeVolumeFailure.java | 2 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 122 ++++++++++++++++++-
5 files changed, 126 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/188f6528/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java
index 8a0050f..7af4609 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java
@@ -150,7 +150,7 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator {
private ClientContext clientContext;
/**
- * Number of bytes to read. -1 indicates no limit.
+ * Number of bytes to read. Must be set to a non-negative value.
*/
private long length = -1;
@@ -341,6 +341,8 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator {
*/
public BlockReader build() throws IOException {
Preconditions.checkNotNull(configuration);
+ Preconditions
+ .checkState(length >= 0, "Length must be set to a non-negative value");
BlockReader reader = tryToCreateExternalBlockReader();
if (reader != null) {
return reader;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/188f6528/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 291ba56..80f510c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -931,7 +931,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
setBlock(block).
setBlockToken(lblock.getBlockToken()).
setStartOffset(0).
- setLength(-1).
+ setLength(block.getNumBytes()).
setVerifyChecksum(true).
setClientName("fsck").
setDatanodeInfo(chosenNode).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/188f6528/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
index 8e65ff6..aa46de2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
@@ -158,7 +158,7 @@ public class TestBlockTokenWithDFS {
setBlockToken(lblock.getBlockToken()).
setInetSocketAddress(targetAddr).
setStartOffset(0).
- setLength(-1).
+ setLength(0).
setVerifyChecksum(true).
setClientName("TestBlockTokenWithDFS").
setDatanodeInfo(nodes[0]).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/188f6528/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
index 5da5cd9..73f02bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
@@ -523,7 +523,7 @@ public class TestDataNodeVolumeFailure {
"test-blockpoolid", block.getBlockId())).
setBlockToken(lblock.getBlockToken()).
setStartOffset(0).
- setLength(-1).
+ setLength(0).
setVerifyChecksum(true).
setClientName("TestDataNodeVolumeFailure").
setDatanodeInfo(datanode).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/188f6528/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index 1818db8..7cb6edc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -35,6 +35,7 @@ import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintStream;
@@ -46,6 +47,8 @@ import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.channels.FileChannel;
import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -54,12 +57,15 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import com.google.common.base.Supplier;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
@@ -101,6 +107,7 @@ import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;
import org.apache.log4j.RollingFileAppender;
+import org.junit.Assert;
import org.junit.Test;
import com.google.common.collect.Sets;
@@ -147,7 +154,7 @@ public class TestFsck {
assertEquals(expectedErrCode, errCode);
}
GenericTestUtils.setLogLevel(FSPermissionChecker.LOG, Level.INFO);
- FSImage.LOG.error("OUTPUT = " + bStream.toString());
+ FSImage.LOG.info("OUTPUT = " + bStream.toString());
return bStream.toString();
}
@@ -475,7 +482,25 @@ public class TestFsck {
}
}
}
-
+
+ public void corruptBlocks(MiniDFSCluster cluster) throws IOException {
+ for (int corruptIdx : blocksToCorrupt) {
+ // Corrupt a block by deleting it
+ ExtendedBlock block = dfsClient.getNamenode().getBlockLocations(name,
+ blockSize * corruptIdx, Long.MAX_VALUE).get(0).getBlock();
+ for (int i = 0; i < numDataNodes; i++) {
+ File blockFile = cluster.getBlockFile(i, block);
+ if(blockFile != null && blockFile.exists()) {
+ FileOutputStream blockFileStream =
+ new FileOutputStream(blockFile, false);
+ blockFileStream.write("corrupt".getBytes());
+ blockFileStream.close();
+ FSImage.LOG.info("Corrupted block file " + blockFile);
+ }
+ }
+ }
+ }
+
public void checkSalvagedRemains() throws IOException {
int chainIdx = 0;
HdfsFileStatus status = dfsClient.getFileInfo(name);
@@ -1890,4 +1915,97 @@ public class TestFsck {
if (cluster != null) {cluster.shutdown();}
}
}
+
+ @Test (timeout = 300000)
+ public void testFsckMoveAfterCorruption() throws Exception {
+ final int DFS_BLOCK_SIZE = 512 * 1024;
+ final int NUM_DATANODES = 1;
+ final int REPLICATION = 1;
+ MiniDFSCluster cluster = null;
+ try {
+ final Configuration conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DFS_BLOCK_SIZE);
+ conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L);
+ conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, REPLICATION);
+ cluster = new MiniDFSCluster.Builder(conf).build();
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ cluster.waitActive();
+
+ final String srcDir = "/srcdat";
+ final DFSTestUtil util = new DFSTestUtil.Builder().setName("TestFsck")
+ .setMinSize(DFS_BLOCK_SIZE * 2).setMaxSize(DFS_BLOCK_SIZE * 3)
+ .setNumFiles(1).build();
+ util.createFiles(dfs, srcDir, (short) REPLICATION);
+ final String fileNames[] = util.getFileNames(srcDir);
+ FSImage.LOG.info("Created files: " + Arrays.toString(fileNames));
+
+ // Run fsck here. The output is automatically logged for easier debugging
+ String outStr = runFsck(conf, 0, true, "/", "-files", "-blocks");
+ assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
+
+ // Corrupt the first block
+ final DFSClient dfsClient = new DFSClient(
+ new InetSocketAddress("localhost", cluster.getNameNodePort()), conf);
+ final String blockFileToCorrupt = fileNames[0];
+ final CorruptedTestFile ctf = new CorruptedTestFile(blockFileToCorrupt,
+ Sets.newHashSet(0), dfsClient, NUM_DATANODES, DFS_BLOCK_SIZE);
+ ctf.corruptBlocks(cluster);
+
+ // Wait for fsck to discover all the missing blocks
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ try {
+ final String str = runFsck(conf, 1, false, "/");
+ String numCorrupt = null;
+ for (String line : str.split(LINE_SEPARATOR)) {
+ Matcher m = numCorruptBlocksPattern.matcher(line);
+ if (m.matches()) {
+ numCorrupt = m.group(1);
+ break;
+ }
+ }
+ if (numCorrupt == null) {
+ Assert.fail("Cannot find corrupt blocks count in fsck output.");
+ }
+ if (Integer.parseInt(numCorrupt) == ctf.getTotalMissingBlocks()) {
+ assertTrue(str.contains(NamenodeFsck.CORRUPT_STATUS));
+ return true;
+ }
+ } catch (Exception e) {
+ FSImage.LOG.error("Exception caught", e);
+ Assert.fail("Caught unexpected exception.");
+ }
+ return false;
+ }
+ }, 1000, 60000);
+
+ runFsck(conf, 1, true, "/", "-files", "-blocks", "-racks");
+ FSImage.LOG.info("Moving blocks to lost+found");
+ // Fsck will return error since we corrupted a block
+ runFsck(conf, 1, false, "/", "-move");
+
+ final List<LocatedFileStatus> retVal = new ArrayList<>();
+ final RemoteIterator<LocatedFileStatus> iter =
+ dfs.listFiles(new Path("/lost+found"), true);
+ while (iter.hasNext()) {
+ retVal.add(iter.next());
+ }
+ FSImage.LOG.info("Items in lost+found: " + retVal);
+
+ // Expect all good blocks moved, only corrupted block skipped.
+ long totalLength = 0;
+ for (LocatedFileStatus lfs: retVal) {
+ totalLength += lfs.getLen();
+ }
+ Assert.assertTrue("Nothing is moved to lost+found!", totalLength > 0);
+ util.cleanup(dfs, srcDir);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
}