You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ra...@apache.org on 2008/04/10 22:14:28 UTC
svn commit: r646942 - in /hadoop/core/trunk: CHANGES.txt
src/java/org/apache/hadoop/dfs/DFSClient.java
src/test/org/apache/hadoop/dfs/DFSTestUtil.java
src/test/org/apache/hadoop/dfs/TestFsck.java
Author: rangadi
Date: Thu Apr 10 13:14:25 2008
New Revision: 646942
URL: http://svn.apache.org/viewvc?rev=646942&view=rev
Log:
HADOOP-2905. 'fsck -move' triggers NPE in NameNode. (Lohit Vjayarenu via rangadi)
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
hadoop/core/trunk/src/test/org/apache/hadoop/dfs/DFSTestUtil.java
hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFsck.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=646942&r1=646941&r2=646942&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Apr 10 13:14:25 2008
@@ -13,6 +13,9 @@
BUG FIXES
+ HADOOP-2905. 'fsck -move' triggers NPE in NameNode.
+ (Lohit Vjayarenu via rangadi)
+
Release 0.17.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java?rev=646942&r1=646941&r2=646942&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java Thu Apr 10 13:14:25 2008
@@ -689,8 +689,7 @@
/**
*/
public boolean mkdirs(String src) throws IOException {
- checkOpen();
- return namenode.mkdirs(src, null);
+ return mkdirs(src, null);
}
/**
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/DFSTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/dfs/DFSTestUtil.java?rev=646942&r1=646941&r2=646942&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/dfs/DFSTestUtil.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/dfs/DFSTestUtil.java Thu Apr 10 13:14:25 2008
@@ -185,6 +185,19 @@
waitReplication(fs, new Path(root, files[idx].getName()), value);
}
}
+
+ /** return list of filenames created as part of createFiles */
+ public String[] getFileNames(String topDir) {
+ if (nFiles == 0)
+ return new String[]{};
+ else {
+ String[] fileNames = new String[nFiles];
+ for (int idx=0; idx < nFiles; idx++) {
+ fileNames[idx] = topDir + "/" + files[idx].getName();
+ }
+ return fileNames;
+ }
+ }
/** wait for the file's replication to be done */
static void waitReplication(FileSystem fs, Path fileName,
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFsck.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFsck.java?rev=646942&r1=646941&r2=646942&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFsck.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFsck.java Thu Apr 10 13:14:25 2008
@@ -20,6 +20,8 @@
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
+import java.net.InetSocketAddress;
+import java.io.File;
import junit.framework.TestCase;
@@ -33,13 +35,13 @@
* A JUnit test for doing fsck
*/
public class TestFsck extends TestCase {
- static String runFsck(Configuration conf, String path) throws Exception {
+ static String runFsck(Configuration conf, String... path) throws Exception {
PrintStream oldOut = System.out;
ByteArrayOutputStream bStream = new ByteArrayOutputStream();
PrintStream newOut = new PrintStream(bStream, true);
System.setOut(newOut);
((Log4JLogger)PermissionChecker.LOG).getLogger().setLevel(Level.ALL);
- assertEquals(0, ToolRunner.run(new DFSck(conf), new String[] {path}));
+ assertEquals(0, ToolRunner.run(new DFSck(conf), path));
((Log4JLogger)PermissionChecker.LOG).getLogger().setLevel(Level.INFO);
System.setOut(oldOut);
return bStream.toString();
@@ -98,4 +100,60 @@
if (cluster != null) { cluster.shutdown(); }
}
}
-}
\ No newline at end of file
+
+ public void testFsckMove() throws Exception {
+ DFSTestUtil util = new DFSTestUtil("TestFsck", 5, 3, 8*1024);
+ MiniDFSCluster cluster = null;
+ FileSystem fs = null;
+ try {
+ Configuration conf = new Configuration();
+ conf.set("dfs.blockreport.intervalMsec", Integer.toString(30));
+ cluster = new MiniDFSCluster(conf, 4, true, null);
+ String topDir = "/srcdat";
+ fs = cluster.getFileSystem();
+ cluster.waitActive();
+ util.createFiles(fs, topDir);
+ String outStr = runFsck(conf, "/");
+ assertTrue(outStr.contains("HEALTHY"));
+
+ // Corrupt a block by deleting it
+ String[] fileNames = util.getFileNames(topDir);
+ DFSClient dfsClient = new DFSClient(new InetSocketAddress("localhost",
+ cluster.getNameNodePort()), conf);
+ String block = dfsClient.namenode.
+ getBlockLocations(fileNames[0], 0, Long.MAX_VALUE).
+ get(0).getBlock().toString();
+ File baseDir = new File(System.getProperty("test.build.data"),"dfs/data");
+ for (int i=0; i<8; i++) {
+ File blockFile = new File(baseDir, "data" +(i+1)+ "/current/" + block);
+ if(blockFile.exists()) {
+ assertTrue(blockFile.delete());
+ }
+ }
+
+ //Sleep for a while until block reports are sent to discover corruption
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ }
+
+ // We excpect the filesystem to be corrupted
+ outStr = runFsck(conf, "/");
+ assertTrue(outStr.contains("CORRUPT"));
+
+ // Fix the filesystem by moving corrupted files to lost+found
+ outStr = runFsck(conf, "/", "-move");
+ assertTrue(outStr.contains("CORRUPT"));
+
+ // Check to make sure we have healthy filesystem
+ outStr = runFsck(conf, "/");
+ assertTrue(outStr.contains("HEALTHY"));
+ util.cleanup(fs, topDir);
+ if (fs != null) {try{fs.close();} catch(Exception e){}}
+ cluster.shutdown();
+ } finally {
+ if (fs != null) {try{fs.close();} catch(Exception e){}}
+ if (cluster != null) { cluster.shutdown(); }
+ }
+ }
+}