You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2012/08/14 07:02:01 UTC
svn commit: r1372729 - in /hadoop/common/branches/branch-1-win:
CHANGES.branch-1-win.txt
src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
Author: suresh
Date: Tue Aug 14 05:02:01 2012
New Revision: 1372729
URL: http://svn.apache.org/viewvc?rev=1372729&view=rev
Log:
HDFS-3766. Fix TestStorageRestore on Windows. Contributed by Brandon Li
Modified:
hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1372729&r1=1372728&r2=1372729&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Tue Aug 14 05:02:01 2012
@@ -71,3 +71,5 @@ BUG FIXES
(Bikas Saha via suresh)
HDFS-3763. TestNameNodeMXBean fails on Windows. (Brandon Li via suresh)
+
+ HDFS-3766. Fix TestStorageRestore on Windows. (Brandon Li via suresh)
Modified: hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1372729&r1=1372728&r2=1372729&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original)
+++ hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Tue Aug 14 05:02:01 2012
@@ -34,6 +34,8 @@ import java.lang.Math;
import java.nio.channels.FileChannel;
import java.nio.ByteBuffer;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.FSConstants;
@@ -52,6 +54,8 @@ import org.apache.hadoop.security.token.
*
*/
public class FSEditLog {
+ public static final Log LOG = LogFactory.getLog(FSEditLog.class.getName());
+
private static final byte OP_INVALID = -1;
private static final byte OP_ADD = 0;
private static final byte OP_RENAME = 1; // rename
@@ -377,7 +381,7 @@ public class FSEditLog {
eStream.flush();
eStream.close();
} catch (IOException ioe) {
- removeEditsAndStorageDir(idx);
+ fsimage.removeStorageDir(getStorageDirForStream(idx));
idx--;
}
}
@@ -442,24 +446,10 @@ public class FSEditLog {
}
}
}
-
- /**
- * Remove the given edits stream and its containing storage dir.
- */
- synchronized void removeEditsAndStorageDir(int idx) {
- exitIfStreamsNotSet();
-
- assert idx < getNumStorageDirs();
- assert getNumStorageDirs() == editStreams.size();
-
- File dir = getStorageDirForStream(idx);
- editStreams.remove(idx);
- exitIfNoStreams();
- fsimage.removeStorageDir(dir);
- }
-
+
/**
* Remove all edits streams for the given storage directory.
+ * Also close the stream and unlock the storage directory.
*/
synchronized void removeEditsForStorageDir(StorageDirectory sd) {
exitIfStreamsNotSet();
@@ -470,7 +460,13 @@ public class FSEditLog {
for (int idx = 0; idx < editStreams.size(); idx++) {
File parentDir = getStorageDirForStream(idx);
if (parentDir.getName().equals(sd.getRoot().getName())) {
- editStreams.remove(idx);
+ EditLogOutputStream s = editStreams.remove(idx);
+ try {
+ s.close();
+ sd.unlock();
+ } catch (IOException e) {
+ LOG.warn("Failed to close the stream or unlock storage dir");
+ }
}
}
exitIfNoStreams();
@@ -490,7 +486,7 @@ public class FSEditLog {
if (-1 == idx) {
fatalExit("Unable to find edits stream with IO error");
}
- removeEditsAndStorageDir(idx);
+ fsimage.removeStorageDir(getStorageDirForStream(idx));
}
fsimage.incrementCheckpointTime();
}
@@ -953,7 +949,7 @@ public class FSEditLog {
try {
eStream.write(op, writables);
} catch (IOException ioe) {
- removeEditsAndStorageDir(idx);
+ fsimage.removeStorageDir(getStorageDirForStream(idx));
idx--;
}
}
@@ -1352,8 +1348,7 @@ public class FSEditLog {
// file exists.
//
getEditFile(sd).delete();
- if (!getEditNewFile(sd).renameTo(getEditFile(sd))) {
- sd.unlock();
+ if (!getEditNewFile(sd).renameTo(getEditFile(sd))) {
removeEditsForStorageDir(sd);
fsimage.updateRemovedDirs(sd);
it.remove();
Modified: hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java?rev=1372729&r1=1372728&r2=1372729&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java (original)
+++ hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java Tue Aug 14 05:02:01 2012
@@ -63,7 +63,7 @@ public class TestStorageRestore extends
private void writeFile(FileSystem fileSys, Path name, int repl)
throws IOException {
FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf()
- .getInt("io.file.buffer.size", 4096), (short) repl, (long) blockSize);
+ .getInt("io.file.buffer.size", 4096), (short) repl, blockSize);
byte[] buffer = new byte[fileSize];
Random rand = new Random(seed);
rand.nextBytes(buffer);
@@ -71,7 +71,8 @@ public class TestStorageRestore extends
stm.close();
}
- protected void setUp() throws Exception {
+ @Override
+ protected void setUp() throws IOException {
config = new Configuration();
String baseDir = System.getProperty("test.build.data", "/tmp");
@@ -113,6 +114,7 @@ public class TestStorageRestore extends
/**
* clean up
*/
+ @Override
public void tearDown() throws Exception {
restoreAccess();
if (hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir)) {
@@ -124,7 +126,7 @@ public class TestStorageRestore extends
/**
* invalidate storage by removing sub-directory "current" in name2 and name3
*/
- public void invalidateStorage(FSImage fi) throws IOException {
+ public void invalidateStorage(FSImage fi) {
for (Iterator<StorageDirectory> it = fi.dirIterator(); it.hasNext();) {
StorageDirectory sd = it.next();
@@ -140,13 +142,6 @@ public class TestStorageRestore extends
* invalidate storage by removing xwr permission from name2 and name3
*/
public void removeStorageAccess(FSImage fi) throws IOException {
- path2.setReadable(false);
- path2.setExecutable(false);
- path2.setWritable(false);
- path3.setReadable(false);
- path3.setExecutable(false);
- path3.setWritable(false);
-
for (Iterator<StorageDirectory> it = fi.dirIterator(); it.hasNext();) {
StorageDirectory sd = it.next();
@@ -156,25 +151,23 @@ public class TestStorageRestore extends
it.remove();
}
}
+ FileUtil.chmod(path2.getAbsolutePath(), "-xwr", false);
+ FileUtil.chmod(path3.getAbsolutePath(), "-xwr", false);
}
- public void restoreAccess() {
+ public void restoreAccess() throws IOException {
if (path2.exists()) {
- path2.setReadable(true);
- path2.setExecutable(true);
- path2.setWritable(true);
+ FileUtil.chmod(path2.getAbsolutePath(), "+xrw", false);
}
if (path3.exists()) {
- path3.setReadable(true);
- path3.setExecutable(true);
- path3.setWritable(true);
+ FileUtil.chmod(path3.getAbsolutePath(), "+xrw", false);
}
}
/**
* get the total number of healthy storage directories
*/
- public int numStorageDirs(FSImage fi) throws IOException {
+ public int numStorageDirs(FSImage fi) {
int sum = 0;
for (Iterator<StorageDirectory> it = fi.dirIterator(); it.hasNext();) {
sum++;
@@ -301,7 +294,8 @@ public class TestStorageRestore extends
SecondaryNameNode secondary = new SecondaryNameNode(config);
System.out.println("****testStorageRestore: Cluster and SNN started");
- printStorages(cluster.getNameNode().getFSImage());
+ FSImage fi = cluster.getNameNode().getFSImage();
+ printStorages(fi);
FileSystem fs = cluster.getFileSystem();
Path path = new Path("/", "test");
@@ -310,8 +304,8 @@ public class TestStorageRestore extends
System.out
.println("****testStorageRestore: file test written, invalidating storage...");
- invalidateStorage(cluster.getNameNode().getFSImage());
- printStorages(cluster.getNameNode().getFSImage());
+ invalidateStorage(fi);
+ printStorages(fi);
System.out
.println("****testStorageRestore: storage invalidated + doCheckpoint");
@@ -328,13 +322,14 @@ public class TestStorageRestore extends
checkFiles(true);
//use the recovered dir to restart nn
- if (!FileUtil.fullyDelete(path1)) {
- throw new Exception("Can't fully delete " + path1);
+ cluster.shutdownNameNode();
+ if (!FileUtil.fullyDelete(new File(path1, Storage.STORAGE_DIR_CURRENT))) {
+ throw new Exception("Can't fully delete current dir under " + path1);
}
- if (!FileUtil.fullyDelete(path3)) {
- throw new Exception("Can't fully delete " + path3);
+ if (!FileUtil.fullyDelete(new File(path3, Storage.STORAGE_DIR_CURRENT))) {
+ throw new Exception("Can't fully delete current dir under " + path3);
}
- cluster.restartDataNodes();
+ cluster.restartNameNode();
cluster.waitActive();
File tmpfile = new File("/test");
assert(tmpfile.exists());
@@ -366,37 +361,38 @@ public class TestStorageRestore extends
cluster.waitActive();
SecondaryNameNode secondary = new SecondaryNameNode(config);
- System.out.println("****testStorageRestore: Cluster and SNN started");
- printStorages(cluster.getNameNode().getFSImage());
+ System.out.println("****testStorageRestoreFailure: Cluster and SNN started");
+ FSImage fi = cluster.getNameNode().getFSImage();
+ printStorages(fi);
FileSystem fs = cluster.getFileSystem();
Path path = new Path("/", "test");
writeFile(fs, path, 2);
System.out
- .println("****testStorageRestore: file test written, invalidating storage...");
+ .println("****testStorageRestoreFailure: file test written, invalidating storage...");
- removeStorageAccess(cluster.getNameNode().getFSImage());
- printStorages(cluster.getNameNode().getFSImage());
+ removeStorageAccess(fi);
+ printStorages(fi);
System.out
- .println("****testStorageRestore: storage invalidated + doCheckpoint");
+ .println("****testStorageRestoreFailure: storage invalidated + doCheckpoint");
path = new Path("/", "test1");
writeFile(fs, path, 2);
- System.out.println("****testStorageRestore: file test1 written");
- assert(numStorageDirs(cluster.getNameNode().getFSImage()) == 1);
+ System.out.println("****testStorageRestoreFailure: file test1 written");
+ assert(numStorageDirs(fi) == 1);
- System.out.println("****testStorageRestore: checkfiles(false) run");
+ System.out.println("****testStorageRestoreFailure: checkfiles(false) run");
secondary.doCheckpoint(); // still can't recover removed storage dirs
- assert(numStorageDirs(cluster.getNameNode().getFSImage()) == 1);
+ assert(numStorageDirs(fi) == 1);
restoreAccess();
secondary.doCheckpoint(); // should restore removed storage dirs
checkFiles(true);
System.out
- .println("****testStorageRestore: second Checkpoint done and checkFiles(true) run");
+ .println("****testStorageRestoreFailure: second Checkpoint done and checkFiles(true) run");
secondary.shutdown();
cluster.shutdown();
}