You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ji...@apache.org on 2012/02/08 04:17:09 UTC
svn commit: r1241757 - in
/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/test/java/org/apache/hadoop/hdfs/server/namenode/
Author: jitendra
Date: Wed Feb 8 03:17:09 2012
New Revision: 1241757
URL: http://svn.apache.org/viewvc?rev=1241757&view=rev
Log:
HDFS-2905. HA: Standby NN NPE when shared edits dir is deleted. Contributed by Bikas Saha.
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt?rev=1241757&r1=1241756&r2=1241757&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt Wed Feb 8 03:17:09 2012
@@ -180,3 +180,5 @@ HDFS-2733. Document HA configuration and
HDFS-2794. Active NN may purge edit log files before standby NN has a chance to read them (todd)
HDFS-2901. Improvements for SBN web UI - not show under-replicated/missing blocks. (Brandon Li via jitendra)
+
+HDFS-2905. HA: Standby NN NPE when shared edits dir is deleted. (Bikas Saha via jitendra)
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java?rev=1241757&r1=1241756&r2=1241757&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java Wed Feb 8 03:17:09 2012
@@ -135,8 +135,7 @@ class FileJournalManager implements Jour
*/
List<RemoteEditLog> getRemoteEditLogs(long firstTxId) throws IOException {
File currentDir = sd.getCurrentDir();
- List<EditLogFile> allLogFiles = matchEditLogs(
- FileUtil.listFiles(currentDir));
+ List<EditLogFile> allLogFiles = matchEditLogs(currentDir);
List<RemoteEditLog> ret = Lists.newArrayListWithCapacity(
allLogFiles.size());
@@ -155,6 +154,20 @@ class FileJournalManager implements Jour
return ret;
}
+ /**
+ * returns matching edit logs via the log directory. Simple helper function
+ * that lists the files in the logDir and calls matchEditLogs(File[])
+ *
+ * @param logDir
+ * directory to match edit logs in
+ * @return matched edit logs
+ * @throws IOException
+ * IOException thrown for invalid logDir
+ */
+ static List<EditLogFile> matchEditLogs(File logDir) throws IOException {
+ return matchEditLogs(FileUtil.listFiles(logDir));
+ }
+
static List<EditLogFile> matchEditLogs(File[] filesInStorage) {
List<EditLogFile> ret = Lists.newArrayList();
for (File f : filesInStorage) {
@@ -278,7 +291,7 @@ class FileJournalManager implements Jour
synchronized public void recoverUnfinalizedSegments() throws IOException {
File currentDir = sd.getCurrentDir();
LOG.info("Recovering unfinalized segments in " + currentDir);
- List<EditLogFile> allLogFiles = matchEditLogs(currentDir.listFiles());
+ List<EditLogFile> allLogFiles = matchEditLogs(currentDir);
for (EditLogFile elf : allLogFiles) {
if (elf.getFile().equals(currentInProgress)) {
@@ -318,7 +331,7 @@ class FileJournalManager implements Jour
private List<EditLogFile> getLogFiles(long fromTxId) throws IOException {
File currentDir = sd.getCurrentDir();
- List<EditLogFile> allLogFiles = matchEditLogs(currentDir.listFiles());
+ List<EditLogFile> allLogFiles = matchEditLogs(currentDir);
List<EditLogFile> logFiles = Lists.newArrayList();
for (EditLogFile elf : allLogFiles) {
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java?rev=1241757&r1=1241756&r2=1241757&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java Wed Feb 8 03:17:09 2012
@@ -440,7 +440,7 @@ public abstract class FSImageTestUtil {
throws IOException {
File currentDir = sd.getCurrentDir();
List<EditLogFile> foundEditLogs
- = Lists.newArrayList(FileJournalManager.matchEditLogs(currentDir.listFiles()));
+ = Lists.newArrayList(FileJournalManager.matchEditLogs(currentDir));
return Collections.max(foundEditLogs, EditLogFile.COMPARE_BY_START_TXID);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java?rev=1241757&r1=1241756&r2=1241757&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java Wed Feb 8 03:17:09 2012
@@ -316,6 +316,15 @@ public class TestFileJournalManager {
}
/**
+ * tests that passing an invalid dir to matchEditLogs throws IOException
+ */
+ @Test(expected = IOException.class)
+ public void testMatchEditLogInvalidDirThrowsIOException() throws IOException {
+ File badDir = new File("does not exist");
+ FileJournalManager.matchEditLogs(badDir);
+ }
+
+ /**
* Make sure that we starting reading the correct op when we request a stream
* with a txid in the middle of an edit log file.
*/