You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2011/05/19 04:37:38 UTC
svn commit: r1124518 - in /hadoop/hdfs/branches/HDFS-1073: ./
src/java/org/apache/hadoop/hdfs/server/namenode/
src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/
Author: todd
Date: Thu May 19 02:37:37 2011
New Revision: 1124518
URL: http://svn.apache.org/viewvc?rev=1124518&view=rev
Log:
HDFS-1926. Remove references to StorageDirectory from JournalManager interface. Contributed by Ivan Kelly.
Modified:
hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt
hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
Modified: hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt?rev=1124518&r1=1124517&r2=1124518&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt (original)
+++ hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt Thu May 19 02:37:37 2011
@@ -27,3 +27,5 @@ HDFS-1800. Extend image checksumming to
per directory. (todd)
HDFS-1725. Set storage directories only at FSImage construction (Ivan Kelly
via todd)
+HDFS-1926. Remove references to StorageDirectory from JournalManager interface
+ (Ivan Kelly via todd)
Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java?rev=1124518&r1=1124517&r2=1124518&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java Thu May 19 02:37:37 2011
@@ -92,6 +92,9 @@ class EditLogFileOutputStream extends Ed
/** {@inheritDoc} */
@Override
public void write(int b) throws IOException {
+ if (fp == null) {
+ throw new IOException("Trying to use aborted output stream");
+ }
bufCurrent.write(b);
}
@@ -105,6 +108,9 @@ class EditLogFileOutputStream extends Ed
* */
@Override
void write(byte op, long txid, Writable... writables) throws IOException {
+ if (fp == null) {
+ throw new IOException("Trying to use aborted output stream");
+ }
int start = bufCurrent.getLength();
write(op);
bufCurrent.writeLong(txid);
@@ -134,6 +140,10 @@ class EditLogFileOutputStream extends Ed
@Override
public void close() throws IOException {
+ if (fp == null) {
+ throw new IOException("Trying to use aborted output stream");
+ }
+
setReadyToFlush();
flush();
@@ -157,6 +167,9 @@ class EditLogFileOutputStream extends Ed
@Override
public void abort() throws IOException {
+ if (fp == null) {
+ return;
+ }
IOUtils.cleanup(LOG, fp);
fp = null;
}
@@ -180,6 +193,10 @@ class EditLogFileOutputStream extends Ed
*/
@Override
protected void flushAndSync() throws IOException {
+ if (fp == null) {
+ throw new IOException("Trying to use aborted output stream");
+ }
+
preallocate(); // preallocate file if necessary
bufReady.writeTo(fp); // write data to file
bufReady.reset(); // erase all data in the buffer
Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1124518&r1=1124517&r2=1124518&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Thu May 19 02:37:37 2011
@@ -61,7 +61,7 @@ import static org.apache.hadoop.hdfs.ser
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
-public class FSEditLog implements NNStorageListener {
+public class FSEditLog {
static final String NO_JOURNAL_STREAMS_WARNING = "!!! WARNING !!!" +
" File system changes are not persistent. No journal streams.";
@@ -135,7 +135,6 @@ public class FSEditLog implements NNStor
FSEditLog(NNStorage storage) {
isSyncRunning = false;
this.storage = storage;
- this.storage.registerListener(this);
metrics = NameNode.getNameNodeMetrics();
lastPrintTime = now();
}
@@ -1056,70 +1055,20 @@ public class FSEditLog implements NNStor
* Called when some journals experience an error in some operation.
* This propagates errors to the storage level.
*/
- void disableAndReportErrorOnJournals(List<JournalAndStream> badJournals) {
+ private void disableAndReportErrorOnJournals(List<JournalAndStream> badJournals) {
if (badJournals == null || badJournals.isEmpty()) {
return; // nothing to do
}
-
- ArrayList<StorageDirectory> errorDirs = new ArrayList<StorageDirectory>();
+
for (JournalAndStream j : badJournals) {
LOG.error("Disabling journal " + j);
- StorageDirectory sd = j.getStorageDirectory();
- if (sd != null) {
- errorDirs.add(sd);
- // We will report this error to storage, which will propagate back
- // to our listener interface, at which point we'll mark it faulty.
- } else {
- // Just mark it faulty ourselves, since it's not associated with a
- // storage directory.
- markJournalFaulty(j);
- }
- }
- }
-
- private synchronized void markJournalFaulty(JournalAndStream jas) {
- try {
- jas.abort();
- } catch (IOException e) {
- LOG.warn("Failed to abort faulty journal " + jas
- + " before removing it (might be OK)", e);
- }
- }
-
- /**
- * Error Handling on a storageDirectory
- *
- */
- // NNStorageListener Interface
- @Override // NNStorageListener
- public synchronized void errorOccurred(StorageDirectory sd)
- throws IOException {
-
- LOG.debug("Error occurred on " + sd);
-
- for (JournalAndStream jas : journals) {
- if (jas.getStorageDirectory() == sd) {
- LOG.warn("Marking corresponding journal " + jas + " faulty");
- markJournalFaulty(jas);
- return;
+ try {
+ j.abort();
+ } catch (IOException ioe) {
+ LOG.warn("Failed to abort faulty journal " + j
+ + " before removing it (might be OK)", ioe);
}
}
-
- LOG.debug("Faulty " + sd + " did not correspond to any live journal manager.");
- }
-
- @Override // NNStorageListener
- public synchronized void formatOccurred(StorageDirectory sd)
- throws IOException {
- if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) {
- createEditLogFile(NNStorage.getStorageFile(sd, NameNodeFile.EDITS));
- }
- };
-
- @Override // NNStorageListener
- public synchronized void directoryAvailable(StorageDirectory sd)
- throws IOException {
- // We'll always just check all of the journals every time we roll.
}
/**
@@ -1156,10 +1105,6 @@ public class FSEditLog implements NNStor
stream = manager.createRevertedStream(source);
}
- public StorageDirectory getStorageDirectory() {
- return manager.getStorageDirectory();
- }
-
public void close() throws IOException {
if (stream == null) return;
stream.close();
@@ -1191,8 +1136,13 @@ public class FSEditLog implements NNStor
}
@VisibleForTesting
- void setCurrentStreamForTests(EditLogFileOutputStream stream) {
+ void setCurrentStreamForTests(EditLogOutputStream stream) {
this.stream = stream;
}
+
+ @VisibleForTesting
+ JournalManager getManager() {
+ return manager;
+ }
}
}
Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1124518&r1=1124517&r2=1124518&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Thu May 19 02:37:37 2011
@@ -1196,6 +1196,10 @@ public class FSImage implements NNStorag
}
LOG.info("Storage directory " + sd.getRoot()
+ " has been successfully formatted.");
+ } else if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) {
+ // TODO this goes away with HDFS-1073
+ File eFile = NNStorage.getEditFile(sd);
+ editLog.createEditLogFile(eFile);
}
};
Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java?rev=1124518&r1=1124517&r2=1124518&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java Thu May 19 02:37:37 2011
@@ -87,7 +87,7 @@ public class FileJournalManager implemen
return createStream();
}
- @Override
+ @VisibleForTesting
public StorageDirectory getStorageDirectory() {
return sd;
}
Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java?rev=1124518&r1=1124517&r2=1124518&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java Thu May 19 02:37:37 2011
@@ -19,8 +19,6 @@ package org.apache.hadoop.hdfs.server.na
import java.io.IOException;
-import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
-
/**
* A JournalManager is responsible for managing a single place of storing
* edit logs. It may correspond to multiple files, a backup node, etc.
@@ -29,13 +27,6 @@ import org.apache.hadoop.hdfs.server.com
* this class, which is created when the EditLog is first opened.
*/
public interface JournalManager {
-
- /**
- * @return the StorageDirectory associated with this journal,
- * or null if this is not a disk-based journal
- */
- StorageDirectory getStorageDirectory();
-
/**
* Prepare the stream to write edits
*/
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java?rev=1124518&r1=1124517&r2=1124518&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java Thu May 19 02:37:37 2011
@@ -29,6 +29,10 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.Random;
+import static org.mockito.Matchers.anyByte;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
+
import junit.framework.TestCase;
import org.apache.commons.logging.Log;
@@ -127,6 +131,19 @@ public class TestStorageRestore extends
}
// simulate an error
fi.getStorage().reportErrorsOnDirectories(al);
+
+ for (FSEditLog.JournalAndStream j : fi.getEditLog().getJournals()) {
+ if (j.getManager() instanceof FileJournalManager) {
+ FileJournalManager fm = (FileJournalManager)j.getManager();
+ if (fm.getStorageDirectory().getRoot().equals(path2)
+ || fm.getStorageDirectory().getRoot().equals(path3)) {
+ EditLogOutputStream mockStream = spy(j.getCurrentStream());
+ j.setCurrentStreamForTests(mockStream);
+ doThrow(new IOException("Injected fault: write")).
+ when(mockStream).write(anyByte());
+ }
+ }
+ }
}
/**