You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by um...@apache.org on 2012/10/30 02:09:58 UTC
svn commit: r1403594 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/main/java/org/apache/hadoo...
Author: umamahesh
Date: Tue Oct 30 01:09:58 2012
New Revision: 1403594
URL: http://svn.apache.org/viewvc?rev=1403594&view=rev
Log:
HDFS-3573. Supply NamespaceInfo when instantiating JournalManagers. Contributed by Todd Lipcon and Ivan Kelly.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1403594&r1=1403593&r2=1403594&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Oct 30 01:09:58 2012
@@ -89,6 +89,9 @@ Release 2.0.3-alpha - Unreleased
HDFS-4121. Add namespace declarations in hdfs .proto files for languages
other than java. (Binglin Chang via suresh)
+ HDFS-3573. Supply NamespaceInfo when instantiating JournalManagers.
+ (todd and ivank via umamahesh)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java?rev=1403594&r1=1403593&r2=1403594&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java Tue Oct 30 01:09:58 2012
@@ -22,6 +22,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream;
import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.conf.Configuration;
import org.apache.bookkeeper.conf.ClientConfiguration;
@@ -158,11 +159,19 @@ public class BookKeeperJournalManager im
(byte)(i) };
}
+ BookKeeperJournalManager(Configuration conf, URI uri) throws IOException {
+ this(conf, uri, null);
+ // TODO(ivank): update BookKeeperJournalManager to do something
+ // with the NamespaceInfo. This constructor has been added
+ // for compatibility with the old tests, and may be removed
+ // when the tests are updated.
+ }
+
/**
* Construct a Bookkeeper journal manager.
*/
- public BookKeeperJournalManager(Configuration conf, URI uri)
- throws IOException {
+ public BookKeeperJournalManager(Configuration conf, URI uri,
+ NamespaceInfo nsInfo) throws IOException {
this.conf = conf;
String zkConnect = uri.getAuthority().replace(";", ",");
String zkPath = uri.getPath();
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1403594&r1=1403593&r2=1403594&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Tue Oct 30 01:09:58 2012
@@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.token.delegation.DelegationKey;
@@ -301,7 +302,7 @@ public class FSEditLog implements LogsPu
endCurrentLogSegment(true);
}
- if (!journalSet.isEmpty()) {
+ if (journalSet != null && !journalSet.isEmpty()) {
try {
journalSet.close();
} catch (IOException ioe) {
@@ -950,7 +951,10 @@ public class FSEditLog implements LogsPu
minTxIdToKeep <= curSegmentTxId :
"cannot purge logs older than txid " + minTxIdToKeep +
" when current segment starts at " + curSegmentTxId;
-
+ if (minTxIdToKeep == 0) {
+ return;
+ }
+
// This could be improved to not need synchronization. But currently,
// journalSet is not threadsafe, so we need to synchronize this method.
try {
@@ -1206,8 +1210,9 @@ public class FSEditLog implements LogsPu
try {
Constructor<? extends JournalManager> cons
- = clazz.getConstructor(Configuration.class, URI.class);
- return cons.newInstance(conf, uri);
+ = clazz.getConstructor(Configuration.class, URI.class,
+ NamespaceInfo.class);
+ return cons.newInstance(conf, uri, storage.getNamespaceInfo());
} catch (Exception e) {
throw new IllegalArgumentException("Unable to construct journal, "
+ uri, e);
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1403594&r1=1403593&r2=1403594&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Tue Oct 30 01:09:58 2012
@@ -126,12 +126,6 @@ public class FSImage implements Closeabl
}
this.editLog = new FSEditLog(conf, storage, editsDirs);
- String nameserviceId = DFSUtil.getNamenodeNameServiceId(conf);
- if (!HAUtil.isHAEnabled(conf, nameserviceId)) {
- editLog.initJournalsForWrite();
- } else {
- editLog.initSharedJournalsForRead();
- }
archivalManager = new NNStorageRetentionManager(conf, storage, editLog);
}
@@ -496,6 +490,7 @@ public class FSImage implements Closeabl
// return back the real image
realImage.getStorage().setStorageInfo(ckptImage.getStorage());
realImage.getEditLog().setNextTxId(ckptImage.getEditLog().getLastWrittenTxId()+1);
+ realImage.initEditLog();
target.dir.fsImage = realImage;
realImage.getStorage().setBlockPoolID(ckptImage.getBlockPoolID());
@@ -568,10 +563,8 @@ public class FSImage implements Closeabl
Iterable<EditLogInputStream> editStreams = null;
- if (editLog.isOpenForWrite()) {
- // We only want to recover streams if we're going into Active mode.
- editLog.recoverUnclosedStreams();
- }
+ initEditLog();
+
if (LayoutVersion.supports(Feature.TXID_BASED_LAYOUT,
getLayoutVersion())) {
// If we're open for write, we're either non-HA or we're the active NN, so
@@ -629,6 +622,17 @@ public class FSImage implements Closeabl
return needToSave;
}
+ public void initEditLog() {
+ Preconditions.checkState(getNamespaceID() != 0,
+ "Must know namespace ID before initting edit log");
+ String nameserviceId = DFSUtil.getNamenodeNameServiceId(conf);
+ if (!HAUtil.isHAEnabled(conf, nameserviceId)) {
+ editLog.initJournalsForWrite();
+ editLog.recoverUnclosedStreams();
+ } else {
+ editLog.initSharedJournalsForRead();
+ }
+ }
/**
* @param imageFile the image file that was loaded
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java?rev=1403594&r1=1403593&r2=1403594&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java Tue Oct 30 01:09:58 2012
@@ -238,12 +238,6 @@ class FileJournalManager implements Jour
static void addStreamsToCollectionFromFiles(Collection<EditLogFile> elfs,
Collection<EditLogInputStream> streams, long fromTxId, boolean inProgressOk) {
for (EditLogFile elf : elfs) {
- if (elf.lastTxId < fromTxId) {
- LOG.debug("passing over " + elf + " because it ends at " +
- elf.lastTxId + ", but we only care about transactions " +
- "as new as " + fromTxId);
- continue;
- }
if (elf.isInProgress()) {
if (!inProgressOk) {
LOG.debug("passing over " + elf + " because it is in progress " +
@@ -258,6 +252,13 @@ class FileJournalManager implements Jour
continue;
}
}
+ if (elf.lastTxId < fromTxId) {
+ assert elf.lastTxId != HdfsConstants.INVALID_TXID;
+ LOG.debug("passing over " + elf + " because it ends at " +
+ elf.lastTxId + ", but we only care about transactions " +
+ "as new as " + fromTxId);
+ continue;
+ }
EditLogFileInputStream elfis = new EditLogFileInputStream(elf.getFile(),
elf.getFirstTxId(), elf.getLastTxId(), elf.isInProgress());
LOG.debug("selecting edit log stream " + elf);
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java?rev=1403594&r1=1403593&r2=1403594&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java Tue Oct 30 01:09:58 2012
@@ -1002,4 +1002,12 @@ public class NNStorage extends Storage i
inspectStorageDirs(inspector);
return inspector;
}
+
+ public NamespaceInfo getNamespaceInfo() {
+ return new NamespaceInfo(
+ getNamespaceID(),
+ getClusterID(),
+ getBlockPoolID(),
+ getCTime());
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1403594&r1=1403593&r2=1403594&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Tue Oct 30 01:09:58 2012
@@ -64,7 +64,6 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
-import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -863,11 +862,7 @@ public class NameNode {
Lists.<URI>newArrayList(),
sharedEditsDirs);
- newSharedStorage.format(new NamespaceInfo(
- existingStorage.getNamespaceID(),
- existingStorage.getClusterID(),
- existingStorage.getBlockPoolID(),
- existingStorage.getCTime()));
+ newSharedStorage.format(existingStorage.getNamespaceInfo());
// Need to make sure the edit log segments are in good shape to initialize
// the shared edits dir.
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java?rev=1403594&r1=1403593&r2=1403594&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java Tue Oct 30 01:09:58 2012
@@ -189,6 +189,8 @@ public class BootstrapStandby implements
// Load the newly formatted image, using all of the directories (including shared
// edits)
FSImage image = new FSImage(conf);
+ image.getStorage().setStorageInfo(storage);
+ image.initEditLog();
assert image.getEditLog().isOpenForRead() :
"Expected edit log to be open for read";
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java?rev=1403594&r1=1403593&r2=1403594&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java Tue Oct 30 01:09:58 2012
@@ -18,18 +18,23 @@
package org.apache.hadoop.hdfs.server.namenode;
import static org.junit.Assert.fail;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import java.io.IOException;
import java.net.URI;
import java.util.Collection;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.Test;
public class TestGenericJournalConf {
+ private static final String DUMMY_URI = "dummy://test";
+
/**
* Test that an exception is thrown if a journal class doesn't exist
* in the configuration
@@ -114,12 +119,17 @@ public class TestGenericJournalConf {
conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_PLUGIN_PREFIX + ".dummy",
DummyJournalManager.class.getName());
- conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
- "dummy://test");
+ conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, DUMMY_URI);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKED_VOLUMES_MINIMUM_KEY, 0);
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
cluster.waitActive();
+
+ assertNotNull(DummyJournalManager.conf);
+ assertEquals(new URI(DUMMY_URI), DummyJournalManager.uri);
+ assertNotNull(DummyJournalManager.nsInfo);
+ assertEquals(DummyJournalManager.nsInfo.getClusterID(),
+ cluster.getNameNode().getNamesystem().getClusterId());
} finally {
if (cluster != null) {
cluster.shutdown();
@@ -128,7 +138,17 @@ public class TestGenericJournalConf {
}
public static class DummyJournalManager implements JournalManager {
- public DummyJournalManager(Configuration conf, URI u) {}
+ static Configuration conf = null;
+ static URI uri = null;
+ static NamespaceInfo nsInfo = null;
+
+ public DummyJournalManager(Configuration conf, URI u,
+ NamespaceInfo nsInfo) {
+ // Set static vars so the test case can verify them.
+ DummyJournalManager.conf = conf;
+ DummyJournalManager.uri = u;
+ DummyJournalManager.nsInfo = nsInfo;
+ }
@Override
public EditLogOutputStream startLogSegment(long txId) throws IOException {
@@ -162,7 +182,7 @@ public class TestGenericJournalConf {
public static class BadConstructorJournalManager extends DummyJournalManager {
public BadConstructorJournalManager() {
- super(null, null);
+ super(null, null, null);
}
}
}