You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2012/04/10 01:25:22 UTC
svn commit: r1311518 [1/3] - in
/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs: ./
dev-support/
src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/
src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/serve...
Author: szetszwo
Date: Mon Apr 9 23:25:17 2012
New Revision: 1311518
URL: http://svn.apache.org/viewvc?rev=1311518&view=rev
Log:
Merge r1310141 through r1311517 from trunk.
Added:
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/MetaRecoveryContext.java
- copied unchanged from r1311517, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/MetaRecoveryContext.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/FenceResponse.java
- copied unchanged from r1311517, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/FenceResponse.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/FencedException.java
- copied unchanged from r1311517, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/FencedException.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/JournalInfo.java
- copied unchanged from r1311517, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/JournalInfo.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsDefaultValue.java
- copied unchanged from r1311517, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsDefaultValue.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java
- copied unchanged from r1311517, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHarFileSystemWithHA.java
- copied unchanged from r1311517, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHarFileSystemWithHA.java
Removed:
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalStream.java
Modified:
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/pom.xml
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/hdfs_user_guide.xml
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/UnregisteredNodeException.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolServerSideTranslatorPB.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/journalservice/JournalService.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/JournalProtocol.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/native/hdfs.c
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/native/hdfs.h
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/native/hdfs_test.c
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/native/tests/test-libhdfs.sh
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/JournalProtocol.proto
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsFileStatusHdfs.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/journalservice/TestJournalService.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestGetConf.java
Propchange: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1310141-1311517
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Apr 9 23:25:17 2012
@@ -62,14 +62,14 @@ Trunk (unreleased changes)
HDFS-3178. Add states and state handler for journal synchronization in
JournalService. (szetszwo)
- HDFS-3204. Minor modification to JournalProtocol.proto to make
- it generic. (suresh)
-
OPTIMIZATIONS
HDFS-2834. Add a ByteBuffer-based read API to DFSInputStream.
(Henry Robinson via todd)
+ HDFS-3110. Use directRead API to reduce the number of buffer copies in
+ libhdfs (Henry Robinson via todd)
+
BUG FIXES
HDFS-2299. TestOfflineEditsViewer is failing on trunk. (Uma Maheswara Rao G
@@ -114,6 +114,13 @@ Trunk (unreleased changes)
HDFS-3126. Journal stream from Namenode to BackupNode needs to have
timeout. (Hari Mankude via suresh)
+
+ HDFS-3121. Add HDFS tests for HADOOP-8014 change. (John George via
+ suresh)
+
+ HDFS-3119. Overreplicated block is not deleted even after the replication
+ factor is reduced after sync follwed by closing that file. (Ashish Singhi
+ via umamahesh)
Release 2.0.0 - UNRELEASED
@@ -192,6 +199,8 @@ Release 2.0.0 - UNRELEASED
HDFS-3102. Add CLI tool to initialize the shared-edits dir. (atm)
+ HDFS-3004. Implement Recovery Mode. (Colin Patrick McCabe via eli)
+
IMPROVEMENTS
HDFS-2018. Move all journal stream management code into one place.
@@ -327,6 +336,17 @@ Release 2.0.0 - UNRELEASED
HDFS-3050. rework OEV to share more code with the NameNode.
(Colin Patrick McCabe via eli)
+ HDFS-3226. Allow GetConf tool to print arbitrary keys (todd)
+
+ HDFS-3204. Minor modification to JournalProtocol.proto to make
+ it generic. (suresh)
+
+ HDFS-2505. Add a test to verify getFileChecksum(..) with ViewFS. (Ravi
+ Prakash via szetszwo)
+
+ HDFS-3211. Add fence(..) and replace NamenodeRegistration with JournalInfo
+ and epoch in JournalProtocol. (suresh via szetszwo)
+
OPTIMIZATIONS
HDFS-3024. Improve performance of stringification in addStoredBlock (todd)
@@ -436,6 +456,12 @@ Release 2.0.0 - UNRELEASED
HDFS-3208. Bogus entries in hosts files are incorrectly displayed
in the report. (eli)
+ HDFS-3136. Remove SLF4J dependency as HDFS does not need it to fix
+ unnecessary warnings. (Jason Lowe via suresh)
+
+ HDFS-3214. InterDatanodeProtocolServerSideTranslatorPB doesn't handle
+ null response from initReplicaRecovery (todd)
+
BREAKDOWN OF HDFS-1623 SUBTASKS
HDFS-2179. Add fencing framework and mechanisms for NameNode HA. (todd)
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml Mon Apr 9 23:25:17 2012
@@ -264,4 +264,10 @@
<Method name="doRefreshNamenodes" />
<Bug category="PERFORMANCE" />
</Match>
+ <!-- Don't complain about System.exit() being called from quit() -->
+ <Match>
+ <Class name="org.apache.hadoop.hdfs.server.namenode.MetaRecoveryContext" />
+ <Method name="quit" />
+ <Bug pattern="DM_EXIT" />
+ </Match>
</FindBugsFilter>
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/pom.xml?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/pom.xml (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/pom.xml Mon Apr 9 23:25:17 2012
@@ -91,16 +91,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<scope>test</scope>
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java Mon Apr 9 23:25:17 2012
@@ -94,8 +94,8 @@ class BookKeeperEditLogInputStream exten
}
@Override
- public FSEditLogOp readOp() throws IOException {
- return reader.readOp();
+ protected FSEditLogOp nextOp() throws IOException {
+ return reader.readOp(false);
}
@Override
@@ -123,12 +123,6 @@ class BookKeeperEditLogInputStream exten
lh.toString(), firstTxId, lastTxId);
}
- @Override
- public JournalType getType() {
- assert (false);
- return null;
- }
-
// TODO(HA): Test this.
@Override
public boolean isInProgress() {
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java Mon Apr 9 23:25:17 2012
@@ -18,13 +18,17 @@
package org.apache.hadoop.hdfs.server.namenode;
import java.io.IOException;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.OpInstanceCache;
/**
* Utilities for testing edit logs
*/
public class FSEditLogTestUtil {
+ private static OpInstanceCache cache = new OpInstanceCache();
+
public static FSEditLogOp getNoOpInstance() {
- return FSEditLogOp.LogSegmentOp.getInstance(FSEditLogOpCodes.OP_END_LOG_SEGMENT);
+ return FSEditLogOp.LogSegmentOp.getInstance(cache,
+ FSEditLogOpCodes.OP_END_LOG_SEGMENT);
}
public static long countTransactionsInStream(EditLogInputStream in)
@@ -32,4 +36,4 @@ public class FSEditLogTestUtil {
FSEditLogLoader.EditLogValidation validation = FSEditLogLoader.validateEditLog(in);
return validation.getNumTransactions();
}
-}
\ No newline at end of file
+}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/hdfs_user_guide.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/hdfs_user_guide.xml?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/hdfs_user_guide.xml (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/hdfs_user_guide.xml Mon Apr 9 23:25:17 2012
@@ -537,7 +537,32 @@
For command usage, see <a href="http://hadoop.apache.org/common/docs/current/commands_manual.html#fetchdt"><code>fetchdt</code> command</a>.
</p>
- </section><section> <title> Upgrade and Rollback </title>
+ </section>
+ <section> <title>Recovery Mode</title>
+ <p>Typically, you will configure multiple metadata storage locations.
+ Then, if one storage location is corrupt, you can read the
+ metadata from one of the other storage locations.</p>
+
+ <p>However, what can you do if the only storage locations available are
+ corrupt? In this case, there is a special NameNode startup mode called
+ Recovery mode that may allow you to recover most of your data.</p>
+
+ <p>You can start the NameNode in recovery mode like so:
+ <code>namenode -recover</code></p>
+
+ <p>When in recovery mode, the NameNode will interactively prompt you at
+ the command line about possible courses of action you can take to
+ recover your data.</p>
+
+ <p>If you don't want to be prompted, you can give the
+ <code>-force</code> option. This option will force
+ recovery mode to always select the first choice. Normally, this
+ will be the most reasonable choice.</p>
+
+ <p>Because Recovery mode can cause you to lose data, you should always
+ back up your edit log and fsimage before using it.</p>
+ </section>
+ <section> <title> Upgrade and Rollback </title>
<p>
When Hadoop is upgraded on an existing cluster, as with any
software upgrade, it is possible there are new bugs or
Propchange: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1310141-1311517
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Mon Apr 9 23:25:17 2012
@@ -348,4 +348,5 @@ public class DFSConfigKeys extends Commo
public static final int DFS_HA_LOGROLL_PERIOD_DEFAULT = 2 * 60; // 2m
public static final String DFS_HA_TAILEDITS_PERIOD_KEY = "dfs.ha.tail-edits.period";
public static final int DFS_HA_TAILEDITS_PERIOD_DEFAULT = 60; // 1m
+ public static final String DFS_HA_FENCE_METHODS_KEY = "dfs.ha.fencing.methods";
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/UnregisteredNodeException.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/UnregisteredNodeException.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/UnregisteredNodeException.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/UnregisteredNodeException.java Mon Apr 9 23:25:17 2012
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.protocol.JournalInfo;
import org.apache.hadoop.hdfs.server.protocol.NodeRegistration;
/**
@@ -33,6 +34,10 @@ import org.apache.hadoop.hdfs.server.pro
public class UnregisteredNodeException extends IOException {
private static final long serialVersionUID = -5620209396945970810L;
+ public UnregisteredNodeException(JournalInfo info) {
+ super("Unregistered server: " + info.toString());
+ }
+
public UnregisteredNodeException(NodeRegistration nodeReg) {
super("Unregistered server: " + nodeReg.toString());
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java Mon Apr 9 23:25:17 2012
@@ -56,9 +56,17 @@ public class InterDatanodeProtocolServer
} catch (IOException e) {
throw new ServiceException(e);
}
- return InitReplicaRecoveryResponseProto.newBuilder()
- .setBlock(PBHelper.convert(r))
- .setState(PBHelper.convert(r.getOriginalReplicaState())).build();
+
+ if (r == null) {
+ return InitReplicaRecoveryResponseProto.newBuilder()
+ .setReplicaFound(false)
+ .build();
+ } else {
+ return InitReplicaRecoveryResponseProto.newBuilder()
+ .setReplicaFound(true)
+ .setBlock(PBHelper.convert(r))
+ .setState(PBHelper.convert(r.getOriginalReplicaState())).build();
+ }
}
@Override
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java Mon Apr 9 23:25:17 2012
@@ -85,6 +85,17 @@ public class InterDatanodeProtocolTransl
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
+ if (!resp.getReplicaFound()) {
+ // No replica found on the remote node.
+ return null;
+ } else {
+ if (!resp.hasBlock() || !resp.hasState()) {
+ throw new IOException("Replica was found but missing fields. " +
+ "Req: " + req + "\n" +
+ "Resp: " + resp);
+ }
+ }
+
BlockProto b = resp.getBlock();
return new ReplicaRecoveryInfo(b.getBlockId(), b.getNumBytes(),
b.getGenStamp(), PBHelper.convert(resp.getState()));
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolServerSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolServerSideTranslatorPB.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolServerSideTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolServerSideTranslatorPB.java Mon Apr 9 23:25:17 2012
@@ -20,10 +20,13 @@ package org.apache.hadoop.hdfs.protocolP
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.FenceRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.FenceResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.StartLogSegmentRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.StartLogSegmentResponseProto;
+import org.apache.hadoop.hdfs.server.protocol.FenceResponse;
import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
import com.google.protobuf.RpcController;
@@ -48,9 +51,8 @@ public class JournalProtocolServerSideTr
public JournalResponseProto journal(RpcController unused,
JournalRequestProto req) throws ServiceException {
try {
- impl.journal(PBHelper.convert(req.getJournalInfo()),
- req.getFirstTxnId(), req.getNumTxns(), req.getRecords()
- .toByteArray());
+ impl.journal(PBHelper.convert(req.getJournalInfo()), req.getEpoch(),
+ req.getFirstTxnId(), req.getNumTxns(), req.getRecords().toByteArray());
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -63,10 +65,24 @@ public class JournalProtocolServerSideTr
StartLogSegmentRequestProto req) throws ServiceException {
try {
impl.startLogSegment(PBHelper.convert(req.getJournalInfo()),
- req.getTxid());
+ req.getEpoch(), req.getTxid());
} catch (IOException e) {
throw new ServiceException(e);
}
return StartLogSegmentResponseProto.newBuilder().build();
}
+
+ @Override
+ public FenceResponseProto fence(RpcController controller,
+ FenceRequestProto req) throws ServiceException {
+ try {
+ FenceResponse resp = impl.fence(PBHelper.convert(req.getJournalInfo()), req.getEpoch(),
+ req.getFencerInfo());
+ return FenceResponseProto.newBuilder().setInSync(resp.isInSync())
+ .setLastTransactionId(resp.getLastTransactionId())
+ .setPreviousEpoch(resp.getPreviousEpoch()).build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java Mon Apr 9 23:25:17 2012
@@ -22,10 +22,13 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.FenceRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.FenceResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.StartLogSegmentRequestProto;
+import org.apache.hadoop.hdfs.server.protocol.FenceResponse;
+import org.apache.hadoop.hdfs.server.protocol.JournalInfo;
import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
-import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtocolMetaInterface;
import org.apache.hadoop.ipc.RPC;
@@ -58,10 +61,11 @@ public class JournalProtocolTranslatorPB
}
@Override
- public void journal(NamenodeRegistration reg, long firstTxnId,
+ public void journal(JournalInfo journalInfo, long epoch, long firstTxnId,
int numTxns, byte[] records) throws IOException {
JournalRequestProto req = JournalRequestProto.newBuilder()
- .setJournalInfo(PBHelper.convertToJournalInfo(reg))
+ .setJournalInfo(PBHelper.convert(journalInfo))
+ .setEpoch(epoch)
.setFirstTxnId(firstTxnId)
.setNumTxns(numTxns)
.setRecords(PBHelper.getByteString(records))
@@ -74,10 +78,11 @@ public class JournalProtocolTranslatorPB
}
@Override
- public void startLogSegment(NamenodeRegistration registration, long txid)
+ public void startLogSegment(JournalInfo journalInfo, long epoch, long txid)
throws IOException {
StartLogSegmentRequestProto req = StartLogSegmentRequestProto.newBuilder()
- .setJournalInfo(PBHelper.convertToJournalInfo(registration))
+ .setJournalInfo(PBHelper.convert(journalInfo))
+ .setEpoch(epoch)
.setTxid(txid)
.build();
try {
@@ -86,6 +91,20 @@ public class JournalProtocolTranslatorPB
throw ProtobufHelper.getRemoteException(e);
}
}
+
+ @Override
+ public FenceResponse fence(JournalInfo journalInfo, long epoch,
+ String fencerInfo) throws IOException {
+ FenceRequestProto req = FenceRequestProto.newBuilder().setEpoch(epoch)
+ .setJournalInfo(PBHelper.convert(journalInfo)).build();
+ try {
+ FenceResponseProto resp = rpcProxy.fence(NULL_CONTROLLER, req);
+ return new FenceResponse(resp.getPreviousEpoch(),
+ resp.getLastTransactionId(), resp.getInSync());
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
@Override
public boolean isMethodSupported(String methodName) throws IOException {
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java Mon Apr 9 23:25:17 2012
@@ -110,6 +110,7 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
+import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
import org.apache.hadoop.hdfs.server.protocol.CheckpointCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
@@ -117,6 +118,7 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;
import org.apache.hadoop.hdfs.server.protocol.FinalizeCommand;
+import org.apache.hadoop.hdfs.server.protocol.JournalInfo;
import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand;
import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand;
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
@@ -127,7 +129,6 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.RegisterCommand;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
-import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
@@ -1347,25 +1348,19 @@ public class PBHelper {
.setStorageID(r.getStorageID()).build();
}
- public static NamenodeRegistration convert(JournalInfoProto info) {
+ public static JournalInfo convert(JournalInfoProto info) {
int lv = info.hasLayoutVersion() ? info.getLayoutVersion() : 0;
int nsID = info.hasNamespaceID() ? info.getNamespaceID() : 0;
- StorageInfo storage = new StorageInfo(lv, nsID, info.getClusterID(), 0);
-
- // Note that the role is always {@link NamenodeRole#NAMENODE} as this
- // conversion happens for messages from Namenode to Journal receivers.
- // Addresses in the registration are unused.
- return new NamenodeRegistration("", "", storage, NamenodeRole.NAMENODE);
+ return new JournalInfo(lv, info.getClusterID(), nsID);
}
/**
* Method used for converting {@link JournalInfoProto} sent from Namenode
* to Journal receivers to {@link NamenodeRegistration}.
*/
- public static JournalInfoProto convertToJournalInfo(
- NamenodeRegistration reg) {
- return JournalInfoProto.newBuilder().setClusterID(reg.getClusterID())
- .setLayoutVersion(reg.getLayoutVersion())
- .setNamespaceID(reg.getNamespaceID()).build();
+ public static JournalInfoProto convert(JournalInfo j) {
+ return JournalInfoProto.newBuilder().setClusterID(j.getClusterId())
+ .setLayoutVersion(j.getLayoutVersion())
+ .setNamespaceID(j.getNamespaceId()).build();
}
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Mon Apr 9 23:25:17 2012
@@ -2767,7 +2767,7 @@ assert storedBlock.findDatanode(dn) < 0
}
}
- public void checkReplication(Block block, int numExpectedReplicas) {
+ public void checkReplication(Block block, short numExpectedReplicas) {
// filter out containingNodes that are marked for decommission.
NumberReplicas number = countNodes(block);
if (isNeededReplication(block, numExpectedReplicas, number.liveReplicas())) {
@@ -2775,6 +2775,10 @@ assert storedBlock.findDatanode(dn) < 0
number.liveReplicas(),
number.decommissionedReplicas(),
numExpectedReplicas);
+ return;
+ }
+ if (number.liveReplicas() > numExpectedReplicas) {
+ processOverReplicatedBlock(block, numExpectedReplicas, null, null);
}
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java Mon Apr 9 23:25:17 2012
@@ -22,6 +22,7 @@ import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.server.namenode.MetaRecoveryContext;
/************************************
* Some handy internal HDFS constants
@@ -54,13 +55,18 @@ public final class HdfsServerConstants {
FINALIZE("-finalize"),
IMPORT ("-importCheckpoint"),
BOOTSTRAPSTANDBY("-bootstrapStandby"),
- INITIALIZESHAREDEDITS("-initializeSharedEdits");
+ INITIALIZESHAREDEDITS("-initializeSharedEdits"),
+ RECOVER ("-recover"),
+ FORCE("-force");
private String name = null;
// Used only with format and upgrade options
private String clusterId = null;
+ // Used only with recovery option
+ private int force = 0;
+
private StartupOption(String arg) {this.name = arg;}
public String getName() {return name;}
public NamenodeRole toNodeRole() {
@@ -77,10 +83,24 @@ public final class HdfsServerConstants {
public void setClusterId(String cid) {
clusterId = cid;
}
-
+
public String getClusterId() {
return clusterId;
}
+
+ public MetaRecoveryContext createRecoveryContext() {
+ if (!name.equals(RECOVER.name))
+ return null;
+ return new MetaRecoveryContext(force);
+ }
+
+ public void setForce(int force) {
+ this.force = force;
+ }
+
+ public int getForce() {
+ return this.force;
+ }
}
// Timeouts for communicating with DataNode for streaming writes/reads
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/journalservice/JournalService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/journalservice/JournalService.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/journalservice/JournalService.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/journalservice/JournalService.java Mon Apr 9 23:25:17 2012
@@ -31,6 +31,9 @@ import org.apache.hadoop.hdfs.protocolPB
import org.apache.hadoop.hdfs.protocolPB.JournalProtocolServerSideTranslatorPB;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
+import org.apache.hadoop.hdfs.server.protocol.FenceResponse;
+import org.apache.hadoop.hdfs.server.protocol.FencedException;
+import org.apache.hadoop.hdfs.server.protocol.JournalInfo;
import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
@@ -40,6 +43,7 @@ import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.BlockingService;
/**
@@ -66,6 +70,8 @@ public class JournalService implements J
private final NamenodeProtocol namenode;
private final StateHandler stateHandler = new StateHandler();
private final RPC.Server rpcServer;
+ private long epoch = 0;
+ private String fencerInfo;
enum State {
/** The service is initialized and ready to start. */
@@ -115,7 +121,7 @@ public class JournalService implements J
current = State.WAITING_FOR_ROLL;
}
- synchronized void startLogSegment() throws IOException {
+ synchronized void startLogSegment() {
if (current == State.WAITING_FOR_ROLL) {
current = State.SYNCING;
}
@@ -232,28 +238,42 @@ public class JournalService implements J
}
@Override
- public void journal(NamenodeRegistration registration, long firstTxnId,
+ public void journal(JournalInfo journalInfo, long epoch, long firstTxnId,
int numTxns, byte[] records) throws IOException {
if (LOG.isTraceEnabled()) {
LOG.trace("Received journal " + firstTxnId + " " + numTxns);
}
stateHandler.isJournalAllowed();
- verify(registration);
+ verify(epoch, journalInfo);
listener.journal(this, firstTxnId, numTxns, records);
}
@Override
- public void startLogSegment(NamenodeRegistration registration, long txid)
+ public void startLogSegment(JournalInfo journalInfo, long epoch, long txid)
throws IOException {
if (LOG.isTraceEnabled()) {
LOG.trace("Received startLogSegment " + txid);
}
stateHandler.isStartLogSegmentAllowed();
- verify(registration);
+ verify(epoch, journalInfo);
listener.rollLogs(this, txid);
stateHandler.startLogSegment();
}
+ @Override
+ public FenceResponse fence(JournalInfo journalInfo, long epoch,
+ String fencerInfo) throws IOException {
+ LOG.info("Fenced by " + fencerInfo + " with epoch " + epoch);
+ verifyFence(epoch, fencerInfo);
+ verify(journalInfo);
+ long previousEpoch = epoch;
+ this.epoch = epoch;
+ this.fencerInfo = fencerInfo;
+
+ // TODO:HDFS-3092 set lastTransId and inSync
+ return new FenceResponse(previousEpoch, 0, false);
+ }
+
/** Create an RPC server. */
private static RPC.Server createRpcServer(Configuration conf,
InetSocketAddress address, JournalProtocol impl) throws IOException {
@@ -267,15 +287,54 @@ public class JournalService implements J
address.getHostName(), address.getPort(), 1, false, conf, null);
}
- private void verify(NamenodeRegistration reg) throws IOException {
- if (!registration.getRegistrationID().equals(reg.getRegistrationID())) {
- LOG.warn("Invalid registrationID - expected: "
- + registration.getRegistrationID() + " received: "
- + reg.getRegistrationID());
- throw new UnregisteredNodeException(reg);
+ private void verifyEpoch(long e) throws FencedException {
+ if (epoch != e) {
+ String errorMsg = "Epoch " + e + " is not valid. "
+ + "Resource has already been fenced by " + fencerInfo
+ + " with epoch " + epoch;
+ LOG.warn(errorMsg);
+ throw new FencedException(errorMsg);
+ }
+ }
+
+ private void verifyFence(long e, String fencer) throws FencedException {
+ if (e <= epoch) {
+ String errorMsg = "Epoch " + e + " from fencer " + fencer
+ + " is not valid. " + "Resource has already been fenced by "
+ + fencerInfo + " with epoch " + epoch;
+ LOG.warn(errorMsg);
+ throw new FencedException(errorMsg);
+ }
+ }
+
+ /**
+ * Verifies a journal request
+ */
+ private void verify(JournalInfo journalInfo) throws IOException {
+ String errorMsg = null;
+ int expectedNamespaceID = registration.getNamespaceID();
+ if (journalInfo.getNamespaceId() != expectedNamespaceID) {
+ errorMsg = "Invalid namespaceID in journal request - expected " + expectedNamespaceID
+ + " actual " + journalInfo.getNamespaceId();
+ LOG.warn(errorMsg);
+ throw new UnregisteredNodeException(journalInfo);
+ }
+ if (!journalInfo.getClusterId().equals(registration.getClusterID())) {
+ errorMsg = "Invalid clusterId in journal request - expected "
+ + journalInfo.getClusterId() + " actual " + registration.getClusterID();
+ LOG.warn(errorMsg);
+ throw new UnregisteredNodeException(journalInfo);
}
}
+ /**
+ * Verifies a journal request
+ */
+ private void verify(long e, JournalInfo journalInfo) throws IOException {
+ verifyEpoch(e);
+ verify(journalInfo);
+ }
+
/**
* Register this service with the active namenode.
*/
@@ -298,4 +357,9 @@ public class JournalService implements J
listener.verifyVersion(this, nsInfo);
registration.setStorageInfo(nsInfo);
}
-}
\ No newline at end of file
+
+ @VisibleForTesting
+ long getEpoch() {
+ return epoch;
+ }
+}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java Mon Apr 9 23:25:17 2012
@@ -213,19 +213,21 @@ public class BackupImage extends FSImage
LOG.debug("data:" + StringUtils.byteToHexString(data));
}
- FSEditLogLoader logLoader = new FSEditLogLoader(namesystem);
+ FSEditLogLoader logLoader =
+ new FSEditLogLoader(namesystem, lastAppliedTxId);
int logVersion = storage.getLayoutVersion();
backupInputStream.setBytes(data, logVersion);
- long numLoaded = logLoader.loadEditRecords(logVersion, backupInputStream,
- true, lastAppliedTxId + 1);
- if (numLoaded != numTxns) {
+ long numTxnsAdvanced = logLoader.loadEditRecords(logVersion,
+ backupInputStream, true, lastAppliedTxId + 1, null);
+ if (numTxnsAdvanced != numTxns) {
throw new IOException("Batch of txns starting at txnid " +
firstTxId + " was supposed to contain " + numTxns +
- " transactions but only was able to apply " + numLoaded);
+ " transactions, but we were only able to advance by " +
+ numTxnsAdvanced);
}
- lastAppliedTxId += numTxns;
-
+ lastAppliedTxId = logLoader.getLastAppliedTxId();
+
namesystem.dir.updateCountForINodeWithQuota(); // inefficient!
} finally {
backupInputStream.clear();
@@ -275,7 +277,7 @@ public class BackupImage extends FSImage
editStreams.add(s);
}
}
- loadEdits(editStreams, namesystem);
+ loadEdits(editStreams, namesystem, null);
}
// now, need to load the in-progress file
@@ -309,12 +311,11 @@ public class BackupImage extends FSImage
LOG.info("Going to finish converging with remaining " + remainingTxns
+ " txns from in-progress stream " + stream);
- FSEditLogLoader loader = new FSEditLogLoader(namesystem);
- long numLoaded = loader.loadFSEdits(stream, lastAppliedTxId + 1);
- lastAppliedTxId += numLoaded;
- assert numLoaded == remainingTxns :
- "expected to load " + remainingTxns + " but loaded " +
- numLoaded + " from " + stream;
+ FSEditLogLoader loader =
+ new FSEditLogLoader(namesystem, lastAppliedTxId);
+ loader.loadFSEdits(stream, lastAppliedTxId + 1, null);
+ lastAppliedTxId = loader.getLastAppliedTxId();
+ assert lastAppliedTxId == getEditLog().getLastWrittenTxId();
} finally {
FSEditLog.closeAllStreams(editStreams);
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java Mon Apr 9 23:25:17 2012
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.na
import java.io.IOException;
+import org.apache.hadoop.hdfs.server.protocol.JournalInfo;
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
/**
@@ -26,19 +27,20 @@ import org.apache.hadoop.hdfs.server.pro
* to a BackupNode.
*/
class BackupJournalManager implements JournalManager {
-
- private final NamenodeRegistration nnReg;
private final NamenodeRegistration bnReg;
+ private final JournalInfo journalInfo;
BackupJournalManager(NamenodeRegistration bnReg,
NamenodeRegistration nnReg) {
+ journalInfo = new JournalInfo(nnReg.getLayoutVersion(),
+ nnReg.getClusterID(), nnReg.getNamespaceID());
this.bnReg = bnReg;
- this.nnReg = nnReg;
}
@Override
public EditLogOutputStream startLogSegment(long txId) throws IOException {
- EditLogBackupOutputStream stm = new EditLogBackupOutputStream(bnReg, nnReg);
+ EditLogBackupOutputStream stm = new EditLogBackupOutputStream(bnReg,
+ journalInfo);
stm.startLogSegment(txId);
return stm;
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java Mon Apr 9 23:25:17 2012
@@ -35,6 +35,8 @@ import org.apache.hadoop.hdfs.protocolPB
import org.apache.hadoop.hdfs.protocolPB.JournalProtocolServerSideTranslatorPB;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.protocol.FenceResponse;
+import org.apache.hadoop.hdfs.server.protocol.JournalInfo;
import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
@@ -217,7 +219,8 @@ public class BackupNode extends NameNode
}
/* @Override */// NameNode
- public boolean setSafeMode(SafeModeAction action) throws IOException {
+ public boolean setSafeMode(@SuppressWarnings("unused") SafeModeAction action)
+ throws IOException {
throw new UnsupportedActionException("setSafeMode");
}
@@ -236,51 +239,56 @@ public class BackupNode extends NameNode
/**
* Verifies a journal request
- * @param nodeReg node registration
- * @throws UnregisteredNodeException if the registration is invalid
*/
- void verifyJournalRequest(NamenodeRegistration reg) throws IOException {
- verifyVersion(reg.getLayoutVersion());
+ private void verifyJournalRequest(JournalInfo journalInfo)
+ throws IOException {
+ verifyVersion(journalInfo.getLayoutVersion());
String errorMsg = null;
int expectedNamespaceID = namesystem.getNamespaceInfo().getNamespaceID();
- if (reg.getNamespaceID() != expectedNamespaceID) {
+ if (journalInfo.getNamespaceId() != expectedNamespaceID) {
errorMsg = "Invalid namespaceID in journal request - expected " + expectedNamespaceID
- + " actual " + reg.getNamespaceID();
+ + " actual " + journalInfo.getNamespaceId();
LOG.warn(errorMsg);
- throw new UnregisteredNodeException(reg);
+ throw new UnregisteredNodeException(journalInfo);
}
- if (!reg.getClusterID().equals(namesystem.getClusterId())) {
+ if (!journalInfo.getClusterId().equals(namesystem.getClusterId())) {
errorMsg = "Invalid clusterId in journal request - expected "
- + reg.getClusterID() + " actual " + namesystem.getClusterId();
+ + journalInfo.getClusterId() + " actual " + namesystem.getClusterId();
LOG.warn(errorMsg);
- throw new UnregisteredNodeException(reg);
+ throw new UnregisteredNodeException(journalInfo);
}
}
-
/////////////////////////////////////////////////////
// BackupNodeProtocol implementation for backup node.
/////////////////////////////////////////////////////
@Override
- public void startLogSegment(NamenodeRegistration registration, long txid)
- throws IOException {
+ public void startLogSegment(JournalInfo journalInfo, long epoch,
+ long txid) throws IOException {
namesystem.checkOperation(OperationCategory.JOURNAL);
- verifyJournalRequest(registration);
+ verifyJournalRequest(journalInfo);
getBNImage().namenodeStartedLogSegment(txid);
}
@Override
- public void journal(NamenodeRegistration nnReg,
- long firstTxId, int numTxns,
- byte[] records) throws IOException {
+ public void journal(JournalInfo journalInfo, long epoch, long firstTxId,
+ int numTxns, byte[] records) throws IOException {
namesystem.checkOperation(OperationCategory.JOURNAL);
- verifyJournalRequest(nnReg);
+ verifyJournalRequest(journalInfo);
getBNImage().journal(firstTxId, numTxns, records);
}
private BackupImage getBNImage() {
return (BackupImage)nn.getFSImage();
}
+
+ @Override
+ public FenceResponse fence(JournalInfo journalInfo, long epoch,
+ String fencerInfo) throws IOException {
+ LOG.info("Fenced by " + fencerInfo + " with epoch " + epoch);
+ throw new UnsupportedOperationException(
+ "BackupNode does not support fence");
+ }
}
//////////////////////////////////////////////////////
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java Mon Apr 9 23:25:17 2012
@@ -292,6 +292,6 @@ class Checkpointer extends Daemon {
}
LOG.info("Checkpointer about to load edits from " +
editsStreams.size() + " stream(s).");
- dstImage.loadEdits(editsStreams, dstNamesystem);
+ dstImage.loadEdits(editsStreams, dstNamesystem, null);
}
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java Mon Apr 9 23:25:17 2012
@@ -70,21 +70,25 @@ class EditLogBackupInputStream extends E
reader = null;
}
- @Override // JournalStream
+ @Override
public String getName() {
return address;
}
- @Override // JournalStream
- public JournalType getType() {
- return JournalType.BACKUP;
- }
-
@Override
- public FSEditLogOp readOp() throws IOException {
+ protected FSEditLogOp nextOp() throws IOException {
Preconditions.checkState(reader != null,
"Must call setBytes() before readOp()");
- return reader.readOp();
+ return reader.readOp(false);
+ }
+
+ @Override
+ protected FSEditLogOp nextValidOp() {
+ try {
+ return reader.readOp(true);
+ } catch (IOException e) {
+ throw new RuntimeException("got unexpected IOException " + e, e);
+ }
}
@Override
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java Mon Apr 9 23:25:17 2012
@@ -24,6 +24,7 @@ import java.util.Arrays;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.NameNodeProxies;
import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.protocol.JournalInfo;
import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
import org.apache.hadoop.io.DataOutputBuffer;
@@ -42,18 +43,18 @@ import org.apache.hadoop.security.UserGr
class EditLogBackupOutputStream extends EditLogOutputStream {
static int DEFAULT_BUFFER_SIZE = 256;
- private JournalProtocol backupNode; // RPC proxy to backup node
- private NamenodeRegistration bnRegistration; // backup node registration
- private NamenodeRegistration nnRegistration; // active node registration
+ private final JournalProtocol backupNode; // RPC proxy to backup node
+ private final NamenodeRegistration bnRegistration; // backup node registration
+ private final JournalInfo journalInfo; // active node registration
+ private final DataOutputBuffer out; // serialized output sent to backup node
private EditsDoubleBuffer doubleBuf;
- private DataOutputBuffer out; // serialized output sent to backup node
EditLogBackupOutputStream(NamenodeRegistration bnReg, // backup node
- NamenodeRegistration nnReg) // active name-node
+ JournalInfo journalInfo) // active name-node
throws IOException {
super();
this.bnRegistration = bnReg;
- this.nnRegistration = nnReg;
+ this.journalInfo = journalInfo;
InetSocketAddress bnAddress =
NetUtils.createSocketAddr(bnRegistration.getAddress());
try {
@@ -127,8 +128,7 @@ class EditLogBackupOutputStream extends
out.reset();
assert out.getLength() == 0 : "Output buffer is not empty";
- backupNode.journal(nnRegistration,
- firstTxToFlush, numReadyTxns, data);
+ backupNode.journal(journalInfo, 0, firstTxToFlush, numReadyTxns, data);
}
}
@@ -140,6 +140,6 @@ class EditLogBackupOutputStream extends
}
void startLogSegment(long txId) throws IOException {
- backupNode.startLogSegment(nnRegistration, txId);
+ backupNode.startLogSegment(journalInfo, 0, txId);
}
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java Mon Apr 9 23:25:17 2012
@@ -89,24 +89,6 @@ public class EditLogFileInputStream exte
this.isInProgress = isInProgress;
}
- /**
- * Skip over a number of transactions. Subsequent calls to
- * {@link EditLogFileInputStream#readOp()} will begin after these skipped
- * transactions. If more transactions are requested to be skipped than remain
- * in the edit log, all edit log ops in the log will be skipped and subsequent
- * calls to {@link EditLogInputStream#readOp} will return null.
- *
- * @param transactionsToSkip number of transactions to skip over.
- * @throws IOException if there's an error while reading an operation
- */
- public void skipTransactions(long transactionsToSkip) throws IOException {
- assert firstTxId != HdfsConstants.INVALID_TXID &&
- lastTxId != HdfsConstants.INVALID_TXID;
- for (long i = 0; i < transactionsToSkip; i++) {
- reader.readOp();
- }
- }
-
@Override
public long getFirstTxId() throws IOException {
return firstTxId;
@@ -117,19 +99,23 @@ public class EditLogFileInputStream exte
return lastTxId;
}
- @Override // JournalStream
+ @Override
public String getName() {
return file.getPath();
}
- @Override // JournalStream
- public JournalType getType() {
- return JournalType.FILE;
+ @Override
+ protected FSEditLogOp nextOp() throws IOException {
+ return reader.readOp(false);
}
-
+
@Override
- public FSEditLogOp readOp() throws IOException {
- return reader.readOp();
+ protected FSEditLogOp nextValidOp() {
+ try {
+ return reader.readOp(true);
+ } catch (IOException e) {
+ return null;
+ }
}
@Override
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java Mon Apr 9 23:25:17 2012
@@ -34,7 +34,14 @@ import org.apache.hadoop.classification.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
-public abstract class EditLogInputStream implements JournalStream, Closeable {
+public abstract class EditLogInputStream implements Closeable {
+ private FSEditLogOp cachedOp = null;
+
+ /**
+ * @return the name of the EditLogInputStream
+ */
+ public abstract String getName();
+
/**
* @return the first transaction which will be found in this stream
*/
@@ -57,9 +64,82 @@ public abstract class EditLogInputStream
* @return an operation from the stream or null if at end of stream
* @throws IOException if there is an error reading from the stream
*/
- public abstract FSEditLogOp readOp() throws IOException;
+ public FSEditLogOp readOp() throws IOException {
+ FSEditLogOp ret;
+ if (cachedOp != null) {
+ ret = cachedOp;
+ cachedOp = null;
+ return ret;
+ }
+ return nextOp();
+ }
/**
+ * Position the stream so that a valid operation can be read from it with
+ * readOp().
+ *
+ * This method can be used to skip over corrupted sections of edit logs.
+ */
+ public void resync() throws IOException {
+ if (cachedOp != null) {
+ return;
+ }
+ cachedOp = nextValidOp();
+ }
+
+ /**
+ * Get the next operation from the stream storage.
+ *
+ * @return an operation from the stream or null if at end of stream
+ * @throws IOException if there is an error reading from the stream
+ */
+ protected abstract FSEditLogOp nextOp() throws IOException;
+
+ /**
+ * Get the next valid operation from the stream storage.
+ *
+ * This is exactly like nextOp, except that we attempt to skip over damaged
+ * parts of the edit log
+ *
+ * @return an operation from the stream or null if at end of stream
+ */
+ protected FSEditLogOp nextValidOp() {
+ // This is a trivial implementation which just assumes that any errors mean
+ // that there is nothing more of value in the log. Subclasses that support
+ // error recovery will want to override this.
+ try {
+ return nextOp();
+ } catch (IOException e) {
+ return null;
+ }
+ }
+
+ /**
+ * Skip edit log operations up to a given transaction ID, or until the
+ * end of the edit log is reached.
+ *
+ * After this function returns, the next call to readOp will return either
+ * end-of-file (null) or a transaction with a txid equal to or higher than
+ * the one we asked for.
+ *
+ * @param txid The transaction ID to read up until.
+ * @return Returns true if we found a transaction ID greater than
+ * or equal to 'txid' in the log.
+ */
+ public boolean skipUntil(long txid) throws IOException {
+ while (true) {
+ FSEditLogOp op = readOp();
+ if (op == null) {
+ return false;
+ }
+ if (op.getTransactionId() >= txid) {
+ cachedOp = op;
+ return true;
+ }
+ }
+ }
+
+ /**
* Get the layout version of the data in the stream.
* @return the layout version of the ops in the stream.
* @throws IOException if there is an error reading the version
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java Mon Apr 9 23:25:17 2012
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.namenode;
import java.io.IOException;
+import java.io.Closeable;
import static org.apache.hadoop.hdfs.server.common.Util.now;
@@ -30,7 +31,7 @@ import org.apache.hadoop.classification.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
-public abstract class EditLogOutputStream {
+public abstract class EditLogOutputStream implements Closeable {
// these are statistics counters
private long numSync; // number of sync(s) to disk
private long totalTimeSync; // total time to sync
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Mon Apr 9 23:25:17 2012
@@ -127,6 +127,14 @@ public class FSEditLog {
private Configuration conf;
private List<URI> editsDirs;
+
+ private ThreadLocal<OpInstanceCache> cache =
+ new ThreadLocal<OpInstanceCache>() {
+ @Override
+ protected OpInstanceCache initialValue() {
+ return new OpInstanceCache();
+ }
+ };
/**
* The edit directories that are shared between primary and secondary.
@@ -596,7 +604,7 @@ public class FSEditLog {
* Records the block locations of the last block.
*/
public void logOpenFile(String path, INodeFileUnderConstruction newNode) {
- AddOp op = AddOp.getInstance()
+ AddOp op = AddOp.getInstance(cache.get())
.setPath(path)
.setReplication(newNode.getReplication())
.setModificationTime(newNode.getModificationTime())
@@ -614,7 +622,7 @@ public class FSEditLog {
* Add close lease record to edit log.
*/
public void logCloseFile(String path, INodeFile newNode) {
- CloseOp op = CloseOp.getInstance()
+ CloseOp op = CloseOp.getInstance(cache.get())
.setPath(path)
.setReplication(newNode.getReplication())
.setModificationTime(newNode.getModificationTime())
@@ -627,7 +635,7 @@ public class FSEditLog {
}
public void logUpdateBlocks(String path, INodeFileUnderConstruction file) {
- UpdateBlocksOp op = UpdateBlocksOp.getInstance()
+ UpdateBlocksOp op = UpdateBlocksOp.getInstance(cache.get())
.setPath(path)
.setBlocks(file.getBlocks());
logEdit(op);
@@ -637,7 +645,7 @@ public class FSEditLog {
* Add create directory record to edit log
*/
public void logMkDir(String path, INode newNode) {
- MkdirOp op = MkdirOp.getInstance()
+ MkdirOp op = MkdirOp.getInstance(cache.get())
.setPath(path)
.setTimestamp(newNode.getModificationTime())
.setPermissionStatus(newNode.getPermissionStatus());
@@ -649,7 +657,7 @@ public class FSEditLog {
* TODO: use String parameters until just before writing to disk
*/
void logRename(String src, String dst, long timestamp) {
- RenameOldOp op = RenameOldOp.getInstance()
+ RenameOldOp op = RenameOldOp.getInstance(cache.get())
.setSource(src)
.setDestination(dst)
.setTimestamp(timestamp);
@@ -660,7 +668,7 @@ public class FSEditLog {
* Add rename record to edit log
*/
void logRename(String src, String dst, long timestamp, Options.Rename... options) {
- RenameOp op = RenameOp.getInstance()
+ RenameOp op = RenameOp.getInstance(cache.get())
.setSource(src)
.setDestination(dst)
.setTimestamp(timestamp)
@@ -672,7 +680,7 @@ public class FSEditLog {
* Add set replication record to edit log
*/
void logSetReplication(String src, short replication) {
- SetReplicationOp op = SetReplicationOp.getInstance()
+ SetReplicationOp op = SetReplicationOp.getInstance(cache.get())
.setPath(src)
.setReplication(replication);
logEdit(op);
@@ -684,7 +692,7 @@ public class FSEditLog {
* @param quota the directory size limit
*/
void logSetQuota(String src, long nsQuota, long dsQuota) {
- SetQuotaOp op = SetQuotaOp.getInstance()
+ SetQuotaOp op = SetQuotaOp.getInstance(cache.get())
.setSource(src)
.setNSQuota(nsQuota)
.setDSQuota(dsQuota);
@@ -693,7 +701,7 @@ public class FSEditLog {
/** Add set permissions record to edit log */
void logSetPermissions(String src, FsPermission permissions) {
- SetPermissionsOp op = SetPermissionsOp.getInstance()
+ SetPermissionsOp op = SetPermissionsOp.getInstance(cache.get())
.setSource(src)
.setPermissions(permissions);
logEdit(op);
@@ -701,7 +709,7 @@ public class FSEditLog {
/** Add set owner record to edit log */
void logSetOwner(String src, String username, String groupname) {
- SetOwnerOp op = SetOwnerOp.getInstance()
+ SetOwnerOp op = SetOwnerOp.getInstance(cache.get())
.setSource(src)
.setUser(username)
.setGroup(groupname);
@@ -712,7 +720,7 @@ public class FSEditLog {
* concat(trg,src..) log
*/
void logConcat(String trg, String [] srcs, long timestamp) {
- ConcatDeleteOp op = ConcatDeleteOp.getInstance()
+ ConcatDeleteOp op = ConcatDeleteOp.getInstance(cache.get())
.setTarget(trg)
.setSources(srcs)
.setTimestamp(timestamp);
@@ -723,7 +731,7 @@ public class FSEditLog {
* Add delete file record to edit log
*/
void logDelete(String src, long timestamp) {
- DeleteOp op = DeleteOp.getInstance()
+ DeleteOp op = DeleteOp.getInstance(cache.get())
.setPath(src)
.setTimestamp(timestamp);
logEdit(op);
@@ -733,7 +741,7 @@ public class FSEditLog {
* Add generation stamp record to edit log
*/
void logGenerationStamp(long genstamp) {
- SetGenstampOp op = SetGenstampOp.getInstance()
+ SetGenstampOp op = SetGenstampOp.getInstance(cache.get())
.setGenerationStamp(genstamp);
logEdit(op);
}
@@ -742,7 +750,7 @@ public class FSEditLog {
* Add access time record to edit log
*/
void logTimes(String src, long mtime, long atime) {
- TimesOp op = TimesOp.getInstance()
+ TimesOp op = TimesOp.getInstance(cache.get())
.setPath(src)
.setModificationTime(mtime)
.setAccessTime(atime);
@@ -754,7 +762,7 @@ public class FSEditLog {
*/
void logSymlink(String path, String value, long mtime,
long atime, INodeSymlink node) {
- SymlinkOp op = SymlinkOp.getInstance()
+ SymlinkOp op = SymlinkOp.getInstance(cache.get())
.setPath(path)
.setValue(value)
.setModificationTime(mtime)
@@ -770,7 +778,7 @@ public class FSEditLog {
*/
void logGetDelegationToken(DelegationTokenIdentifier id,
long expiryTime) {
- GetDelegationTokenOp op = GetDelegationTokenOp.getInstance()
+ GetDelegationTokenOp op = GetDelegationTokenOp.getInstance(cache.get())
.setDelegationTokenIdentifier(id)
.setExpiryTime(expiryTime);
logEdit(op);
@@ -778,26 +786,26 @@ public class FSEditLog {
void logRenewDelegationToken(DelegationTokenIdentifier id,
long expiryTime) {
- RenewDelegationTokenOp op = RenewDelegationTokenOp.getInstance()
+ RenewDelegationTokenOp op = RenewDelegationTokenOp.getInstance(cache.get())
.setDelegationTokenIdentifier(id)
.setExpiryTime(expiryTime);
logEdit(op);
}
void logCancelDelegationToken(DelegationTokenIdentifier id) {
- CancelDelegationTokenOp op = CancelDelegationTokenOp.getInstance()
+ CancelDelegationTokenOp op = CancelDelegationTokenOp.getInstance(cache.get())
.setDelegationTokenIdentifier(id);
logEdit(op);
}
void logUpdateMasterKey(DelegationKey key) {
- UpdateMasterKeyOp op = UpdateMasterKeyOp.getInstance()
+ UpdateMasterKeyOp op = UpdateMasterKeyOp.getInstance(cache.get())
.setDelegationKey(key);
logEdit(op);
}
void logReassignLease(String leaseHolder, String src, String newHolder) {
- ReassignLeaseOp op = ReassignLeaseOp.getInstance()
+ ReassignLeaseOp op = ReassignLeaseOp.getInstance(cache.get())
.setLeaseHolder(leaseHolder)
.setPath(src)
.setNewHolder(newHolder);
@@ -896,7 +904,7 @@ public class FSEditLog {
state = State.IN_SEGMENT;
if (writeHeaderTxn) {
- logEdit(LogSegmentOp.getInstance(
+ logEdit(LogSegmentOp.getInstance(cache.get(),
FSEditLogOpCodes.OP_START_LOG_SEGMENT));
logSync();
}
@@ -912,7 +920,7 @@ public class FSEditLog {
"Bad state: %s", state);
if (writeEndTxn) {
- logEdit(LogSegmentOp.getInstance(
+ logEdit(LogSegmentOp.getInstance(cache.get(),
FSEditLogOpCodes.OP_END_LOG_SEGMENT));
logSync();
}