You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by um...@apache.org on 2012/12/10 19:52:36 UTC
svn commit: r1419661 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project: ./ hadoop-hdfs/
hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/
hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contr...
Author: umamahesh
Date: Mon Dec 10 18:52:35 2012
New Revision: 1419661
URL: http://svn.apache.org/viewvc?rev=1419661&view=rev
Log:
HDFS-4130. BKJM: The reading for editlog at NN starting using bkjm is not efficient. Contributed by Han Xiao.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project:r1419649
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1419649
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1419661&r1=1419660&r2=1419661&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Dec 10 18:52:35 2012
@@ -148,6 +148,9 @@ Release 2.0.3-alpha - Unreleased
HDFS-3680. Allow customized audit logging in HDFS FSNamesystem. (Marcelo
Vanzin via atm)
+ HDFS-4130. BKJM: The reading for editlog at NN starting using bkjm is not efficient.
+ (Han Xiao via umamahesh)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java?rev=1419661&r1=1419660&r2=1419661&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java Mon Dec 10 18:52:35 2012
@@ -500,16 +500,18 @@ public class BookKeeperJournalManager im
}
}
- EditLogInputStream getInputStream(long fromTxId, boolean inProgressOk)
- throws IOException {
- for (EditLogLedgerMetadata l : getLedgerList(inProgressOk)) {
- long lastTxId = l.getLastTxId();
- if (l.isInProgress()) {
- lastTxId = recoverLastTxId(l, false);
- }
-
- if (fromTxId >= l.getFirstTxId() && fromTxId <= lastTxId) {
- try {
+ @Override
+ public void selectInputStreams(Collection<EditLogInputStream> streams,
+ long fromTxId, boolean inProgressOk) throws IOException {
+ List<EditLogLedgerMetadata> currentLedgerList = getLedgerList(inProgressOk);
+ try {
+ BookKeeperEditLogInputStream elis = null;
+ for (EditLogLedgerMetadata l : currentLedgerList) {
+ long lastTxId = l.getLastTxId();
+ if (l.isInProgress()) {
+ lastTxId = recoverLastTxId(l, false);
+ }
+ if (fromTxId >= l.getFirstTxId() && fromTxId <= lastTxId) {
LedgerHandle h;
if (l.isInProgress()) { // we don't want to fence the current journal
h = bkc.openLedgerNoRecovery(l.getLedgerId(),
@@ -518,42 +520,22 @@ public class BookKeeperJournalManager im
h = bkc.openLedger(l.getLedgerId(), BookKeeper.DigestType.MAC,
digestpw.getBytes());
}
- BookKeeperEditLogInputStream s = new BookKeeperEditLogInputStream(h,
- l);
- s.skipTo(fromTxId);
- return s;
- } catch (BKException e) {
- throw new IOException("Could not open ledger for " + fromTxId, e);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- throw new IOException("Interrupted opening ledger for "
- + fromTxId, ie);
+ elis = new BookKeeperEditLogInputStream(h, l);
+ elis.skipTo(fromTxId);
+ } else {
+ return;
}
+ streams.add(elis);
+ if (elis.getLastTxId() == HdfsConstants.INVALID_TXID) {
+ return;
+ }
+ fromTxId = elis.getLastTxId() + 1;
}
- }
- return null;
- }
-
- @Override
- public void selectInputStreams(Collection<EditLogInputStream> streams,
- long fromTxId, boolean inProgressOk) {
- // NOTE: could probably be rewritten more efficiently
- while (true) {
- EditLogInputStream elis;
- try {
- elis = getInputStream(fromTxId, inProgressOk);
- } catch (IOException e) {
- LOG.error(e);
- return;
- }
- if (elis == null) {
- return;
- }
- streams.add(elis);
- if (elis.getLastTxId() == HdfsConstants.INVALID_TXID) {
- return;
- }
- fromTxId = elis.getLastTxId() + 1;
+ } catch (BKException e) {
+ throw new IOException("Could not open ledger for " + fromTxId, e);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted opening ledger for " + fromTxId, ie);
}
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java?rev=1419661&r1=1419660&r2=1419661&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java Mon Dec 10 18:52:35 2012
@@ -28,6 +28,7 @@ import org.mockito.Mockito;
import java.io.IOException;
import java.net.URI;
+import java.util.ArrayList;
import java.util.List;
import java.util.ArrayList;
import java.util.Random;
@@ -315,13 +316,13 @@ public class TestBookKeeperJournalManage
out.close();
bkjm.finalizeLogSegment(1, numTransactions);
-
- EditLogInputStream in = bkjm.getInputStream(1, true);
+ List<EditLogInputStream> in = new ArrayList<EditLogInputStream>();
+ bkjm.selectInputStreams(in, 1, true);
try {
assertEquals(numTransactions,
- FSEditLogTestUtil.countTransactionsInStream(in));
+ FSEditLogTestUtil.countTransactionsInStream(in.get(0)));
} finally {
- in.close();
+ in.get(0).close();
}
}