You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zookeeper.apache.org by si...@apache.org on 2012/03/31 09:31:54 UTC
svn commit: r1307732 - in /zookeeper/bookkeeper/trunk: CHANGES.txt
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCacheImpl.java
bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java
Author: sijie
Date: Sat Mar 31 07:31:54 2012
New Revision: 1307732
URL: http://svn.apache.org/viewvc?rev=1307732&view=rev
Log:
BOOKKEEPER-198: replaying entries of deleted ledgers would exhaust ledger cache. (sijie)
Modified:
zookeeper/bookkeeper/trunk/CHANGES.txt
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCacheImpl.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java
Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1307732&r1=1307731&r2=1307732&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Sat Mar 31 07:31:54 2012
@@ -72,6 +72,8 @@ Trunk (unreleased changes)
BOOKKEEPER-193: Ledger is garbage collected by mistake. (sijie, ivank via sijie)
+ BOOKKEEPER-198: replaying entries of deleted ledgers would exhaust ledger cache. (sijie)
+
hedwig-server/
BOOKKEEPER-140: Hub server doesn't subscribe remote region correctly when a region is down. (Sijie Gou via ivank)
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCacheImpl.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCacheImpl.java?rev=1307732&r1=1307731&r2=1307732&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCacheImpl.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCacheImpl.java Sat Mar 31 07:31:54 2012
@@ -153,6 +153,40 @@ public class LedgerCacheImpl implements
}
}
+ /**
+ * Grab ledger entry page whose first entry is <code>pageEntry</code>.
+ *
+ * If the page doesn't existed before, we allocate a memory page.
+ * Otherwise, we grab a clean page and read it from disk.
+ *
+ * @param ledger
+ * Ledger Id
+ * @param pageEntry
+ * Start entry of this entry page.
+ */
+ private LedgerEntryPage grabLedgerEntryPage(long ledger, long pageEntry) throws IOException {
+ LedgerEntryPage lep = grabCleanPage(ledger, pageEntry);
+ try {
+ // should update page before we put it into table
+ // otherwise we would put an empty page in it
+ updatePage(lep);
+ synchronized(this) {
+ putIntoTable(pages, lep);
+ }
+ } catch (IOException ie) {
+ // if we grab a clean page, but failed to update the page
+ // we are exhuasting the count of ledger entry pages.
+ // since this page will be never used, so we need to decrement
+ // page count of ledger cache.
+ lep.releasePage();
+ synchronized (this) {
+ --pageCount;
+ }
+ throw ie;
+ }
+ return lep;
+ }
+
@Override
public void putEntryOffset(long ledger, long entry, long offset) throws IOException {
int offsetInPage = (int) (entry % entriesPerPage);
@@ -161,12 +195,7 @@ public class LedgerCacheImpl implements
long pageEntry = entry-offsetInPage;
LedgerEntryPage lep = getLedgerEntryPage(ledger, pageEntry, false);
if (lep == null) {
- // find a free page
- lep = grabCleanPage(ledger, pageEntry);
- updatePage(lep);
- synchronized(this) {
- putIntoTable(pages, lep);
- }
+ lep = grabLedgerEntryPage(ledger, pageEntry);
}
if (lep != null) {
lep.setOffset(offset, offsetInPage*8);
@@ -184,13 +213,7 @@ public class LedgerCacheImpl implements
LedgerEntryPage lep = getLedgerEntryPage(ledger, pageEntry, false);
try {
if (lep == null) {
- lep = grabCleanPage(ledger, pageEntry);
- // should update page before we put it into table
- // otherwise we would put an empty page in it
- updatePage(lep);
- synchronized(this) {
- putIntoTable(pages, lep);
- }
+ lep = grabLedgerEntryPage(ledger, pageEntry);
}
return lep.getOffset(offsetInPage*8);
} finally {
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java?rev=1307732&r1=1307731&r2=1307732&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java Sat Mar 31 07:31:54 2012
@@ -24,6 +24,7 @@ package org.apache.bookkeeper.bookie;
import java.io.File;
import java.io.IOException;
+import org.apache.bookkeeper.bookie.Bookie.NoLedgerException;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.meta.LedgerManager;
import org.apache.bookkeeper.meta.LedgerManagerFactory;
@@ -124,4 +125,51 @@ public class LedgerCacheTest extends Tes
}
}
+ @Test
+ public void testPageEviction() throws Exception {
+ int numLedgers = 10;
+ byte[] masterKey = "blah".getBytes();
+ // limit page count
+ conf.setOpenFileLimit(999999).setPageLimit(3);
+ // create ledger cache
+ newLedgerCache();
+ try {
+ // create serveral ledgers
+ for (int i=1; i<=numLedgers; i++) {
+ ledgerCache.setMasterKey((long)i, masterKey);
+ ledgerCache.putEntryOffset(i, 0, i*8);
+ ledgerCache.putEntryOffset(i, 1, i*8);
+ }
+
+ // flush all first to clean previous dirty ledgers
+ ledgerCache.flushLedger(true);
+ // flush all
+ ledgerCache.flushLedger(true);
+
+ // delete serveral ledgers
+ for (int i=1; i<=numLedgers/2; i++) {
+ ledgerCache.deleteLedger(i);
+ }
+
+ // bookie restarts
+ newLedgerCache();
+
+ // simulate replaying journals to add entries again
+ for (int i=1; i<=numLedgers; i++) {
+ try {
+ ledgerCache.putEntryOffset(i, 1, i*8);
+ } catch (NoLedgerException nsle) {
+ if (i<=numLedgers/2) {
+ // it is ok
+ } else {
+ LOG.error("Error put entry offset : ", nsle);
+ fail("Should not reach here.");
+ }
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("Got Exception.", e);
+ fail("Failed to add entry.");
+ }
+ }
}