You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zookeeper.apache.org by si...@apache.org on 2012/03/21 06:13:24 UTC
svn commit: r1303286 - in /zookeeper/bookkeeper/trunk: ./
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/
bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/
Author: sijie
Date: Wed Mar 21 05:13:23 2012
New Revision: 1303286
URL: http://svn.apache.org/viewvc?rev=1303286&view=rev
Log:
BOOKKEEPER-188: Garbage collection code is in the wrong place (ivank via sijie)
Modified:
zookeeper/bookkeeper/trunk/CHANGES.txt
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/EntryLogTest.java
Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1303286&r1=1303285&r2=1303286&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Wed Mar 21 05:13:23 2012
@@ -94,6 +94,8 @@ Trunk (unreleased changes)
BOOKKEEPER-175: Bookie code is very coupled (ivank)
+ BOOKKEEPER-188: Garbage collection code is in the wrong place (ivank via sijie)
+
hedwig-server/
BOOKKEEPER-77: Add a console client for hedwig (Sijie Guo via ivank)
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java?rev=1303286&r1=1303285&r2=1303286&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java Wed Mar 21 05:13:23 2012
@@ -57,7 +57,7 @@ public class EntryLogger {
private static final Logger LOG = LoggerFactory.getLogger(EntryLogger.class);
private File dirs[];
- private long logId;
+ long logId;
/**
* The maximum size of a entry logger file.
*/
@@ -76,65 +76,6 @@ public class EntryLogger {
final static long MB = 1024 * 1024;
/**
- * Records the total size, remaining size and the set of ledgers that comprise a entry log.
- */
- static class EntryLogMetadata {
- long entryLogId;
- long totalSize;
- long remainingSize;
- ConcurrentHashMap<Long, Long> ledgersMap;
-
- public EntryLogMetadata(long logId) {
- this.entryLogId = logId;
-
- totalSize = remainingSize = 0;
- ledgersMap = new ConcurrentHashMap<Long, Long>();
- }
-
- public void addLedgerSize(long ledgerId, long size) {
- totalSize += size;
- remainingSize += size;
- Long ledgerSize = ledgersMap.get(ledgerId);
- if (null == ledgerSize) {
- ledgerSize = 0L;
- }
- ledgerSize += size;
- ledgersMap.put(ledgerId, ledgerSize);
- }
-
- public void removeLedger(long ledgerId) {
- Long size = ledgersMap.remove(ledgerId);
- if (null == size) {
- return;
- }
- remainingSize -= size;
- }
-
- public boolean containsLedger(long ledgerId) {
- return ledgersMap.containsKey(ledgerId);
- }
-
- public double getUsage() {
- if (totalSize == 0L) {
- return 0.0f;
- }
- return (double)remainingSize / totalSize;
- }
-
- public boolean isEmpty() {
- return ledgersMap.isEmpty();
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("{ totalSize = ").append(totalSize).append(", remainingSize = ")
- .append(remainingSize).append(", ledgersMap = ").append(ledgersMap).append(" }");
- return sb.toString();
- }
- }
-
- /**
* Scan entries in a entry log file.
*/
static interface EntryLogScanner {
@@ -433,76 +374,6 @@ public class EntryLogger {
}
/**
- * A scanner used to extract entry log meta from entry log files.
- */
- class ExtractionScanner implements EntryLogScanner {
- EntryLogMetadata meta;
-
- public ExtractionScanner(EntryLogMetadata meta) {
- this.meta = meta;
- }
-
- @Override
- public boolean accept(long ledgerId) {
- return true;
- }
- @Override
- public void process(long ledgerId, ByteBuffer entry) {
- // add new entry size of a ledger to entry log meta
- meta.addLedgerSize(ledgerId, entry.limit() + 4);
- }
- }
-
- /**
- * Method to read in all of the entry logs (those that we haven't done so yet),
- * and find the set of ledger ID's that make up each entry log file.
- *
- * @param entryLogMetaMap
- * Existing EntryLogs to Meta
- * @throws IOException
- */
- protected Map<Long, EntryLogMetadata> extractMetaFromEntryLogs(Map<Long, EntryLogMetadata> entryLogMetaMap) throws IOException {
- // Extract it for every entry log except for the current one.
- // Entry Log ID's are just a long value that starts at 0 and increments
- // by 1 when the log fills up and we roll to a new one.
- long curLogId = logId;
- for (long entryLogId = 0; entryLogId < curLogId; entryLogId++) {
- // Comb the current entry log file if it has not already been extracted.
- if (entryLogMetaMap.containsKey(entryLogId)) {
- continue;
- }
- LOG.info("Extracting entry log meta from entryLogId: " + entryLogId);
- EntryLogMetadata entryLogMeta = new EntryLogMetadata(entryLogId);
- ExtractionScanner scanner = new ExtractionScanner(entryLogMeta);
- // Read through the entry log file and extract the entry log meta
- try {
- scanEntryLog(entryLogId, scanner);
- LOG.info("Retrieved entry log meta data entryLogId: " + entryLogId + ", meta: " + entryLogMeta);
- entryLogMetaMap.put(entryLogId, entryLogMeta);
- } catch(IOException e) {
- LOG.warn("Premature exception when processing " + entryLogId +
- "recovery will take care of the problem", e);
- }
-
- }
- return entryLogMetaMap;
- }
-
- protected EntryLogMetadata extractMetaFromEntryLog(long entryLogId) {
- EntryLogMetadata entryLogMeta = new EntryLogMetadata(entryLogId);
- ExtractionScanner scanner = new ExtractionScanner(entryLogMeta);
- // Read through the entry log file and extract the entry log meta
- try {
- scanEntryLog(entryLogId, scanner);
- LOG.info("Retrieved entry log meta data entryLogId: " + entryLogId + ", meta: " + entryLogMeta);
- } catch(IOException e) {
- LOG.warn("Premature exception when processing " + entryLogId +
- "recovery will take care of the problem", e);
- }
- return entryLogMeta;
- }
-
- /**
* Scan entry log
*
* @param entryLogId
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java?rev=1303286&r1=1303285&r2=1303286&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java Wed Mar 21 05:13:23 2012
@@ -34,7 +34,6 @@ import java.util.concurrent.atomic.Atomi
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.bookkeeper.bookie.EntryLogger.EntryLogMetadata;
import org.apache.bookkeeper.bookie.EntryLogger.EntryLogScanner;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.meta.LedgerManager;
@@ -198,7 +197,7 @@ public class GarbageCollectorThread exte
// Extract all of the ledger ID's that comprise all of the entry logs
// (except for the current new one which is still being written to).
try {
- entryLogMetaMap = entryLogger.extractMetaFromEntryLogs(entryLogMetaMap);
+ entryLogMetaMap = extractMetaFromEntryLogs(entryLogMetaMap);
} catch (IOException ie) {
LOG.warn("Exception when extracting entry log meta from entry logs : ", ie);
}
@@ -379,4 +378,129 @@ public class GarbageCollectorThread exte
compacting.set(false);
}
}
+
+ /**
+ * Records the total size, remaining size and the set of ledgers that comprise a entry log.
+ */
+ static class EntryLogMetadata {
+ long entryLogId;
+ long totalSize;
+ long remainingSize;
+ ConcurrentHashMap<Long, Long> ledgersMap;
+
+ public EntryLogMetadata(long logId) {
+ this.entryLogId = logId;
+
+ totalSize = remainingSize = 0;
+ ledgersMap = new ConcurrentHashMap<Long, Long>();
+ }
+
+ public void addLedgerSize(long ledgerId, long size) {
+ totalSize += size;
+ remainingSize += size;
+ Long ledgerSize = ledgersMap.get(ledgerId);
+ if (null == ledgerSize) {
+ ledgerSize = 0L;
+ }
+ ledgerSize += size;
+ ledgersMap.put(ledgerId, ledgerSize);
+ }
+
+ public void removeLedger(long ledgerId) {
+ Long size = ledgersMap.remove(ledgerId);
+ if (null == size) {
+ return;
+ }
+ remainingSize -= size;
+ }
+
+ public boolean containsLedger(long ledgerId) {
+ return ledgersMap.containsKey(ledgerId);
+ }
+
+ public double getUsage() {
+ if (totalSize == 0L) {
+ return 0.0f;
+ }
+ return (double)remainingSize / totalSize;
+ }
+
+ public boolean isEmpty() {
+ return ledgersMap.isEmpty();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("{ totalSize = ").append(totalSize).append(", remainingSize = ")
+ .append(remainingSize).append(", ledgersMap = ").append(ledgersMap).append(" }");
+ return sb.toString();
+ }
+ }
+
+ /**
+ * A scanner used to extract entry log meta from entry log files.
+ */
+ static class ExtractionScanner implements EntryLogScanner {
+ EntryLogMetadata meta;
+
+ public ExtractionScanner(EntryLogMetadata meta) {
+ this.meta = meta;
+ }
+
+ @Override
+ public boolean accept(long ledgerId) {
+ return true;
+ }
+ @Override
+ public void process(long ledgerId, ByteBuffer entry) {
+ // add new entry size of a ledger to entry log meta
+ meta.addLedgerSize(ledgerId, entry.limit() + 4);
+ }
+ }
+
+ /**
+ * Method to read in all of the entry logs (those that we haven't done so yet),
+ * and find the set of ledger ID's that make up each entry log file.
+ *
+ * @param entryLogMetaMap
+ * Existing EntryLogs to Meta
+ * @throws IOException
+ */
+ protected Map<Long, EntryLogMetadata> extractMetaFromEntryLogs(Map<Long, EntryLogMetadata> entryLogMetaMap)
+ throws IOException {
+ // Extract it for every entry log except for the current one.
+ // Entry Log ID's are just a long value that starts at 0 and increments
+ // by 1 when the log fills up and we roll to a new one.
+ long curLogId = entryLogger.logId;
+ for (long entryLogId = 0; entryLogId < curLogId; entryLogId++) {
+ // Comb the current entry log file if it has not already been extracted.
+ if (entryLogMetaMap.containsKey(entryLogId)) {
+ continue;
+ }
+ LOG.info("Extracting entry log meta from entryLogId: " + entryLogId);
+
+ // Read through the entry log file and extract the entry log meta
+ entryLogMetaMap.put(entryLogId,
+ extractMetaFromEntryLog(entryLogger, entryLogId));
+ }
+ return entryLogMetaMap;
+ }
+
+ static EntryLogMetadata extractMetaFromEntryLog(EntryLogger entryLogger, long entryLogId)
+ throws IOException {
+ EntryLogMetadata entryLogMeta = new EntryLogMetadata(entryLogId);
+ ExtractionScanner scanner = new ExtractionScanner(entryLogMeta);
+ try {
+ // Read through the entry log file and extract the entry log meta
+ entryLogger.scanEntryLog(entryLogId, scanner);
+ LOG.info("Retrieved entry log meta data entryLogId: "
+ + entryLogId + ", meta: " + entryLogMeta);
+ } catch(IOException e) {
+ LOG.warn("Premature exception when processing " + entryLogId +
+ "recovery will take care of the problem", e);
+ }
+
+ return entryLogMeta;
+ }
}
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/EntryLogTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/EntryLogTest.java?rev=1303286&r1=1303285&r2=1303286&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/EntryLogTest.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/EntryLogTest.java Wed Mar 21 05:13:23 2012
@@ -29,7 +29,7 @@ import java.util.HashMap;
import junit.framework.TestCase;
-import org.apache.bookkeeper.bookie.EntryLogger.EntryLogMetadata;
+import org.apache.bookkeeper.bookie.GarbageCollectorThread.EntryLogMetadata;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.junit.After;
import org.junit.Before;
@@ -70,8 +70,9 @@ public class EntryLogTest extends TestCa
raf.close();
// now see which ledgers are in the log
logger = new EntryLogger(conf);
+
EntryLogMetadata meta =
- logger.extractMetaFromEntryLog(0L);
+ GarbageCollectorThread.extractMetaFromEntryLog(logger, 0L);
LOG.info("Extracted Meta From Entry Log {}", meta);
assertNotNull(meta.ledgersMap.get(1L));