You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zookeeper.apache.org by iv...@apache.org on 2013/10/30 16:23:55 UTC
svn commit: r1537130 - in /zookeeper/bookkeeper/trunk: CHANGES.txt
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java
Author: ivank
Date: Wed Oct 30 15:23:54 2013
New Revision: 1537130
URL: http://svn.apache.org/r1537130
Log:
BOOKKEEPER-700: GarbageCollectorThread exsiting with ArrayIndexOutOfBoundsException (rakeshr via ivank)
Modified:
zookeeper/bookkeeper/trunk/CHANGES.txt
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java
Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1537130&r1=1537129&r2=1537130&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Wed Oct 30 15:23:54 2013
@@ -120,6 +120,8 @@ Trunk (unreleased changes)
BOOKKEEPER-659: LRU page management in ledger cache. (Aniruddha, Robin Dhamankar & sijie via ivank)
+ BOOKKEEPER-700: GarbageCollectorThread exsiting with ArrayIndexOutOfBoundsException (rakeshr via ivank)
+
hedwig-server:
BOOKKEEPER-601: readahead cache size isn't updated correctly (sijie via fpj)
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java?rev=1537130&r1=1537129&r2=1537130&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java Wed Oct 30 15:23:54 2013
@@ -31,6 +31,7 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.RateLimiter;
import org.apache.bookkeeper.bookie.EntryLogger.EntryLogScanner;
@@ -147,6 +148,10 @@ public class GarbageCollectorThread exte
synchronized private void waitEntrylogFlushed() throws IOException {
try {
+ if (offsets.size() <= 0) {
+ LOG.debug("Skipping entry log flushing, as there is no offset!");
+ return;
+ }
synchronized (flushLock) {
Offset lastOffset = offsets.get(offsets.size()-1);
long lastOffsetLogId = EntryLogger.logIdForOffset(lastOffset.offset);
@@ -351,7 +356,8 @@ public class GarbageCollectorThread exte
* would not be compacted.
* </p>
*/
- private void doCompactEntryLogs(double threshold) {
+ @VisibleForTesting
+ void doCompactEntryLogs(double threshold) {
LOG.info("Do compaction to compact those files lower than " + threshold);
// sort the ledger meta by occupied unused space
Comparator<EntryLogMetadata> sizeComparator = new Comparator<EntryLogMetadata>() {
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java?rev=1537130&r1=1537129&r2=1537130&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java Wed Oct 30 15:23:54 2013
@@ -22,7 +22,6 @@ package org.apache.bookkeeper.bookie;
*/
import java.io.File;
import java.io.IOException;
-import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -321,54 +320,7 @@ public class CompactionTest extends Book
tearDown(); // I dont want the test infrastructure
ServerConfiguration conf = new ServerConfiguration();
final Set<Long> ledgers = Collections.newSetFromMap(new ConcurrentHashMap<Long, Boolean>());
- LedgerManager manager = new LedgerManager() {
- @Override
- public void createLedger(LedgerMetadata metadata, GenericCallback<Long> cb) {
- unsupported();
- }
- @Override
- public void removeLedgerMetadata(long ledgerId, Version version,
- GenericCallback<Void> vb) {
- unsupported();
- }
- @Override
- public void readLedgerMetadata(long ledgerId, GenericCallback<LedgerMetadata> readCb) {
- unsupported();
- }
- @Override
- public void writeLedgerMetadata(long ledgerId, LedgerMetadata metadata,
- GenericCallback<Void> cb) {
- unsupported();
- }
- @Override
- public void asyncProcessLedgers(Processor<Long> processor,
- AsyncCallback.VoidCallback finalCb,
- Object context, int successRc, int failureRc) {
- unsupported();
- }
- @Override
- public void close() throws IOException {}
-
- void unsupported() {
- LOG.error("Unsupported operation called", new Exception());
- throw new RuntimeException("Unsupported op");
- }
- @Override
- public LedgerRangeIterator getLedgerRanges() {
- final AtomicBoolean hasnext = new AtomicBoolean(true);
- return new LedgerManager.LedgerRangeIterator() {
- @Override
- public boolean hasNext() throws IOException {
- return hasnext.get();
- }
- @Override
- public LedgerManager.LedgerRange next() throws IOException {
- hasnext.set(false);
- return new LedgerManager.LedgerRange(ledgers);
- }
- };
- }
- };
+ LedgerManager manager = getLedgerManager(ledgers);
File tmpDir = File.createTempFile("bkTest", ".dir");
tmpDir.delete();
@@ -451,8 +403,98 @@ public class CompactionTest extends Book
storage.getEntry(1, 1); // entry should exist
}
+ private LedgerManager getLedgerManager(final Set<Long> ledgers) {
+ LedgerManager manager = new LedgerManager() {
+ @Override
+ public void createLedger(LedgerMetadata metadata, GenericCallback<Long> cb) {
+ unsupported();
+ }
+ @Override
+ public void removeLedgerMetadata(long ledgerId, Version version,
+ GenericCallback<Void> vb) {
+ unsupported();
+ }
+ @Override
+ public void readLedgerMetadata(long ledgerId, GenericCallback<LedgerMetadata> readCb) {
+ unsupported();
+ }
+ @Override
+ public void writeLedgerMetadata(long ledgerId, LedgerMetadata metadata,
+ GenericCallback<Void> cb) {
+ unsupported();
+ }
+ @Override
+ public void asyncProcessLedgers(Processor<Long> processor,
+ AsyncCallback.VoidCallback finalCb,
+ Object context, int successRc, int failureRc) {
+ unsupported();
+ }
+ @Override
+ public void close() throws IOException {}
+
+ void unsupported() {
+ LOG.error("Unsupported operation called", new Exception());
+ throw new RuntimeException("Unsupported op");
+ }
+ @Override
+ public LedgerRangeIterator getLedgerRanges() {
+ final AtomicBoolean hasnext = new AtomicBoolean(true);
+ return new LedgerManager.LedgerRangeIterator() {
+ @Override
+ public boolean hasNext() throws IOException {
+ return hasnext.get();
+ }
+ @Override
+ public LedgerManager.LedgerRange next() throws IOException {
+ hasnext.set(false);
+ return new LedgerManager.LedgerRange(ledgers);
+ }
+ };
+ }
+ };
+ return manager;
+ }
+
+ /**
+ * Test that compaction should execute silently when there is no entry logs
+ * to compact. {@see https://issues.apache.org/jira/browse/BOOKKEEPER-700}
+ */
+ @Test(timeout = 60000)
+ public void testWhenNoLogsToCompact() throws Exception {
+ tearDown(); // I dont want the test infrastructure
+ ServerConfiguration conf = new ServerConfiguration();
+ File tmpDir = File.createTempFile("bkTest", ".dir");
+ tmpDir.delete();
+ tmpDir.mkdir();
+ File curDir = Bookie.getCurrentDirectory(tmpDir);
+ Bookie.checkDirectoryStructure(curDir);
+ conf.setLedgerDirNames(new String[] { tmpDir.toString() });
+
+ LedgerDirsManager dirs = new LedgerDirsManager(conf);
+ final Set<Long> ledgers = Collections
+ .newSetFromMap(new ConcurrentHashMap<Long, Boolean>());
+ LedgerManager manager = getLedgerManager(ledgers);
+ CheckpointSource checkpointSource = new CheckpointSource() {
+
+ @Override
+ public Checkpoint newCheckpoint() {
+ return null;
+ }
+
+ @Override
+ public void checkpointComplete(Checkpoint checkpoint,
+ boolean compact) throws IOException {
+ }
+ };
+ InterleavedLedgerStorage storage = new InterleavedLedgerStorage(conf,
+ manager, dirs, checkpointSource);
+
+ double threshold = 0.1;
+ // shouldn't throw exception
+ storage.gcThread.doCompactEntryLogs(threshold);
+ }
+
private ByteBuffer genEntry(long ledger, long entry, int size) {
- byte[] data = new byte[size];
ByteBuffer bb = ByteBuffer.wrap(new byte[size]);
bb.putLong(ledger);
bb.putLong(entry);