You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by GitBox <gi...@apache.org> on 2018/05/02 17:10:27 UTC
[GitHub] sijie closed pull request #1687: Don't offload empty ledgers
sijie closed pull request #1687: Don't offload empty ledgers
URL: https://github.com/apache/incubator-pulsar/pull/1687
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
index f1d71007a4..1a606b7d4a 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -1966,7 +1966,8 @@ public void asyncOffloadPrefix(Position pos, OffloadCallback callback, Object ct
long firstLedgerRetained = current;
for (LedgerInfo ls : ledgers.headMap(current).values()) {
if (requestOffloadTo.getLedgerId() > ls.getLedgerId()) {
- if (!ls.getOffloadContext().getComplete()) {
+ // don't offload if ledger has already been offloaded, or is empty
+ if (!ls.getOffloadContext().getComplete() && ls.getSize() > 0) {
ledgersToOffload.add(ls);
}
} else {
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java
index 278182cef3..76cbbb6de5 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java
@@ -20,6 +20,7 @@
import com.google.common.collect.ImmutableSet;
+import java.lang.reflect.Field;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
@@ -40,6 +41,7 @@
import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
import org.apache.bookkeeper.mledger.ManagedLedgerException;
import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo;
import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
import org.apache.commons.lang3.tuple.Pair;
@@ -633,6 +635,47 @@ public void testOffloadDeleteIncomplete() throws Exception {
assertEventuallyTrue(() -> offloader.deletedOffloads().contains(firstLedger));
}
+ @Test
+ public void testDontOffloadEmpty() throws Exception {
+ MockLedgerOffloader offloader = new MockLedgerOffloader();
+ ManagedLedgerConfig config = new ManagedLedgerConfig();
+ config.setMaxEntriesPerLedger(10);
+ config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
+ config.setRetentionTime(10, TimeUnit.MINUTES);
+ config.setLedgerOffloader(offloader);
+ ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
+
+ int i = 0;
+ for (; i < 35; i++) {
+ String content = "entry-" + i;
+ ledger.addEntry(content.getBytes());
+ }
+ Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 4);
+
+ long firstLedgerId = ledger.getLedgersInfoAsList().get(0).getLedgerId();
+ long secondLedgerId = ledger.getLedgersInfoAsList().get(1).getLedgerId();
+ long thirdLedgerId = ledger.getLedgersInfoAsList().get(2).getLedgerId();
+ long fourthLedgerId = ledger.getLedgersInfoAsList().get(3).getLedgerId();
+
+ // make an ledger empty
+ Field ledgersField = ledger.getClass().getDeclaredField("ledgers");
+ ledgersField.setAccessible(true);
+ Map<Long, LedgerInfo> ledgers = (Map<Long,LedgerInfo>)ledgersField.get(ledger);
+ ledgers.put(secondLedgerId,
+ ledgers.get(secondLedgerId).toBuilder().setEntries(0).setSize(0).build());
+
+ PositionImpl firstUnoffloaded = (PositionImpl)ledger.offloadPrefix(ledger.getLastConfirmedEntry());
+ Assert.assertEquals(firstUnoffloaded.getLedgerId(), fourthLedgerId);
+ Assert.assertEquals(firstUnoffloaded.getEntryId(), 0);
+
+ Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 4);
+ Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
+ .filter(e -> e.getOffloadContext().getComplete())
+ .map(e -> e.getLedgerId()).collect(Collectors.toSet()),
+ offloader.offloadedLedgers());
+ Assert.assertEquals(offloader.offloadedLedgers(), ImmutableSet.of(firstLedgerId, thirdLedgerId));
+ }
+
static void assertEventuallyTrue(BooleanSupplier predicate) throws Exception {
// wait up to 3 seconds
for (int i = 0; i < 30 && !predicate.getAsBoolean(); i++) {
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services