You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by gh...@apache.org on 2020/11/21 17:45:54 UTC
[bookkeeper] branch master updated: Opportunistic Striping
This is an automated email from the ASF dual-hosted git repository.
ghatage pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 34074b7 Opportunistic Striping
34074b7 is described below
commit 34074b722b60a4c13041f9c9cb6db4e88e2cec6d
Author: Enrico Olivelli <eo...@gmail.com>
AuthorDate: Sat Nov 21 18:45:47 2020 +0100
Opportunistic Striping
### Motivation
If you have a very small cluster, 3 bookies, you must use **2-2-2** as replication parameters in order to guarantee durability and tolerate the failure of at least one bookie.
With this situation you cannot leverage the striping feature of BookKeeper, when **ensembleSize > writeQuorumSize**.
With striping you can distribute your data among all of the bookies, and use better the available disk space.
You can also exploit the ability to write to/read from more disks (bookies) in parallel, achieving better overall performances.
With this change we introduce a new client option "opportunistic striping", that basically tells to the BK client to use as much bookies as configured in the ensembleSize parameter in the best case scenario, and to use less and less bookies, up to the configured writeQuorumSize.
### Changes
- New ClientConfiguration option "opportunisticStriping"
- in case of opportunisticStriping the client is allowed to use less bookies than the requested ensemble size, up to writeQuorumSize
- add test cases
Reviewers: Nicolò Boschi <bo...@gmail.com>, Anup Ghatage <gh...@apache.org>
This closes #2493 from eolivelli/fix/bp-43-impl-auto-striping
---
.../apache/bookkeeper/client/LedgerCreateOp.java | 57 +++++++++++++++++-----
.../bookkeeper/conf/ClientConfiguration.java | 31 ++++++++++++
.../bookkeeper/client/MockBookKeeperTestCase.java | 8 ++-
.../client/api/BookKeeperBuildersTest.java | 50 ++++++++++++++++++-
4 files changed, 131 insertions(+), 15 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java
index ac27c86..221dd06 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java
@@ -125,9 +125,52 @@ class LedgerCreateOp {
* Initiates the operation.
*/
public void initiate() {
+ int actualEnsembleSize = ensembleSize;
+ List<BookieId> ensemble = null;
+ // select bookies for first ensemble
+ if (bk.getConf().getOpportunisticStriping()) {
+ BKNotEnoughBookiesException lastError = null;
+ // we would like to select ensembleSize bookies, but
+ // we can settle to writeQuorumSize
+ while (actualEnsembleSize >= writeQuorumSize) {
+ try {
+ ensemble = bk.getBookieWatcher()
+ .newEnsemble(actualEnsembleSize, writeQuorumSize, ackQuorumSize, customMetadata);
+ lastError = null;
+ break;
+ } catch (BKNotEnoughBookiesException e) {
+ if (actualEnsembleSize >= writeQuorumSize + 1) {
+ LOG.info("Not enough bookies to create ledger with ensembleSize={},"
+ + " writeQuorumSize={} and ackQuorumSize={}, opportusticStriping enabled, try again",
+ actualEnsembleSize, writeQuorumSize, ackQuorumSize);
+ }
+ lastError = e;
+ actualEnsembleSize--;
+ }
+ }
+ if (lastError != null) {
+ LOG.error("Not enough bookies to create ledger with ensembleSize={},"
+ + " writeQuorumSize={} and ackQuorumSize={}",
+ actualEnsembleSize, writeQuorumSize, ackQuorumSize);
+ createComplete(lastError.getCode(), null);
+ return;
+ }
+ } else {
+ try {
+ ensemble = bk.getBookieWatcher()
+ .newEnsemble(actualEnsembleSize, writeQuorumSize, ackQuorumSize, customMetadata);
+ } catch (BKNotEnoughBookiesException e) {
+ LOG.error("Not enough bookies to create ledger with ensembleSize={},"
+ + " writeQuorumSize={} and ackQuorumSize={}",
+ actualEnsembleSize, writeQuorumSize, ackQuorumSize);
+ createComplete(e.getCode(), null);
+ return;
+ }
+ }
LedgerMetadataBuilder metadataBuilder = LedgerMetadataBuilder.create()
- .withEnsembleSize(ensembleSize).withWriteQuorumSize(writeQuorumSize).withAckQuorumSize(ackQuorumSize)
+ .withEnsembleSize(actualEnsembleSize).withWriteQuorumSize(writeQuorumSize).withAckQuorumSize(ackQuorumSize)
.withDigestType(digestType.toApiDigestType()).withPassword(passwd);
+ metadataBuilder.newEnsembleEntry(0L, ensemble);
if (customMetadata != null) {
metadataBuilder.withCustomMetadata(customMetadata);
}
@@ -135,18 +178,6 @@ class LedgerCreateOp {
metadataBuilder.withCreationTime(System.currentTimeMillis()).storingCreationTime(true);
}
- // select bookies for first ensemble
- try {
- List<BookieId> ensemble = bk.getBookieWatcher()
- .newEnsemble(ensembleSize, writeQuorumSize, ackQuorumSize, customMetadata);
- metadataBuilder.newEnsembleEntry(0L, ensemble);
- } catch (BKNotEnoughBookiesException e) {
- LOG.error("Not enough bookies to create ledger");
- createComplete(e.getCode(), null);
- return;
- }
-
-
if (this.generateLedgerId) {
generateLedgerIdAndCreateLedger(metadataBuilder);
} else {
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ClientConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ClientConfiguration.java
index 76ef7bd..80c2ad4 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ClientConfiguration.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ClientConfiguration.java
@@ -116,6 +116,7 @@ public class ClientConfiguration extends AbstractConfiguration<ClientConfigurati
protected static final String REORDER_READ_SEQUENCE_ENABLED = "reorderReadSequenceEnabled";
protected static final String STICKY_READS_ENABLED = "stickyReadSEnabled";
// Add Parameters
+ protected static final String OPPORTUNISTIC_STRIPING = "opportunisticStriping";
protected static final String DELAY_ENSEMBLE_CHANGE = "delayEnsembleChange";
protected static final String MAX_ALLOWED_ENSEMBLE_CHANGES = "maxNumEnsembleChanges";
// Timeout Setting
@@ -1742,6 +1743,36 @@ public class ClientConfiguration extends AbstractConfiguration<ClientConfigurati
}
/**
+ * Whether to allow opportunistic striping.
+ *
+ * @return true if opportunistic striping is enabled
+ */
+ public boolean getOpportunisticStriping() {
+ return getBoolean(OPPORTUNISTIC_STRIPING, false);
+ }
+
+ /**
+ * Enable/Disable opportunistic striping.
+ * <p>
+ * If set to true, when you are creating a ledger with a given
+ * ensemble size, the system will automatically handle the
+ * lack of enough bookies, reducing ensemble size up to
+ * the write quorum size. This way in little clusters
+ * you can try to use striping (ensemble size > write quorum size)
+ * in case that you have enough bookies up and running,
+ * and degrade automatically to the minimum requested replication count.
+ * </p>
+ *
+ * @param enabled
+ * flag to enable/disable opportunistic striping.
+ * @return client configuration.
+ */
+ public ClientConfiguration setOpportunisticStriping(boolean enabled) {
+ setProperty(OPPORTUNISTIC_STRIPING, enabled);
+ return this;
+ }
+
+ /**
* Whether to delay ensemble change or not?
*
* @return true if to delay ensemble change, otherwise false.
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockBookKeeperTestCase.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockBookKeeperTestCase.java
index e85771d..0c5ee6d 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockBookKeeperTestCase.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockBookKeeperTestCase.java
@@ -109,6 +109,7 @@ public abstract class MockBookKeeperTestCase {
List<BookieId> failedBookies;
Set<BookieId> availableBookies;
private int lastIndexForBK;
+ protected int maxNumberOfAvailableBookies = Integer.MAX_VALUE;
private Map<BookieId, Map<Long, MockEntry>> getMockLedgerContents(long ledgerId) {
return mockLedgerData.computeIfAbsent(ledgerId, (id) -> new ConcurrentHashMap<>());
@@ -140,6 +141,7 @@ public abstract class MockBookKeeperTestCase {
@Before
public void setup() throws Exception {
+ maxNumberOfAvailableBookies = Integer.MAX_VALUE;
deferredBookieForceLedgerResponses = new ConcurrentHashMap<>();
suspendedBookiesForForceLedgerAcks = Collections.synchronizedSet(new HashSet<>());
mockLedgerMetadataRegistry = new ConcurrentHashMap<>();
@@ -314,7 +316,11 @@ public abstract class MockBookKeeperTestCase {
return new BookieSocketAddress("localhost", 1111 + index).toBookieId();
}
- protected ArrayList<BookieId> generateNewEnsemble(int ensembleSize) {
+ protected ArrayList<BookieId> generateNewEnsemble(int ensembleSize) throws BKException.BKNotEnoughBookiesException {
+ LOG.info("generateNewEnsemble {}", ensembleSize);
+ if (ensembleSize > maxNumberOfAvailableBookies) {
+ throw new BKException.BKNotEnoughBookiesException();
+ }
ArrayList<BookieId> ensemble = new ArrayList<>(ensembleSize);
for (int i = 0; i < ensembleSize; i++) {
ensemble.add(generateBookieSocketAddress(i));
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperBuildersTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperBuildersTest.java
index ae3ed6a..246c3d3 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperBuildersTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperBuildersTest.java
@@ -28,6 +28,7 @@ import static org.junit.Assert.fail;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map;
+import org.apache.bookkeeper.client.BKException;
import org.apache.bookkeeper.client.BKException.BKClientClosedException;
import org.apache.bookkeeper.client.BKException.BKIncorrectParameterException;
import org.apache.bookkeeper.client.BKException.BKNoSuchLedgerExistsOnMetadataServerException;
@@ -404,7 +405,7 @@ public class BookKeeperBuildersTest extends MockBookKeeperTestCase {
protected LedgerMetadata generateLedgerMetadata(int ensembleSize,
int writeQuorumSize, int ackQuorumSize, byte[] password,
- Map<String, byte[]> customMetadata) {
+ Map<String, byte[]> customMetadata) throws BKException.BKNotEnoughBookiesException {
return LedgerMetadataBuilder.create()
.withId(12L)
.withEnsembleSize(ensembleSize)
@@ -417,4 +418,51 @@ public class BookKeeperBuildersTest extends MockBookKeeperTestCase {
.newEnsembleEntry(0, generateNewEnsemble(ensembleSize)).build();
}
+ @Test
+ public void testCreateLedgerWithOpportunisticStriping() throws Exception {
+
+ maxNumberOfAvailableBookies = 4;
+ int bigEnsembleSize = 15;
+ int expectedWriteQuorumSize = 4;
+
+ ClientConfiguration config = new ClientConfiguration();
+ config.setOpportunisticStriping(true);
+ setBookKeeperConfig(config);
+
+ setNewGeneratedLedgerId(ledgerId);
+ WriteHandle writer = newCreateLedgerOp()
+ .withAckQuorumSize(expectedWriteQuorumSize)
+ .withEnsembleSize(bigEnsembleSize)
+ .withWriteQuorumSize(expectedWriteQuorumSize)
+ .withCustomMetadata(customMetadata)
+ .withPassword(password)
+ .execute()
+ .get();
+ assertEquals(ledgerId, writer.getId());
+ LedgerMetadata metadata = getLedgerMetadata(ledgerId);
+ assertEquals(expectedWriteQuorumSize, metadata.getEnsembleSize());
+ assertEquals(expectedWriteQuorumSize, metadata.getAckQuorumSize());
+ assertEquals(expectedWriteQuorumSize, metadata.getWriteQuorumSize());
+ assertArrayEquals(password, metadata.getPassword());
+
+ }
+
+ @Test(expected = BKException.BKNotEnoughBookiesException.class)
+ public void testNotEnoughBookies() throws Exception {
+
+ maxNumberOfAvailableBookies = 1;
+ ClientConfiguration config = new ClientConfiguration();
+ config.setOpportunisticStriping(false);
+ setBookKeeperConfig(config);
+
+ setNewGeneratedLedgerId(ledgerId);
+ result(newCreateLedgerOp()
+ .withAckQuorumSize(ackQuorumSize)
+ .withEnsembleSize(ensembleSize)
+ .withWriteQuorumSize(writeQuorumSize)
+ .withCustomMetadata(customMetadata)
+ .withPassword(password)
+ .execute());
+ }
+
}