You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by eo...@apache.org on 2020/11/24 13:04:54 UTC

[bookkeeper] branch branch-4.11 updated: Opportunistic Striping

This is an automated email from the ASF dual-hosted git repository.

eolivelli pushed a commit to branch branch-4.11
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git


The following commit(s) were added to refs/heads/branch-4.11 by this push:
     new 27879ea  Opportunistic Striping
27879ea is described below

commit 27879ea00eece6904223951860c9b39250dbc68b
Author: Anup Ghatage <gh...@apache.org>
AuthorDate: Tue Nov 24 05:04:47 2020 -0800

    Opportunistic Striping
    
    If you have a very small cluster, 3 bookies, you must use **2-2-2** as replication parameters in order to guarantee durability and tolerate the failure of at least one bookie.
    With this situation you cannot leverage the striping feature of BookKeeper, when **ensembleSize > writeQuorumSize**.
    
    With striping you can distribute your data among all of the bookies, and use better the available disk space.
    You can also exploit the ability to write to/read from more disks (bookies) in parallel, achieving better overall performances.
    
    With this change we introduce a new client option "opportunistic striping", that basically tells to the BK client to use as much bookies as configured in the ensembleSize parameter in the best case scenario, and to use less and less bookies, up to the configured writeQuorumSize.
    
    - New ClientConfiguration option "opportunisticStriping"
    - in case of opportunisticStriping the client is allowed to use less bookies than the requested ensemble size, up to writeQuorumSize
    - add test cases
    
    Reviewers: Nicolò Boschi <boschi1997gmail.com>, Anup Ghatage <ghatageapache.org>
    
    This closes #2493 from eolivelli/fix/bp-43-impl-auto-striping
    
    Descriptions of the changes in this PR:
    
    
    
    ### Motivation
    
    (Explain: why you're making that change, what is the problem you're trying to solve)
    
    ### Changes
    
    (Describe: what changes you have made)
    
    Master Issue: #<master-issue-number>
    
    
    
    Reviewers: Enrico Olivelli <eo...@gmail.com>
    
    This closes #2495 from Ghatage/branch-4.11
---
 .../apache/bookkeeper/client/LedgerCreateOp.java   | 57 +++++++++++++++++-----
 .../bookkeeper/conf/ClientConfiguration.java       | 31 ++++++++++++
 .../bookkeeper/client/MockBookKeeperTestCase.java  |  8 ++-
 .../client/api/BookKeeperBuildersTest.java         | 50 ++++++++++++++++++-
 4 files changed, 131 insertions(+), 15 deletions(-)

diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java
index 9e9a2c4..01f5a53 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java
@@ -125,9 +125,52 @@ class LedgerCreateOp {
      * Initiates the operation.
      */
     public void initiate() {
+        int actualEnsembleSize = ensembleSize;
+        List<BookieSocketAddress> ensemble = null;
+        // select bookies for first ensemble
+        if (bk.getConf().getOpportunisticStriping()) {
+            BKNotEnoughBookiesException lastError = null;
+            // we would like to select ensembleSize bookies, but
+            // we can settle to writeQuorumSize
+            while (actualEnsembleSize >= writeQuorumSize) {
+                try {
+                    ensemble = bk.getBookieWatcher()
+                        .newEnsemble(actualEnsembleSize, writeQuorumSize, ackQuorumSize, customMetadata);
+                    lastError = null;
+                    break;
+                } catch (BKNotEnoughBookiesException e) {
+                    if (actualEnsembleSize >= writeQuorumSize + 1) {
+                        LOG.info("Not enough bookies to create ledger with ensembleSize={},"
+                                + " writeQuorumSize={} and ackQuorumSize={}, opportusticStriping enabled, try again",
+                                    actualEnsembleSize, writeQuorumSize, ackQuorumSize);
+                    }
+                    lastError = e;
+                    actualEnsembleSize--;
+                }
+            }
+            if (lastError != null) {
+                LOG.error("Not enough bookies to create ledger with ensembleSize={},"
+                        + " writeQuorumSize={} and ackQuorumSize={}",
+                        actualEnsembleSize, writeQuorumSize, ackQuorumSize);
+                createComplete(lastError.getCode(), null);
+                return;
+            }
+        } else {
+            try {
+                ensemble = bk.getBookieWatcher()
+                        .newEnsemble(actualEnsembleSize, writeQuorumSize, ackQuorumSize, customMetadata);
+            } catch (BKNotEnoughBookiesException e) {
+                LOG.error("Not enough bookies to create ledger with ensembleSize={},"
+                        + " writeQuorumSize={} and ackQuorumSize={}",
+                            actualEnsembleSize, writeQuorumSize, ackQuorumSize);
+                createComplete(e.getCode(), null);
+                return;
+            }
+        }
         LedgerMetadataBuilder metadataBuilder = LedgerMetadataBuilder.create()
-            .withEnsembleSize(ensembleSize).withWriteQuorumSize(writeQuorumSize).withAckQuorumSize(ackQuorumSize)
+            .withEnsembleSize(actualEnsembleSize).withWriteQuorumSize(writeQuorumSize).withAckQuorumSize(ackQuorumSize)
             .withDigestType(digestType.toApiDigestType()).withPassword(passwd);
+        metadataBuilder.newEnsembleEntry(0L, ensemble);
         if (customMetadata != null) {
             metadataBuilder.withCustomMetadata(customMetadata);
         }
@@ -135,18 +178,6 @@ class LedgerCreateOp {
             metadataBuilder.withCreationTime(System.currentTimeMillis()).storingCreationTime(true);
         }
 
-        // select bookies for first ensemble
-        try {
-            List<BookieSocketAddress> ensemble = bk.getBookieWatcher()
-                .newEnsemble(ensembleSize, writeQuorumSize, ackQuorumSize, customMetadata);
-            metadataBuilder.newEnsembleEntry(0L, ensemble);
-        } catch (BKNotEnoughBookiesException e) {
-            LOG.error("Not enough bookies to create ledger");
-            createComplete(e.getCode(), null);
-            return;
-        }
-
-
         this.metadata = metadataBuilder.build();
         if (this.generateLedgerId) {
             generateLedgerIdAndCreateLedger();
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ClientConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ClientConfiguration.java
index 663ad63..7e3826a 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ClientConfiguration.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ClientConfiguration.java
@@ -116,6 +116,7 @@ public class ClientConfiguration extends AbstractConfiguration<ClientConfigurati
     protected static final String REORDER_READ_SEQUENCE_ENABLED = "reorderReadSequenceEnabled";
     protected static final String STICKY_READS_ENABLED = "stickyReadSEnabled";
     // Add Parameters
+    protected static final String OPPORTUNISTIC_STRIPING = "opportunisticStriping";
     protected static final String DELAY_ENSEMBLE_CHANGE = "delayEnsembleChange";
     protected static final String MAX_ALLOWED_ENSEMBLE_CHANGES = "maxNumEnsembleChanges";
     // Timeout Setting
@@ -1739,6 +1740,36 @@ public class ClientConfiguration extends AbstractConfiguration<ClientConfigurati
     }
 
     /**
+     * Whether to allow opportunistic striping.
+     *
+     * @return true if opportunistic striping is enabled
+     */
+    public boolean getOpportunisticStriping() {
+        return getBoolean(OPPORTUNISTIC_STRIPING, false);
+    }
+
+    /**
+     * Enable/Disable opportunistic striping.
+     * <p>
+     * If set to true, when you are creating a ledger with a given
+     * ensemble size, the system will automatically handle the
+     * lack of enough bookies, reducing ensemble size up to
+     * the write quorum size. This way in little clusters
+     * you can try to use striping (ensemble size > write quorum size)
+     * in case that you have enough bookies up and running,
+     * and degrade automatically to the minimum requested replication count.
+     * </p>
+     *
+     * @param enabled
+     *          flag to enable/disable opportunistic striping.
+     * @return client configuration.
+     */
+    public ClientConfiguration setOpportunisticStriping(boolean enabled) {
+        setProperty(OPPORTUNISTIC_STRIPING, enabled);
+        return this;
+    }
+
+    /**
      * Whether to delay ensemble change or not?
      *
      * @return true if to delay ensemble change, otherwise false.
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockBookKeeperTestCase.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockBookKeeperTestCase.java
index c5878f1..cfde866 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockBookKeeperTestCase.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockBookKeeperTestCase.java
@@ -107,6 +107,7 @@ public abstract class MockBookKeeperTestCase {
     List<BookieSocketAddress> failedBookies;
     Set<BookieSocketAddress> availableBookies;
     private int lastIndexForBK;
+    protected int maxNumberOfAvailableBookies = Integer.MAX_VALUE;
 
     private Map<BookieSocketAddress, Map<Long, MockEntry>> getMockLedgerContents(long ledgerId) {
         return mockLedgerData.computeIfAbsent(ledgerId, (id) -> new ConcurrentHashMap<>());
@@ -138,6 +139,7 @@ public abstract class MockBookKeeperTestCase {
 
     @Before
     public void setup() throws Exception {
+        maxNumberOfAvailableBookies = Integer.MAX_VALUE;
         deferredBookieForceLedgerResponses = new ConcurrentHashMap<>();
         suspendedBookiesForForceLedgerAcks = Collections.synchronizedSet(new HashSet<>());
         mockLedgerMetadataRegistry = new ConcurrentHashMap<>();
@@ -308,7 +310,11 @@ public abstract class MockBookKeeperTestCase {
         return new BookieSocketAddress("localhost", 1111 + index);
     }
 
-    protected ArrayList<BookieSocketAddress> generateNewEnsemble(int ensembleSize) {
+    protected ArrayList<BookieSocketAddress> generateNewEnsemble(int ensembleSize) throws BKException.BKNotEnoughBookiesException {
+        LOG.info("generateNewEnsemble {}", ensembleSize);
+        if (ensembleSize > maxNumberOfAvailableBookies) {
+            throw new BKException.BKNotEnoughBookiesException();
+        }
         ArrayList<BookieSocketAddress> ensemble = new ArrayList<>(ensembleSize);
         for (int i = 0; i < ensembleSize; i++) {
             ensemble.add(generateBookieSocketAddress(i));
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperBuildersTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperBuildersTest.java
index f0e07e2..4c7c44d 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperBuildersTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperBuildersTest.java
@@ -28,6 +28,7 @@ import static org.junit.Assert.fail;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Map;
+import org.apache.bookkeeper.client.BKException;
 import org.apache.bookkeeper.client.BKException.BKClientClosedException;
 import org.apache.bookkeeper.client.BKException.BKIncorrectParameterException;
 import org.apache.bookkeeper.client.BKException.BKNoSuchLedgerExistsOnMetadataServerException;
@@ -404,7 +405,7 @@ public class BookKeeperBuildersTest extends MockBookKeeperTestCase {
 
     protected LedgerMetadata generateLedgerMetadata(int ensembleSize,
         int writeQuorumSize, int ackQuorumSize, byte[] password,
-        Map<String, byte[]> customMetadata) {
+        Map<String, byte[]> customMetadata) throws BKException.BKNotEnoughBookiesException {
         return LedgerMetadataBuilder.create()
             .withEnsembleSize(ensembleSize)
             .withWriteQuorumSize(writeQuorumSize)
@@ -416,4 +417,51 @@ public class BookKeeperBuildersTest extends MockBookKeeperTestCase {
             .newEnsembleEntry(0, generateNewEnsemble(ensembleSize)).build();
     }
 
+    @Test
+    public void testCreateLedgerWithOpportunisticStriping() throws Exception {
+
+        maxNumberOfAvailableBookies =  4;
+        int bigEnsembleSize = 15;
+        int expectedWriteQuorumSize = 4;
+
+        ClientConfiguration config = new ClientConfiguration();
+        config.setOpportunisticStriping(true);
+        setBookKeeperConfig(config);
+
+        setNewGeneratedLedgerId(ledgerId);
+        WriteHandle writer = newCreateLedgerOp()
+            .withAckQuorumSize(expectedWriteQuorumSize)
+            .withEnsembleSize(bigEnsembleSize)
+            .withWriteQuorumSize(expectedWriteQuorumSize)
+            .withCustomMetadata(customMetadata)
+            .withPassword(password)
+            .execute()
+            .get();
+        assertEquals(ledgerId, writer.getId());
+        LedgerMetadata metadata = getLedgerMetadata(ledgerId);
+        assertEquals(expectedWriteQuorumSize, metadata.getEnsembleSize());
+        assertEquals(expectedWriteQuorumSize, metadata.getAckQuorumSize());
+        assertEquals(expectedWriteQuorumSize, metadata.getWriteQuorumSize());
+        assertArrayEquals(password, metadata.getPassword());
+
+    }
+
+    @Test(expected = BKException.BKNotEnoughBookiesException.class)
+    public void testNotEnoughBookies() throws Exception {
+
+        maxNumberOfAvailableBookies =  1;
+        ClientConfiguration config = new ClientConfiguration();
+        config.setOpportunisticStriping(false);
+        setBookKeeperConfig(config);
+
+        setNewGeneratedLedgerId(ledgerId);
+        result(newCreateLedgerOp()
+            .withAckQuorumSize(ackQuorumSize)
+            .withEnsembleSize(ensembleSize)
+            .withWriteQuorumSize(writeQuorumSize)
+            .withCustomMetadata(customMetadata)
+            .withPassword(password)
+            .execute());
+    }
+
 }