You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by eo...@apache.org on 2020/05/19 06:23:19 UTC

[bookkeeper] branch branch-4.10 updated: [BOOKKEEPER-TOOL] Fix update rate-limiting for update-ledger command

This is an automated email from the ASF dual-hosted git repository.

eolivelli pushed a commit to branch branch-4.10
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git


The following commit(s) were added to refs/heads/branch-4.10 by this push:
     new 87a9fff  [BOOKKEEPER-TOOL] Fix update rate-limiting for update-ledger command
87a9fff is described below

commit 87a9fff1382e14d02950507832d760605804bdf9
Author: Rajan Dhabalia <rd...@apache.org>
AuthorDate: Mon May 18 23:22:35 2020 -0700

    [BOOKKEEPER-TOOL] Fix update rate-limiting for update-ledger command
    
    ### Motivation
    Right now, `UpdateLedgerCmd` provides option `updatespersec` to throttle number of writes on zk. However, it throttles number of reads instead writes. Because of that it takes long time to complete this command as it slows down the read instead applying throttling while writing to zk.
    
    ### Modification
    - Apply throttling while updating zk.
    - If writes are being throttled then we also want to avoid accumulating reads so, added `maxOutstandingReads` option to manage max concurrent reads.
    
    **Note:** We also need this change for #2321 .. so, I will rebase #2321 once this PR is merged.
    
    Reviewers: Enrico Olivelli <eo...@gmail.com>
    
    This closes #2336 from rdhabalia/update_rate
    
    (cherry picked from commit e9021a607d55ed1802c44fd503fbf7640ad3ae98)
    Signed-off-by: Enrico Olivelli <eo...@apache.org>
---
 .../org/apache/bookkeeper/bookie/BookieShell.java  |  7 +++--
 .../bookkeeper/client/MetadataUpdateLoop.java      | 31 ++++++++++++++++------
 .../apache/bookkeeper/client/UpdateLedgerOp.java   | 12 ++++++---
 .../cli/commands/bookie/FlipBookieIdCommand.java   | 13 ++++++++-
 .../bookkeeper/client/UpdateLedgerOpTest.java      | 14 +++++-----
 .../commands/bookie/FlipBookieIdCommandTest.java   |  4 +--
 6 files changed, 57 insertions(+), 24 deletions(-)

diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
index d7f5575..bf4b57c 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
@@ -1492,6 +1492,7 @@ public class BookieShell implements Tool {
             super(CMD_UPDATELEDGER);
             opts.addOption("b", "bookieId", true, "Bookie Id");
             opts.addOption("s", "updatespersec", true, "Number of ledgers updating per second (default: 5 per sec)");
+            opts.addOption("r", "maxOutstandingReads", true, "Max outstanding reads (default: 5 * updatespersec)");
             opts.addOption("l", "limit", true, "Maximum number of ledgers to update (default: no limit)");
             opts.addOption("v", "verbose", true, "Print status of the ledger updation (default: false)");
             opts.addOption("p", "printprogress", true,
@@ -1510,8 +1511,8 @@ public class BookieShell implements Tool {
 
         @Override
         String getUsage() {
-            return "updateledgers -bookieId <hostname|ip> [-updatespersec N] [-limit N] [-verbose true/false] "
-                    + "[-printprogress N]";
+            return "updateledgers -bookieId <hostname|ip> [-updatespersec N] [-maxOutstandingReads N] [-limit N] "
+                    + "[-verbose true/false] [-printprogress N]";
         }
 
         @Override
@@ -1532,6 +1533,7 @@ public class BookieShell implements Tool {
             }
             boolean useHostName = getOptionalValue(bookieId, "hostname");
             final int rate = getOptionIntValue(cmdLine, "updatespersec", 5);
+            final int maxOutstandingReads = getOptionIntValue(cmdLine, "maxOutstandingReads", (rate * 5));
             final int limit = getOptionIntValue(cmdLine, "limit", Integer.MIN_VALUE);
             final boolean verbose = getOptionBooleanValue(cmdLine, "verbose", false);
             final long printprogress;
@@ -1548,6 +1550,7 @@ public class BookieShell implements Tool {
             flags.printProgress(printprogress);
             flags.limit(limit);
             flags.updatePerSec(rate);
+            flags.maxOutstandingReads(maxOutstandingReads);
             flags.verbose(verbose);
 
             boolean result = cmd.apply(bkConf, flags);
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/MetadataUpdateLoop.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/MetadataUpdateLoop.java
index 5c8cb14..d4f46f3 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/MetadataUpdateLoop.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/MetadataUpdateLoop.java
@@ -19,6 +19,8 @@
  */
 package org.apache.bookkeeper.client;
 
+import com.google.common.util.concurrent.RateLimiter;
+
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
 import java.util.function.Supplier;
@@ -56,6 +58,7 @@ class MetadataUpdateLoop {
     private final NeedsUpdatePredicate needsTransformation;
     private final MetadataTransform transform;
     private final LocalValueUpdater updateLocalValue;
+    private final RateLimiter throttler;
 
     private final String logContext;
     private volatile int writeLoopCount = 0;
@@ -74,6 +77,14 @@ class MetadataUpdateLoop {
         boolean updateValue(Versioned<LedgerMetadata> oldValue, Versioned<LedgerMetadata> newValue);
     }
 
+    MetadataUpdateLoop(LedgerManager lm,
+            long ledgerId,
+            Supplier<Versioned<LedgerMetadata>> currentLocalValue,
+            NeedsUpdatePredicate needsTransformation,
+            MetadataTransform transform,
+            LocalValueUpdater updateLocalValue) {
+        this(lm, ledgerId, currentLocalValue, needsTransformation, transform, updateLocalValue, null);
+    }
     /**
      * Construct the loop. This takes a set of functions which may be called multiple times
      * during the loop.
@@ -89,20 +100,21 @@ class MetadataUpdateLoop {
      *                         second parameter and return true, return false otherwise
      */
     MetadataUpdateLoop(LedgerManager lm,
-                       long ledgerId,
-                       Supplier<Versioned<LedgerMetadata>> currentLocalValue,
-                       NeedsUpdatePredicate needsTransformation,
-                       MetadataTransform transform,
-                       LocalValueUpdater updateLocalValue) {
+            long ledgerId,
+            Supplier<Versioned<LedgerMetadata>> currentLocalValue,
+            NeedsUpdatePredicate needsTransformation,
+            MetadataTransform transform,
+            LocalValueUpdater updateLocalValue,
+            RateLimiter throttler) {
         this.lm = lm;
         this.ledgerId = ledgerId;
         this.currentLocalValue = currentLocalValue;
         this.needsTransformation = needsTransformation;
         this.transform = transform;
         this.updateLocalValue = updateLocalValue;
+        this.throttler = throttler;
 
-        this.logContext = String.format("UpdateLoop(ledgerId=%d,loopId=%08x)",
-                                        ledgerId, System.identityHashCode(this));
+        this.logContext = String.format("UpdateLoop(ledgerId=%d,loopId=%08x)", ledgerId, System.identityHashCode(this));
     }
 
     CompletableFuture<Versioned<LedgerMetadata>> run() {
@@ -120,7 +132,10 @@ class MetadataUpdateLoop {
         try {
             if (needsTransformation.needsUpdate(currentLocal.getValue())) {
                 LedgerMetadata transformed = transform.transform(currentLocal.getValue());
-
+                if (throttler != null) {
+                    // throttler to control updates per second
+                    throttler.acquire();
+                }
                 lm.writeLedgerMetadata(ledgerId, transformed, currentLocal.getVersion())
                     .whenComplete((writtenMetadata, ex) -> {
                             if (ex == null) {
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/UpdateLedgerOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/UpdateLedgerOp.java
index 83143fb..14951fd 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/UpdateLedgerOp.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/UpdateLedgerOp.java
@@ -30,6 +30,7 @@ import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Semaphore;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
@@ -75,7 +76,8 @@ public class UpdateLedgerOp {
      *             metadata
      */
     public void updateBookieIdInLedgers(final BookieSocketAddress oldBookieId, final BookieSocketAddress newBookieId,
-                                        final int rate, final int limit, final UpdateLedgerNotifier progressable)
+                                        final int rate, int maxOutstandingReads, final int limit,
+                                        final UpdateLedgerNotifier progressable)
             throws IOException, InterruptedException {
 
         final AtomicInteger issuedLedgerCnt = new AtomicInteger();
@@ -84,13 +86,14 @@ public class UpdateLedgerOp {
         final Set<CompletableFuture<?>> outstanding =
             Collections.newSetFromMap(new ConcurrentHashMap<CompletableFuture<?>, Boolean>());
         final RateLimiter throttler = RateLimiter.create(rate);
+        final Semaphore outstandingReads = new Semaphore(maxOutstandingReads);
         final Iterator<Long> ledgerItr = admin.listLedgers().iterator();
 
         // iterate through all the ledgers
         while (ledgerItr.hasNext() && !finalPromise.isDone()
                && (limit == Integer.MIN_VALUE || issuedLedgerCnt.get() < limit)) {
-            // throttler to control updates per second
-            throttler.acquire();
+            // semaphore to control reads according to update throttling
+            outstandingReads.acquire();
 
             final long ledgerId = ledgerItr.next();
             issuedLedgerCnt.incrementAndGet();
@@ -109,7 +112,7 @@ public class UpdateLedgerOp {
                             (metadata) -> {
                                 return replaceBookieInEnsembles(metadata, oldBookieId, newBookieId);
                             },
-                            ref::compareAndSet).run();
+                            ref::compareAndSet, throttler).run();
                 });
 
             outstanding.add(writePromise);
@@ -127,6 +130,7 @@ public class UpdateLedgerOp {
                             updatedLedgerCnt.incrementAndGet();
                             progressable.progress(updatedLedgerCnt.get(), issuedLedgerCnt.get());
                         }
+                        outstandingReads.release();
                         outstanding.remove(writePromise);
                     });
         }
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/FlipBookieIdCommand.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/FlipBookieIdCommand.java
index 852379f..701d562 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/FlipBookieIdCommand.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/FlipBookieIdCommand.java
@@ -77,6 +77,10 @@ public class FlipBookieIdCommand extends BookieCommand<FlipBookieIdCommand.FlipB
             description = "Number of ledgers updating per second (default: 5 per sec)")
         private int updatePerSec = 5;
 
+        @Parameter(names = { "-r",
+                "--maxOutstandingReads" }, description = "Max outstanding reads (default: 5 * updatespersec)")
+        private int maxOutstandingReads = updatePerSec * 5;
+
         @Parameter(names = {"-l", "--limit"},
             description = "Maximum number of ledgers of ledgers to update (default: no limit)")
         private int limit = Integer.MIN_VALUE;
@@ -115,6 +119,12 @@ public class FlipBookieIdCommand extends BookieCommand<FlipBookieIdCommand.FlipB
             return false;
         }
 
+        final int maxOutstandingReads = flags.maxOutstandingReads;
+        if (maxOutstandingReads <= 0) {
+            LOG.error("Invalid maxOutstandingReads {}, should be > 0", maxOutstandingReads);
+            return false;
+        }
+
         final int limit = flags.limit;
         if (limit <= 0 && limit != Integer.MIN_VALUE) {
             LOG.error("Invalid limit {}, should be > 0", limit);
@@ -154,7 +164,8 @@ public class FlipBookieIdCommand extends BookieCommand<FlipBookieIdCommand.FlipB
         };
 
         try {
-            updateLedgerOp.updateBookieIdInLedgers(oldBookieId, newBookieId, rate, limit, progressable);
+            updateLedgerOp.updateBookieIdInLedgers(oldBookieId, newBookieId, rate, maxOutstandingReads, limit,
+                    progressable);
         } catch (IOException e) {
             LOG.error("Failed to update ledger metadata", e);
             return false;
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/UpdateLedgerOpTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/UpdateLedgerOpTest.java
index c2b0d75..5ead2c9 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/UpdateLedgerOpTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/UpdateLedgerOpTest.java
@@ -108,7 +108,7 @@ public class UpdateLedgerOpTest extends BookKeeperClusterTestCase {
             BookieSocketAddress toBookieAddr = new BookieSocketAddress(curBookieId.getHostName() + ":"
                     + curBookieAddr.getPort());
             UpdateLedgerOp updateLedgerOp = new UpdateLedgerOp(bk, bkadmin);
-            updateLedgerOp.updateBookieIdInLedgers(curBookieAddr, toBookieAddr, 5, Integer.MIN_VALUE, progressable);
+            updateLedgerOp.updateBookieIdInLedgers(curBookieAddr, toBookieAddr, 5, 25, Integer.MIN_VALUE, progressable);
 
             for (LedgerHandle lh : ledgers) {
                 lh.close();
@@ -146,22 +146,22 @@ public class UpdateLedgerOpTest extends BookKeeperClusterTestCase {
             BookieSocketAddress toBookieAddr = new BookieSocketAddress(toBookieId.getHostName() + ":"
                     + curBookieAddr.getPort());
             UpdateLedgerOp updateLedgerOp = new UpdateLedgerOp(bk, bkadmin);
-            updateLedgerOp.updateBookieIdInLedgers(curBookieAddr, toBookieAddr, 7, 4, progressable);
+            updateLedgerOp.updateBookieIdInLedgers(curBookieAddr, toBookieAddr, 7, 35, 4, progressable);
             int updatedLedgersCount = getUpdatedLedgersCount(bk, ledgers, toBookieAddr);
             assertEquals("Failed to update the ledger metadata to use bookie host name", 4, updatedLedgersCount);
 
             // next execution
-            updateLedgerOp.updateBookieIdInLedgers(curBookieAddr, toBookieAddr, 2, 10, progressable);
+            updateLedgerOp.updateBookieIdInLedgers(curBookieAddr, toBookieAddr, 2, 10, 10, progressable);
             updatedLedgersCount = getUpdatedLedgersCount(bk, ledgers, toBookieAddr);
             assertEquals("Failed to update the ledger metadata to use bookie host name", 10, updatedLedgersCount);
 
             // no ledgers
-            updateLedgerOp.updateBookieIdInLedgers(curBookieAddr, toBookieAddr, 3, 20, progressable);
+            updateLedgerOp.updateBookieIdInLedgers(curBookieAddr, toBookieAddr, 3, 15, 20, progressable);
             updatedLedgersCount = getUpdatedLedgersCount(bk, ledgers, toBookieAddr);
             assertEquals("Failed to update the ledger metadata to use bookie host name", 10, updatedLedgersCount);
 
             // no ledgers
-            updateLedgerOp.updateBookieIdInLedgers(curBookieAddr, toBookieAddr, 3, Integer.MIN_VALUE, progressable);
+            updateLedgerOp.updateBookieIdInLedgers(curBookieAddr, toBookieAddr, 3, 15, Integer.MIN_VALUE, progressable);
             updatedLedgersCount = getUpdatedLedgersCount(bk, ledgers, toBookieAddr);
             assertEquals("Failed to update the ledger metadata to use bookie host name", 10, updatedLedgersCount);
         }
@@ -208,7 +208,7 @@ public class UpdateLedgerOpTest extends BookKeeperClusterTestCase {
             BookieSocketAddress toBookieAddr = new BookieSocketAddress(toBookieId.getHostName() + ":"
                     + curBookieAddr.getPort());
             UpdateLedgerOp updateLedgerOp = new UpdateLedgerOp(bk, bkadmin);
-            updateLedgerOp.updateBookieIdInLedgers(curBookieAddr, toBookieAddr, 5, 100, progressable);
+            updateLedgerOp.updateBookieIdInLedgers(curBookieAddr, toBookieAddr, 5, 25, 100, progressable);
 
             bookieServer.shutdown();
 
@@ -278,7 +278,7 @@ public class UpdateLedgerOpTest extends BookKeeperClusterTestCase {
             BookieSocketAddress curBookieAddr = ensemble.get(0);
             BookieSocketAddress toBookieAddr = new BookieSocketAddress("localhost:" + curBookieAddr.getPort());
             UpdateLedgerOp updateLedgerOp = new UpdateLedgerOp(bk, bkadmin);
-            updateLedgerOp.updateBookieIdInLedgers(curBookieAddr, toBookieAddr, 5, 100, progressable);
+            updateLedgerOp.updateBookieIdInLedgers(curBookieAddr, toBookieAddr, 5, 25, 100, progressable);
 
             if (!latch.await(120, TimeUnit.SECONDS)) {
                 throw new Exception("Entries took too long to add");
diff --git a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookie/FlipBookieIdCommandTest.java b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookie/FlipBookieIdCommandTest.java
index a57a758..3be6b3d 100644
--- a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookie/FlipBookieIdCommandTest.java
+++ b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookie/FlipBookieIdCommandTest.java
@@ -101,8 +101,8 @@ public class FlipBookieIdCommandTest extends BookieCommandTestBase {
         verifyNew(UpdateLedgerOp.class, times(1)).withArguments(eq(bookKeeper), eq(bookKeeperAdmin));
         verifyNew(ServerConfiguration.class, times(1)).withArguments(eq(conf));
         verify(serverConfiguration, times(1)).setUseHostNameAsBookieID(anyBoolean());
-        verify(updateLedgerOp, times(1))
-            .updateBookieIdInLedgers(eq(bookieSocketAddress), eq(bookieSocketAddress), anyInt(), anyInt(), any());
+        verify(updateLedgerOp, times(1)).updateBookieIdInLedgers(eq(bookieSocketAddress), eq(bookieSocketAddress),
+                anyInt(), anyInt(), anyInt(), any());
     }
 
 }