You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by zh...@apache.org on 2018/01/17 15:04:06 UTC
[bookkeeper] branch master updated: LedgerHandle: add counters to
track ledger->bookie counts
This is an automated email from the ASF dual-hosted git repository.
zhaijia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 83f0796 LedgerHandle: add counters to track ledger->bookie counts
83f0796 is described below
commit 83f0796923992ec486d0828c2ba34bb8de7727c6
Author: Samuel Just <sj...@salesforce.com>
AuthorDate: Wed Jan 17 23:03:52 2018 +0800
LedgerHandle: add counters to track ledger->bookie counts
We can use this to detect distribution issues.
(bug W-4188471)
Signed-off-by: Venkateswararao Jujjuri (JV) <vjujjurisalesforce.com>
[Ported to master, cleaned up a bit, added test]
Signed-off-by: Samuel Just <sjustsalesforce.com>
Author: Samuel Just <sj...@salesforce.com>
Reviewers: Enrico Olivelli <eo...@gmail.com>, Jia Zhai <None>
This closes #977 from athanatos/forupstream/stats1/edist
---
.../org/apache/bookkeeper/client/BookKeeper.java | 6 +++---
.../bookkeeper/client/BookKeeperClientStats.java | 1 +
.../apache/bookkeeper/client/LedgerCreateOp.java | 24 ++++++++++++++++++----
.../bookkeeper/client/TestDelayEnsembleChange.java | 8 ++++++++
4 files changed, 32 insertions(+), 7 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java
index f07986d..a58b501 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java
@@ -782,7 +782,7 @@ public class BookKeeper implements org.apache.bookkeeper.client.api.BookKeeper {
}
new LedgerCreateOp(BookKeeper.this, ensSize, writeQuorumSize,
ackQuorumSize, digestType, passwd, cb, ctx,
- customMetadata, EnumSet.noneOf(WriteFlag.class))
+ customMetadata, EnumSet.noneOf(WriteFlag.class), getStatsLogger())
.initiate();
} finally {
closeLock.readLock().unlock();
@@ -986,7 +986,7 @@ public class BookKeeper implements org.apache.bookkeeper.client.api.BookKeeper {
}
new LedgerCreateOp(BookKeeper.this, ensSize, writeQuorumSize,
ackQuorumSize, digestType, passwd, cb, ctx,
- customMetadata, EnumSet.noneOf(WriteFlag.class))
+ customMetadata, EnumSet.noneOf(WriteFlag.class), getStatsLogger())
.initiateAdv(-1L);
} finally {
closeLock.readLock().unlock();
@@ -1097,7 +1097,7 @@ public class BookKeeper implements org.apache.bookkeeper.client.api.BookKeeper {
}
new LedgerCreateOp(BookKeeper.this, ensSize, writeQuorumSize,
ackQuorumSize, digestType, passwd, cb, ctx,
- customMetadata, EnumSet.noneOf(WriteFlag.class))
+ customMetadata, EnumSet.noneOf(WriteFlag.class), getStatsLogger())
.initiateAdv(ledgerId);
} finally {
closeLock.readLock().unlock();
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperClientStats.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperClientStats.java
index 02947a4..8822b93 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperClientStats.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperClientStats.java
@@ -35,6 +35,7 @@ public interface BookKeeperClientStats {
String RECOVER_OP = "LEDGER_RECOVER";
String LEDGER_RECOVER_READ_ENTRIES = "LEDGER_RECOVER_READ_ENTRIES";
String LEDGER_RECOVER_ADD_ENTRIES = "LEDGER_RECOVER_ADD_ENTRIES";
+ String LEDGER_ENSEMBLE_BOOKIE_DISTRIBUTION = "LEDGER_ENSEMBLE_BOOKIE_DISTRIBUTION";
// Data Operations
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java
index a3d5239..aa6cd69 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java
@@ -25,6 +25,7 @@ import java.security.GeneralSecurityException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
@@ -44,6 +45,7 @@ import org.apache.bookkeeper.meta.LedgerIdGenerator;
import org.apache.bookkeeper.net.BookieSocketAddress;
import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.GenericCallback;
import org.apache.bookkeeper.stats.OpStatsLogger;
+import org.apache.bookkeeper.stats.StatsLogger;
import org.apache.bookkeeper.util.MathUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,6 +71,7 @@ class LedgerCreateOp implements GenericCallback<Void> {
final OpStatsLogger createOpLogger;
boolean adv = false;
boolean generateLedgerId = true;
+ private final StatsLogger statsLogger;
/**
* Constructor.
@@ -93,9 +96,11 @@ class LedgerCreateOp implements GenericCallback<Void> {
* A map of user specified custom metadata about the ledger to be persisted; will not try to
* preserve the order(e.g. sortedMap) upon later retireval.
*/
- LedgerCreateOp(BookKeeper bk, int ensembleSize, int writeQuorumSize, int ackQuorumSize, DigestType digestType,
+ LedgerCreateOp(
+ BookKeeper bk, int ensembleSize, int writeQuorumSize, int ackQuorumSize, DigestType digestType,
byte[] passwd, CreateCallback cb, Object ctx, final Map<String, byte[]> customMetadata,
- EnumSet<WriteFlag> writeFlags) {
+ EnumSet<WriteFlag> writeFlags,
+ StatsLogger statsLogger) {
this.bk = bk;
this.metadata = new LedgerMetadata(
ensembleSize,
@@ -112,6 +117,7 @@ class LedgerCreateOp implements GenericCallback<Void> {
this.ctx = ctx;
this.startTime = MathUtils.nowInNano();
this.createOpLogger = bk.getCreateOpLogger();
+ this.statsLogger = statsLogger;
}
/**
@@ -209,6 +215,15 @@ class LedgerCreateOp implements GenericCallback<Void> {
createComplete(BKException.Code.IncorrectParameterException, null);
return;
}
+
+ List<BookieSocketAddress> curEns = lh.getLedgerMetadata().getEnsemble(0L);
+ LOG.info("Ensemble: {} for ledger: {}", curEns, lh.getId());
+
+ for (BookieSocketAddress bsa : curEns) {
+ String ensSpread = BookKeeperClientStats.LEDGER_ENSEMBLE_BOOKIE_DISTRIBUTION + "-" + bsa;
+ statsLogger.getCounter(ensSpread).inc();
+ }
+
// return the ledger handle back
createComplete(BKException.Code.OK, lh);
}
@@ -341,7 +356,7 @@ class LedgerCreateOp implements GenericCallback<Void> {
}
LedgerCreateOp op = new LedgerCreateOp(bk, builderEnsembleSize,
builderWriteQuorumSize, builderAckQuorumSize, DigestType.fromApiDigestType(builderDigestType),
- builderPassword, cb, null, builderCustomMetadata, builderWriteFlags);
+ builderPassword, cb, null, builderCustomMetadata, builderWriteFlags, bk.getStatsLogger());
ReentrantReadWriteLock closeLock = bk.getCloseLock();
closeLock.readLock().lock();
try {
@@ -400,7 +415,8 @@ class LedgerCreateOp implements GenericCallback<Void> {
parent.builderWriteQuorumSize, parent.builderAckQuorumSize,
DigestType.fromApiDigestType(parent.builderDigestType),
parent.builderPassword, cb, null, parent.builderCustomMetadata,
- parent.builderWriteFlags);
+ parent.builderWriteFlags,
+ parent.bk.getStatsLogger());
ReentrantReadWriteLock closeLock = parent.bk.getCloseLock();
closeLock.readLock().lock();
try {
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestDelayEnsembleChange.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestDelayEnsembleChange.java
index b89f800..954df02 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestDelayEnsembleChange.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestDelayEnsembleChange.java
@@ -24,6 +24,7 @@ import static org.apache.bookkeeper.bookie.BookKeeperServerStats.NEW_ENSEMBLE_TI
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.REPLACE_BOOKIE_TIME;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.WATCHER_SCOPE;
import static org.apache.bookkeeper.client.BookKeeperClientStats.CLIENT_SCOPE;
+import static org.apache.bookkeeper.client.BookKeeperClientStats.LEDGER_ENSEMBLE_BOOKIE_DISTRIBUTION;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -189,6 +190,13 @@ public class TestDelayEnsembleChange extends BookKeeperClusterTestCase {
lh.addEntry(data);
}
+ for (BookieSocketAddress addr : lh.getLedgerMetadata().getEnsembles().get(0L)) {
+ assertTrue(
+ LEDGER_ENSEMBLE_BOOKIE_DISTRIBUTION + " should be > 0 for " + addr,
+ bkc.getTestStatsProvider().getCounter(
+ CLIENT_SCOPE + "." + LEDGER_ENSEMBLE_BOOKIE_DISTRIBUTION + "-" + addr)
+ .get() > 0);
+ }
assertTrue(
"Stats should have captured a new ensemble",
bkc.getTestStatsProvider().getOpStatsLogger(
--
To stop receiving notification emails like this one, please contact
['"commits@bookkeeper.apache.org" <co...@bookkeeper.apache.org>'].