You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by si...@apache.org on 2018/12/19 11:47:17 UTC
[bookkeeper] branch master updated: Bring back statslogger to
BookKeeper client in ReplicationWorker process.
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new bd2b16e Bring back statslogger to BookKeeper client in ReplicationWorker process.
bd2b16e is described below
commit bd2b16e880d172d4761461fdbf85c1bd19b24e36
Author: Charan Reddy Guttapalem <re...@gmail.com>
AuthorDate: Wed Dec 19 03:47:12 2018 -0800
Bring back statslogger to BookKeeper client in ReplicationWorker process.
Descriptions of the changes in this PR:
- https://github.com/apache/bookkeeper/commit/2837f6257baf15dc9dd9eb4bcac34596b442be33
had inadvertently removed StatsLogger to BookKeeper client instance in ReplicationWorker process.
So restore StatsLogger in BookKeeper client object.
- Also introduce new config called - 'limitStatsLogging', which would be used to limit statslogging
as and when needed.
- currently this config is used to limit the stats from PCBC. Because if AR process is running in
each Bookie node, then for each AR there will be n number of PCBCs and totally it comes out to
n^2 PCBCs in the cluster. Which is unmanageable from stats collector perspective. So this config
value can be set to true in AR config.
Reviewers: Sijie Guo <si...@apache.org>
This closes #1888 from reddycharan/bringbackmetricsforarprocess
---
.../bookkeeper/conf/AbstractConfiguration.java | 26 ++++++++++++++++++++++
.../apache/bookkeeper/proto/BookieClientImpl.java | 8 +++++--
.../org/apache/bookkeeper/replication/Auditor.java | 9 ++++++--
site/_data/config/bk_server.yaml | 3 +++
4 files changed, 42 insertions(+), 4 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
index 7f00d09..f12200f 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
@@ -153,6 +153,9 @@ public abstract class AbstractConfiguration<T extends AbstractConfiguration>
// enforce minimum number of racks per write quorum
public static final String ENFORCE_MIN_NUM_RACKS_PER_WRITE_QUORUM = "enforceMinNumRacksPerWriteQuorum";
+ // option to limit stats logging
+ public static final String LIMIT_STATS_LOGGING = "limitStatsLogging";
+
protected AbstractConfiguration() {
super();
if (READ_SYSTEM_PROPERTIES) {
@@ -875,6 +878,29 @@ public abstract class AbstractConfiguration<T extends AbstractConfiguration>
setProperty(PRESERVE_MDC_FOR_TASK_EXECUTION, enabled);
return getThis();
}
+
+ /**
+ * Return the flag indicating whether to limit stats logging.
+ *
+ * @return
+ * the boolean flag indicating whether to limit stats logging
+ */
+ public boolean getLimitStatsLogging() {
+ return getBoolean(LIMIT_STATS_LOGGING, false);
+ }
+
+ /**
+ * Sets flag to limit the stats logging.
+ *
+ * @param limitStatsLogging
+ * flag to limit the stats logging.
+ * @return configuration.
+ */
+ public T setLimitStatsLogging(boolean limitStatsLogging) {
+ setProperty(LIMIT_STATS_LOGGING, limitStatsLogging);
+ return getThis();
+ }
+
/**
* Trickery to allow inheritance with fluent style.
*/
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieClientImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieClientImpl.java
index 50dd85f..18f48a2 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieClientImpl.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieClientImpl.java
@@ -171,8 +171,12 @@ public class BookieClientImpl implements BookieClient, PerChannelBookieClientFac
@Override
public PerChannelBookieClient create(BookieSocketAddress address, PerChannelBookieClientPool pcbcPool,
SecurityHandlerFactory shFactory) throws SecurityException {
- return new PerChannelBookieClient(conf, executor, eventLoopGroup, address, statsLogger,
- authProviderFactory, registry, pcbcPool, shFactory);
+ StatsLogger statsLoggerForPCBC = statsLogger;
+ if (conf.getLimitStatsLogging()) {
+ statsLoggerForPCBC = NullStatsLogger.INSTANCE;
+ }
+ return new PerChannelBookieClient(conf, executor, eventLoopGroup, address, statsLoggerForPCBC,
+ authProviderFactory, registry, pcbcPool, shFactory);
}
public PerChannelBookieClientPool lookupClient(BookieSocketAddress addr) {
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java
index 89883b0..870cea3 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java
@@ -71,6 +71,7 @@ import org.apache.bookkeeper.replication.ReplicationException.BKAuditException;
import org.apache.bookkeeper.replication.ReplicationException.CompatibilityException;
import org.apache.bookkeeper.replication.ReplicationException.UnavailableException;
import org.apache.bookkeeper.stats.Counter;
+import org.apache.bookkeeper.stats.NullStatsLogger;
import org.apache.bookkeeper.stats.OpStatsLogger;
import org.apache.bookkeeper.stats.StatsLogger;
import org.apache.bookkeeper.stats.annotations.StatsDoc;
@@ -161,12 +162,16 @@ public class Auditor implements AutoCloseable {
)
private final Counter numDelayedBookieAuditsCancelled;
- static BookKeeper createBookKeeperClient(ServerConfiguration conf)
+ static BookKeeper createBookKeeperClient(ServerConfiguration conf) throws InterruptedException, IOException {
+ return createBookKeeperClient(conf, NullStatsLogger.INSTANCE);
+ }
+
+ static BookKeeper createBookKeeperClient(ServerConfiguration conf, StatsLogger statsLogger)
throws InterruptedException, IOException {
ClientConfiguration clientConfiguration = new ClientConfiguration(conf);
clientConfiguration.setClientRole(ClientConfiguration.CLIENT_ROLE_SYSTEM);
try {
- return BookKeeper.forConfig(clientConfiguration).build();
+ return BookKeeper.forConfig(clientConfiguration).statsLogger(statsLogger).build();
} catch (BKException e) {
throw new IOException("Failed to create bookkeeper client", e);
}
diff --git a/site/_data/config/bk_server.yaml b/site/_data/config/bk_server.yaml
index b60bfbf..a711598 100644
--- a/site/_data/config/bk_server.yaml
+++ b/site/_data/config/bk_server.yaml
@@ -538,6 +538,9 @@ groups:
- Twitter Ostrich : org.apache.bookkeeper.stats.twitter.ostrich.OstrichProvider
- Twitter Science : org.apache.bookkeeper.stats.twitter.science.TwitterStatsProvider
default: org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider
+ - param: limitStatsLogging
+ description: option to limit stats logging
+ default: 'false'
- name: Prometheus Metrics Provider Settings
params: