You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by si...@apache.org on 2018/12/13 14:41:40 UTC
[bookkeeper] branch master updated: [STATS] [DOC] Add @StatsDoc
annotation for memtable stats
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 6b7c2ef [STATS] [DOC] Add @StatsDoc annotation for memtable stats
6b7c2ef is described below
commit 6b7c2efc249783033965c6d79e0a5180a4083bef
Author: Sijie Guo <gu...@gmail.com>
AuthorDate: Thu Dec 13 22:41:34 2018 +0800
[STATS] [DOC] Add @StatsDoc annotation for memtable stats
Descriptions of the changes in this PR:
*Motivation*
As part of [BP-36](https://github.com/apache/bookkeeper/issues/1785), this PR is to document the memtable stats.
*Changes*
- convert memtable stats to use StatsDoc for documenting metrics
Master Issue: #1785
Reviewers: Jia Zhai <None>, Enrico Olivelli <eo...@gmail.com>
This closes #1873 from sijie/memtable_stats
---
.../apache/bookkeeper/bookie/EntryMemTable.java | 56 ++++++-------
.../bookie/EntryMemTableWithParallelFlusher.java | 2 +-
.../bookie/stats/EntryMemTableStats.java | 91 ++++++++++++++++++++++
3 files changed, 115 insertions(+), 34 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryMemTable.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryMemTable.java
index 0a95fe9..a27e54e 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryMemTable.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryMemTable.java
@@ -19,13 +19,6 @@
package org.apache.bookkeeper.bookie;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.SKIP_LIST_FLUSH_BYTES;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.SKIP_LIST_GET_ENTRY;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.SKIP_LIST_PUT_ENTRY;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.SKIP_LIST_SNAPSHOT;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.SKIP_LIST_THROTTLING;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.SKIP_LIST_THROTTLING_LATENCY;
-
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.concurrent.ConcurrentSkipListMap;
@@ -36,9 +29,8 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.bookkeeper.bookie.Bookie.NoLedgerException;
import org.apache.bookkeeper.bookie.CheckpointSource.Checkpoint;
+import org.apache.bookkeeper.bookie.stats.EntryMemTableStats;
import org.apache.bookkeeper.conf.ServerConfiguration;
-import org.apache.bookkeeper.stats.Counter;
-import org.apache.bookkeeper.stats.OpStatsLogger;
import org.apache.bookkeeper.stats.StatsLogger;
import org.apache.bookkeeper.util.MathUtils;
import org.slf4j.Logger;
@@ -117,12 +109,7 @@ public class EntryMemTable implements AutoCloseable{
}
// Stats
- private final OpStatsLogger snapshotStats;
- private final OpStatsLogger putEntryStats;
- private final OpStatsLogger getEntryStats;
- final Counter flushBytesCounter;
- private final Counter throttlingCounter;
- private final OpStatsLogger throttlingStats;
+ protected final EntryMemTableStats memTableStats;
/**
* Constructor.
@@ -150,12 +137,7 @@ public class EntryMemTable implements AutoCloseable{
this.skipListSemaphore = new Semaphore((int) skipListSizeLimit * 2);
// Stats
- this.snapshotStats = statsLogger.getOpStatsLogger(SKIP_LIST_SNAPSHOT);
- this.putEntryStats = statsLogger.getOpStatsLogger(SKIP_LIST_PUT_ENTRY);
- this.getEntryStats = statsLogger.getOpStatsLogger(SKIP_LIST_GET_ENTRY);
- this.flushBytesCounter = statsLogger.getCounter(SKIP_LIST_FLUSH_BYTES);
- this.throttlingCounter = statsLogger.getCounter(SKIP_LIST_THROTTLING);
- this.throttlingStats = statsLogger.getOpStatsLogger(SKIP_LIST_THROTTLING_LATENCY);
+ this.memTableStats = new EntryMemTableStats(statsLogger);
}
void dump() {
@@ -203,9 +185,11 @@ public class EntryMemTable implements AutoCloseable{
}
if (null != cp) {
- snapshotStats.registerSuccessfulEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
+ memTableStats.getSnapshotStats()
+ .registerSuccessfulEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
} else {
- snapshotStats.registerFailedEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
+ memTableStats.getSnapshotStats()
+ .registerFailedEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
}
}
return cp;
@@ -272,7 +256,7 @@ public class EntryMemTable implements AutoCloseable{
}
}
}
- flushBytesCounter.add(size);
+ memTableStats.getFlushBytesCounter().add(size);
clearSnapshot(keyValues);
}
}
@@ -322,11 +306,11 @@ public class EntryMemTable implements AutoCloseable{
final int len = entry.remaining();
if (!skipListSemaphore.tryAcquire(len)) {
- throttlingCounter.inc();
+ memTableStats.getThrottlingCounter().inc();
final long throttlingStartTimeNanos = MathUtils.nowInNano();
skipListSemaphore.acquireUninterruptibly(len);
- throttlingStats.registerSuccessfulEvent(MathUtils.elapsedNanos(throttlingStartTimeNanos),
- TimeUnit.NANOSECONDS);
+ memTableStats.getThrottlingStats()
+ .registerSuccessfulEvent(MathUtils.elapsedNanos(throttlingStartTimeNanos), TimeUnit.NANOSECONDS);
}
this.lock.readLock().lock();
@@ -340,9 +324,11 @@ public class EntryMemTable implements AutoCloseable{
return size;
} finally {
if (success) {
- putEntryStats.registerSuccessfulEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
+ memTableStats.getPutEntryStats()
+ .registerSuccessfulEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
} else {
- putEntryStats.registerFailedEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
+ memTableStats.getPutEntryStats()
+ .registerFailedEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
}
}
}
@@ -406,9 +392,11 @@ public class EntryMemTable implements AutoCloseable{
} finally {
this.lock.readLock().unlock();
if (success) {
- getEntryStats.registerSuccessfulEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
+ memTableStats.getGetEntryStats()
+ .registerSuccessfulEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
} else {
- getEntryStats.registerFailedEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
+ memTableStats.getGetEntryStats()
+ .registerFailedEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
}
}
@@ -435,9 +423,11 @@ public class EntryMemTable implements AutoCloseable{
} finally {
this.lock.readLock().unlock();
if (success) {
- getEntryStats.registerSuccessfulEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
+ memTableStats.getGetEntryStats()
+ .registerSuccessfulEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
} else {
- getEntryStats.registerFailedEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
+ memTableStats.getGetEntryStats()
+ .registerFailedEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
}
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryMemTableWithParallelFlusher.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryMemTableWithParallelFlusher.java
index 4f2cf02..d092f20 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryMemTableWithParallelFlusher.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryMemTableWithParallelFlusher.java
@@ -139,7 +139,7 @@ class EntryMemTableWithParallelFlusher extends EntryMemTable {
throw new IOException("Failed to complete the flushSnapshotByParallelizing",
exceptionWhileFlushingParallelly.get());
}
- flushBytesCounter.add(flushedSize.get());
+ memTableStats.getFlushBytesCounter().add(flushedSize.get());
clearSnapshot(keyValues);
}
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/EntryMemTableStats.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/EntryMemTableStats.java
new file mode 100644
index 0000000..62840f9
--- /dev/null
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/EntryMemTableStats.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bookkeeper.bookie.stats;
+
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_ADD_ENTRY;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_SCOPE;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.CATEGORY_SERVER;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.SKIP_LIST_FLUSH_BYTES;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.SKIP_LIST_GET_ENTRY;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.SKIP_LIST_PUT_ENTRY;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.SKIP_LIST_SNAPSHOT;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.SKIP_LIST_THROTTLING;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.SKIP_LIST_THROTTLING_LATENCY;
+
+import lombok.Getter;
+import org.apache.bookkeeper.stats.Counter;
+import org.apache.bookkeeper.stats.OpStatsLogger;
+import org.apache.bookkeeper.stats.StatsLogger;
+import org.apache.bookkeeper.stats.annotations.StatsDoc;
+
+/**
+ * A umbrella class for memtable related stats.
+ */
+@StatsDoc(
+ name = BOOKIE_SCOPE,
+ category = CATEGORY_SERVER,
+ help = "EntryMemTable related stats"
+)
+@Getter
+public class EntryMemTableStats {
+
+ @StatsDoc(
+ name = SKIP_LIST_SNAPSHOT,
+ help = "operation stats of taking memtable snapshots"
+ )
+ private final OpStatsLogger snapshotStats;
+ @StatsDoc(
+ name = SKIP_LIST_PUT_ENTRY,
+ help = "operation stats of putting entries to memtable",
+ parent = BOOKIE_ADD_ENTRY
+ )
+ private final OpStatsLogger putEntryStats;
+ @StatsDoc(
+ name = SKIP_LIST_GET_ENTRY,
+ help = "operation stats of getting entries from memtable",
+ parent = BOOKIE_ADD_ENTRY
+ )
+ private final OpStatsLogger getEntryStats;
+ @StatsDoc(
+ name = SKIP_LIST_FLUSH_BYTES,
+ help = "The number of bytes flushed from memtable to entry log files"
+ )
+ private final Counter flushBytesCounter;
+ @StatsDoc(
+ name = SKIP_LIST_THROTTLING,
+ help = "The number of requests throttled due to memtables are full"
+ )
+ private final Counter throttlingCounter;
+ @StatsDoc(
+ name = SKIP_LIST_THROTTLING_LATENCY,
+ help = "The distribution of request throttled duration"
+ )
+ private final OpStatsLogger throttlingStats;
+
+ public EntryMemTableStats(StatsLogger statsLogger) {
+ this.snapshotStats = statsLogger.getOpStatsLogger(SKIP_LIST_SNAPSHOT);
+ this.putEntryStats = statsLogger.getOpStatsLogger(SKIP_LIST_PUT_ENTRY);
+ this.getEntryStats = statsLogger.getOpStatsLogger(SKIP_LIST_GET_ENTRY);
+ this.flushBytesCounter = statsLogger.getCounter(SKIP_LIST_FLUSH_BYTES);
+ this.throttlingCounter = statsLogger.getCounter(SKIP_LIST_THROTTLING);
+ this.throttlingStats = statsLogger.getOpStatsLogger(SKIP_LIST_THROTTLING_LATENCY);
+ }
+
+}