You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by mm...@apache.org on 2022/08/24 21:00:55 UTC
[pulsar] branch master updated: [enh][broker] Add metrics for entry cache insertion, eviction (#17248)
This is an automated email from the ASF dual-hosted git repository.
mmarshall pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new e3b25403e4a [enh][broker] Add metrics for entry cache insertion, eviction (#17248)
e3b25403e4a is described below
commit e3b25403e4abb6a0c16073e5fb28a533497d094c
Author: Michael Marshall <mm...@apache.org>
AuthorDate: Wed Aug 24 14:00:45 2022 -0700
[enh][broker] Add metrics for entry cache insertion, eviction (#17248)
Fixes https://github.com/apache/pulsar/issues/16584
### Motivation
With the `RangeCache`, it is hard to reason about its behavior other than cache hits/misses or the cache's size hitting the limit and triggering a size based eviction. This PR adds 3 new metrics to help provide additional insight into the cache's behavior. It adds `pulsar_ml_cache_inserted_entries_total`, `pulsar_ml_cache_evicted_entries_total`, and `pulsar_ml_cache_entries`.
### Modifications
* Add new metrics for cache insertion, eviction, and current number of entries.
* Add new methods to the `ManagedLedgerFactoryMXBean` interface.
* Update several method return values in the `RangeCache`.
* Update tests.
### Verifying this change
This change is covered by modified tests that already existed.
### Does this pull request potentially affect one of the following parts:
There is a breaking change to the `RangeCache` class for the `clear` and the `evictLEntriesBeforeTimestamp` methods. The previous result was a `long`, and now it is a `Pair<Integer, Long>`. The new result matches the same style as `evictLeastAccessedEntries`. Given that this class is only meant for use within the broker, I think it is reasonable to break these methods. I will send a note to the mailing list.
### Documentation
- [x] `doc`
---
.../mledger/ManagedLedgerFactoryMXBean.java | 15 ++++++++++++
.../impl/ManagedLedgerFactoryMBeanImpl.java | 27 ++++++++++++++++++++++
.../mledger/impl/cache/RangeEntryCacheImpl.java | 14 +++++------
.../impl/cache/RangeEntryCacheManagerImpl.java | 4 +++-
.../apache/bookkeeper/mledger/util/RangeCache.java | 12 ++++++----
.../mledger/impl/EntryCacheManagerTest.java | 18 +++++++++++++++
.../bookkeeper/mledger/util/RangeCacheTest.java | 5 ++--
.../stats/metrics/ManagedLedgerCacheMetrics.java | 3 +++
site2/docs/reference-metrics.md | 3 +++
9 files changed, 87 insertions(+), 14 deletions(-)
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryMXBean.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryMXBean.java
index ea5b2074ffa..f71583ab886 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryMXBean.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryMXBean.java
@@ -66,4 +66,19 @@ public interface ManagedLedgerFactoryMXBean {
* Get the number of cache evictions during the last minute.
*/
long getNumberOfCacheEvictions();
+
+ /**
+ * Cumulative number of entries inserted into the cache.
+ */
+ long getCacheInsertedEntriesCount();
+
+ /**
+ * Cumulative number of entries evicted from the cache.
+ */
+ long getCacheEvictedEntriesCount();
+
+ /**
+ * Current number of entries in the cache.
+ */
+ long getCacheEntriesCount();
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java
index d514d8381d9..a5f0c67e68a 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java
@@ -19,6 +19,7 @@
package org.apache.bookkeeper.mledger.impl;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.LongAdder;
import org.apache.bookkeeper.mledger.ManagedLedgerFactoryMXBean;
import org.apache.pulsar.common.stats.Rate;
@@ -31,6 +32,10 @@ public class ManagedLedgerFactoryMBeanImpl implements ManagedLedgerFactoryMXBean
final Rate cacheMisses = new Rate();
final Rate cacheEvictions = new Rate();
+ private final LongAdder insertedEntryCount = new LongAdder();
+ private final LongAdder evictedEntryCount = new LongAdder();
+ private final LongAdder cacheEntryCount = new LongAdder();
+
public ManagedLedgerFactoryMBeanImpl(ManagedLedgerFactoryImpl factory) throws Exception {
this.factory = factory;
}
@@ -64,6 +69,16 @@ public class ManagedLedgerFactoryMBeanImpl implements ManagedLedgerFactoryMXBean
cacheEvictions.recordEvent();
}
+ public void recordCacheInsertion() {
+ insertedEntryCount.increment();
+ cacheEntryCount.increment();
+ }
+
+ public void recordNumberOfCacheEntriesEvicted(int count) {
+ evictedEntryCount.add(count);
+ cacheEntryCount.add(-count);
+ }
+
@Override
public int getNumberOfManagedLedgers() {
return factory.ledgers.size();
@@ -104,4 +119,16 @@ public class ManagedLedgerFactoryMBeanImpl implements ManagedLedgerFactoryMXBean
return cacheEvictions.getCount();
}
+ public long getCacheInsertedEntriesCount() {
+ return insertedEntryCount.sum();
+ }
+
+ public long getCacheEvictedEntriesCount() {
+ return evictedEntryCount.sum();
+ }
+
+ public long getCacheEntriesCount() {
+ return cacheEntryCount.sum();
+ }
+
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java
index 0d29194a883..f8f5c328cd9 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java
@@ -168,7 +168,7 @@ public class RangeEntryCacheImpl implements EntryCache {
lastPosition, entriesRemoved, sizeRemoved);
}
- manager.entriesRemoved(sizeRemoved);
+ manager.entriesRemoved(sizeRemoved, entriesRemoved);
}
@Override
@@ -184,7 +184,7 @@ public class RangeEntryCacheImpl implements EntryCache {
ml.getName(), ledgerId, entriesRemoved, sizeRemoved);
}
- manager.entriesRemoved(sizeRemoved);
+ manager.entriesRemoved(sizeRemoved, entriesRemoved);
}
@Override
@@ -338,8 +338,8 @@ public class RangeEntryCacheImpl implements EntryCache {
@Override
public void clear() {
- long removedSize = entries.clear();
- manager.entriesRemoved(removedSize);
+ Pair<Integer, Long> removedPair = entries.clear();
+ manager.entriesRemoved(removedPair.getRight(), removedPair.getLeft());
}
@Override
@@ -364,14 +364,14 @@ public class RangeEntryCacheImpl implements EntryCache {
+ " -- Current Size: {} Mb",
ml.getName(), sizeToFree / MB, evictedEntries, evictedSize / MB, entries.getSize() / MB);
}
- manager.entriesRemoved(evictedSize);
+ manager.entriesRemoved(evictedSize, evictedEntries);
return evicted;
}
@Override
public void invalidateEntriesBeforeTimestamp(long timestamp) {
- long evictedSize = entries.evictLEntriesBeforeTimestamp(timestamp);
- manager.entriesRemoved(evictedSize);
+ Pair<Integer, Long> evictedPair = entries.evictLEntriesBeforeTimestamp(timestamp);
+ manager.entriesRemoved(evictedPair.getRight(), evictedPair.getLeft());
}
private static final Logger log = LoggerFactory.getLogger(RangeEntryCacheImpl.class);
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java
index e8a463c46d1..657740c39ec 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java
@@ -141,10 +141,12 @@ public class RangeEntryCacheManagerImpl implements EntryCacheManager {
}
void entryAdded(long size) {
+ mlFactoryMBean.recordCacheInsertion();
currentSize.addAndGet(size);
}
- void entriesRemoved(long size) {
+ void entriesRemoved(long size, int count) {
+ mlFactoryMBean.recordNumberOfCacheEntriesEvicted(count);
currentSize.addAndGet(-size);
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java
index c9181e02f2c..1b82aa1318e 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java
@@ -187,8 +187,9 @@ public class RangeCache<Key extends Comparable<Key>, Value extends ReferenceCoun
* @param maxTimestamp the max timestamp of the entries to be evicted
* @return the tota
*/
- public long evictLEntriesBeforeTimestamp(long maxTimestamp) {
+ public Pair<Integer, Long> evictLEntriesBeforeTimestamp(long maxTimestamp) {
long removedSize = 0;
+ int removedCount = 0;
while (true) {
Map.Entry<Key, Value> entry = entries.firstEntry();
@@ -203,11 +204,12 @@ public class RangeCache<Key extends Comparable<Key>, Value extends ReferenceCoun
Value value = entry.getValue();
removedSize += weighter.getSize(value);
+ removedCount++;
value.release();
}
size.addAndGet(-removedSize);
- return removedSize;
+ return Pair.of(removedCount, removedSize);
}
/**
@@ -226,8 +228,9 @@ public class RangeCache<Key extends Comparable<Key>, Value extends ReferenceCoun
*
* @return size of removed entries
*/
- public synchronized long clear() {
+ public synchronized Pair<Integer, Long> clear() {
long removedSize = 0;
+ int removedCount = 0;
while (true) {
Map.Entry<Key, Value> entry = entries.pollFirstEntry();
@@ -236,12 +239,13 @@ public class RangeCache<Key extends Comparable<Key>, Value extends ReferenceCoun
}
Value value = entry.getValue();
removedSize += weighter.getSize(value);
+ removedCount++;
value.release();
}
entries.clear();
size.getAndAdd(-removedSize);
- return removedSize;
+ return Pair.of(removedCount, removedSize);
}
/**
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/EntryCacheManagerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/EntryCacheManagerTest.java
index a1dcb4ea02f..5b34dc3eb59 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/EntryCacheManagerTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/EntryCacheManagerTest.java
@@ -94,6 +94,9 @@ public class EntryCacheManagerTest extends MockedBookKeeperTestCase {
assertEquals(factory2.getMbean().getCacheMissesRate(), 0.0);
assertEquals(factory2.getMbean().getCacheHitsThroughput(), 0.0);
assertEquals(factory2.getMbean().getNumberOfCacheEvictions(), 0);
+ assertEquals(factory2.getMbean().getCacheInsertedEntriesCount(), 2);
+ assertEquals(factory2.getMbean().getCacheEvictedEntriesCount(), 0);
+ assertEquals(factory2.getMbean().getCacheEntriesCount(), 2);
cache2.insert(EntryImpl.create(2, 0, new byte[1]));
cache2.insert(EntryImpl.create(2, 1, new byte[1]));
@@ -129,6 +132,9 @@ public class EntryCacheManagerTest extends MockedBookKeeperTestCase {
assertEquals(factory2.getMbean().getCacheMissesRate(), 0.0);
assertEquals(factory2.getMbean().getCacheHitsThroughput(), 0.0);
assertEquals(factory2.getMbean().getNumberOfCacheEvictions(), 1);
+ assertEquals(factory2.getMbean().getCacheInsertedEntriesCount(), 5);
+ assertEquals(factory2.getMbean().getCacheEntriesCount(), 2);
+ assertEquals(factory2.getMbean().getCacheEvictedEntriesCount(), 3);
}
@Test
@@ -153,6 +159,9 @@ public class EntryCacheManagerTest extends MockedBookKeeperTestCase {
assertEquals(cache1.getSize(), 7);
assertEquals(cacheManager.getSize(), 7);
+ assertEquals(factory2.getMbean().getCacheInsertedEntriesCount(), 2);
+ assertEquals(factory2.getMbean().getCacheEntriesCount(), 2);
+ assertEquals(factory2.getMbean().getCacheEvictedEntriesCount(), 0);
}
@Test
@@ -185,6 +194,9 @@ public class EntryCacheManagerTest extends MockedBookKeeperTestCase {
cacheManager.removeEntryCache(ml1.getName());
assertTrue(cacheManager.getSize() > 0);
+ assertEquals(factory2.getMbean().getCacheInsertedEntriesCount(), 20);
+ assertEquals(factory2.getMbean().getCacheEntriesCount(), 0);
+ assertEquals(factory2.getMbean().getCacheEvictedEntriesCount(), 20);
}
@@ -217,6 +229,9 @@ public class EntryCacheManagerTest extends MockedBookKeeperTestCase {
assertEquals(factory2.getMbean().getCacheMissesRate(), 0.0);
assertEquals(factory2.getMbean().getCacheHitsThroughput(), 0.0);
assertEquals(factory2.getMbean().getNumberOfCacheEvictions(), 0);
+ assertEquals(factory2.getMbean().getCacheInsertedEntriesCount(), 0);
+ assertEquals(factory2.getMbean().getCacheEntriesCount(), 0);
+ assertEquals(factory2.getMbean().getCacheEvictedEntriesCount(), 0);
cache2.insert(EntryImpl.create(2, 0, new byte[1]));
cache2.insert(EntryImpl.create(2, 1, new byte[1]));
@@ -253,6 +268,9 @@ public class EntryCacheManagerTest extends MockedBookKeeperTestCase {
assertEquals(factory2.getMbean().getCacheMissesRate(), 0.0);
assertEquals(factory2.getMbean().getCacheHitsThroughput(), 0.0);
assertEquals(factory2.getMbean().getNumberOfCacheEvictions(), 0);
+ assertEquals(factory2.getMbean().getCacheInsertedEntriesCount(), 0);
+ assertEquals(factory2.getMbean().getCacheEntriesCount(), 0);
+ assertEquals(factory2.getMbean().getCacheEvictedEntriesCount(), 0);
}
@Test
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java
index f31aa4a74f9..341a4928cc7 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java
@@ -139,8 +139,9 @@ public class RangeCacheTest {
assertEquals(cache.getSize(), 10);
assertEquals(cache.getNumberOfEntries(), 4);
- long evictedSize = cache.evictLEntriesBeforeTimestamp(3);
- assertEquals(evictedSize, 6);
+ Pair<Integer, Long> evictedSize = cache.evictLEntriesBeforeTimestamp(3);
+ assertEquals(evictedSize.getRight().longValue(), 6);
+ assertEquals(evictedSize.getLeft().longValue(), 3);
assertEquals(cache.getSize(), 4);
assertEquals(cache.getNumberOfEntries(), 1);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerCacheMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerCacheMetrics.java
index 8e46b4cf254..1f4181f887f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerCacheMetrics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerCacheMetrics.java
@@ -48,6 +48,9 @@ public class ManagedLedgerCacheMetrics extends AbstractMetrics {
m.put("brk_ml_count", mlCacheStats.getNumberOfManagedLedgers());
m.put("brk_ml_cache_used_size", mlCacheStats.getCacheUsedSize());
+ m.put("brk_ml_cache_inserted_entries_total", mlCacheStats.getCacheInsertedEntriesCount());
+ m.put("brk_ml_cache_evicted_entries_total", mlCacheStats.getCacheEvictedEntriesCount());
+ m.put("brk_ml_cache_entries", mlCacheStats.getCacheEntriesCount());
m.put("brk_ml_cache_evictions", mlCacheStats.getNumberOfCacheEvictions());
m.put("brk_ml_cache_hits_rate", mlCacheStats.getCacheHitsRate());
m.put("brk_ml_cache_misses_rate", mlCacheStats.getCacheMissesRate());
diff --git a/site2/docs/reference-metrics.md b/site2/docs/reference-metrics.md
index aa59c02fee8..824f32cafcb 100644
--- a/site2/docs/reference-metrics.md
+++ b/site2/docs/reference-metrics.md
@@ -159,6 +159,9 @@ All the broker metrics are labelled with the following labels:
| Name | Type | Description |
|---|---|---|
| pulsar_ml_cache_evictions | Gauge | The number of cache evictions during the last minute. |
+| pulsar_ml_cache_inserted_entries_total | Counter | The number of entries inserted into the entry cache. |
+| pulsar_ml_cache_evicted_entries_total | Counter | The number of entries evicted from the entry cache. |
+| pulsar_ml_cache_entries | Gauge | The number of entries in the entry cache. |
| pulsar_ml_cache_hits_rate | Gauge | The number of cache hits per second on the broker side. |
| pulsar_ml_cache_hits_throughput | Gauge | The amount of data (byte per second) retrieved from the cache on the broker side. |
| pulsar_ml_cache_misses_rate | Gauge | The number of cache missed per second on the broker side. |