You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by md...@apache.org on 2022/09/09 15:49:31 UTC
[solr] branch branch_9x updated: SOLR-16357 CaffeineCache ramBytes incorrect accounting on updates and deletes (#985)
This is an automated email from the ASF dual-hosted git repository.
mdrob pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new ebc823a5991 SOLR-16357 CaffeineCache ramBytes incorrect accounting on updates and deletes (#985)
ebc823a5991 is described below
commit ebc823a599160914f7f0f2694f64a53d9cd20268
Author: Alex <st...@users.noreply.github.com>
AuthorDate: Fri Sep 9 08:16:35 2022 -0700
SOLR-16357 CaffeineCache ramBytes incorrect accounting on updates and deletes (#985)
(cherry picked from commit 7904f367cef85e0da4c14e872ac292055a22dc72)
---
solr/CHANGES.txt | 2 +
.../java/org/apache/solr/search/CaffeineCache.java | 24 +++-----
.../org/apache/solr/search/TestCaffeineCache.java | 67 ++++++++++++++++++++++
3 files changed, 77 insertions(+), 16 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index d713c36fc95..6225c153eea 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -110,6 +110,8 @@ Bug Fixes
* SOLR-16399: ExportWriter fails with max values for fields (Kevin Risden)
+* SOLR-16357: CaffeineCache ramBytes incorrect accounting on updates and deletes (Alex Deparvu)
+
Other Changes
---------------------
* SOLR-16351: Upgrade Carrot2 to 4.4.3, upgrade randomizedtesting to 2.8.0. (Dawid Weiss)
diff --git a/solr/core/src/java/org/apache/solr/search/CaffeineCache.java b/solr/core/src/java/org/apache/solr/search/CaffeineCache.java
index 6072bad0abe..e1263a6ed49 100644
--- a/solr/core/src/java/org/apache/solr/search/CaffeineCache.java
+++ b/solr/core/src/java/org/apache/solr/search/CaffeineCache.java
@@ -33,10 +33,8 @@ import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
-import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ForkJoinPool;
@@ -101,7 +99,6 @@ public class CaffeineCache<K, V> extends SolrCacheBase
private boolean cleanupThread;
private boolean async;
- private Set<String> metricNames = ConcurrentHashMap.newKeySet();
private MetricsMap cacheMap;
private SolrMetricsContext solrMetricsContext;
@@ -191,6 +188,9 @@ public class CaffeineCache<K, V> extends SolrCacheBase
-(RamUsageEstimator.sizeOfObject(key, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED)
+ RamUsageEstimator.sizeOfObject(value, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED)
+ RamUsageEstimator.LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY));
+ if (async) {
+ ramBytes.add(-RAM_BYTES_PER_FUTURE);
+ }
}
@Override
@@ -289,10 +289,10 @@ public class CaffeineCache<K, V> extends SolrCacheBase
*/
private void recordRamBytes(K key, V oldValue, V newValue) {
ramBytes.add(
- RamUsageEstimator.sizeOfObject(key, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED)
- + RamUsageEstimator.sizeOfObject(
- newValue, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED));
+ RamUsageEstimator.sizeOfObject(newValue, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED));
if (oldValue == null) {
+ ramBytes.add(
+ RamUsageEstimator.sizeOfObject(key, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED));
ramBytes.add(RamUsageEstimator.LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY);
if (async) ramBytes.add(RAM_BYTES_PER_FUTURE);
} else {
@@ -304,16 +304,8 @@ public class CaffeineCache<K, V> extends SolrCacheBase
@Override
public V remove(K key) {
- V existing = cache.asMap().remove(key);
- if (existing != null) {
- ramBytes.add(
- -RamUsageEstimator.sizeOfObject(key, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED));
- ramBytes.add(
- -RamUsageEstimator.sizeOfObject(
- existing, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED));
- ramBytes.add(-RamUsageEstimator.LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY);
- }
- return existing;
+ // ramBytes adjustment happens via #onRemoval
+ return cache.asMap().remove(key);
}
@Override
diff --git a/solr/core/src/test/org/apache/solr/search/TestCaffeineCache.java b/solr/core/src/test/org/apache/solr/search/TestCaffeineCache.java
index c70b0620ff0..30758369373 100644
--- a/solr/core/src/test/org/apache/solr/search/TestCaffeineCache.java
+++ b/solr/core/src/test/org/apache/solr/search/TestCaffeineCache.java
@@ -19,6 +19,7 @@ package org.apache.solr.search;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.RemovalCause;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -288,4 +289,70 @@ public class TestCaffeineCache extends SolrTestCase {
assertTrue("total ram bytes exceeded limit", total < 1024 * 1024);
cache.close();
}
+
+ @Test
+ public void testRamBytesSync() throws IOException {
+ CaffeineCache<Integer, String> cache = new CaffeineCache<>();
+ Map<String, String> params =
+ Map.of(
+ SolrCache.SIZE_PARAM, "100",
+ SolrCache.INITIAL_SIZE_PARAM, "10",
+ SolrCache.ASYNC_PARAM, Boolean.FALSE.toString());
+ cache.init(params, null, new NoOpRegenerator());
+
+ long emptySize = cache.ramBytesUsed();
+
+ // noop rm
+ cache.remove(0);
+ assertEquals(emptySize, cache.ramBytesUsed());
+
+ cache.put(0, "test");
+ long nonEmptySize = cache.ramBytesUsed();
+ cache.put(0, "test");
+ assertEquals(nonEmptySize, cache.ramBytesUsed());
+
+ cache.remove(0);
+ assertEquals(emptySize, cache.ramBytesUsed());
+
+ cache.put(1, "test2");
+ cache.clear();
+ assertEquals(emptySize, cache.ramBytesUsed());
+
+ cache.put(1, "test3");
+ cache.close();
+ assertEquals(emptySize, cache.ramBytesUsed());
+ }
+
+ @Test
+ public void testRamBytesAsync() throws IOException {
+ CaffeineCache<Integer, String> cache = new CaffeineCache<>();
+ Map<String, String> params =
+ Map.of(
+ SolrCache.SIZE_PARAM, "100",
+ SolrCache.INITIAL_SIZE_PARAM, "10",
+ SolrCache.ASYNC_PARAM, Boolean.TRUE.toString());
+ cache.init(params, null, new NoOpRegenerator());
+
+ long emptySize = cache.ramBytesUsed();
+
+ // noop rm
+ cache.remove(0);
+ assertEquals(emptySize, cache.ramBytesUsed());
+
+ cache.put(0, "test");
+ long nonEmptySize = cache.ramBytesUsed();
+ cache.put(0, "test");
+ assertEquals(nonEmptySize, cache.ramBytesUsed());
+
+ cache.remove(0);
+ assertEquals(emptySize, cache.ramBytesUsed());
+
+ cache.put(1, "test2");
+ cache.clear();
+ assertEquals(emptySize, cache.ramBytesUsed());
+
+ cache.put(1, "test3");
+ cache.close();
+ assertEquals(emptySize, cache.ramBytesUsed());
+ }
}