You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2015/04/11 06:23:32 UTC
svn commit: r1672811 - in /lucene/dev/trunk/solr: ./
core/src/java/org/apache/solr/search/ core/src/test/org/apache/solr/search/
server/solr/configsets/basic_configs/conf/
server/solr/configsets/data_driven_schema_configs/conf/
server/solr/configsets/s...
Author: shalin
Date: Sat Apr 11 04:23:31 2015
New Revision: 1672811
URL: http://svn.apache.org/r1672811
Log:
SOLR-7372: Limit memory consumed by LRUCache with a new 'maxRamMB' config parameter
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/LRUCache.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestLRUCache.java
lucene/dev/trunk/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml
lucene/dev/trunk/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml
lucene/dev/trunk/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1672811&r1=1672810&r2=1672811&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Sat Apr 11 04:23:31 2015
@@ -90,6 +90,9 @@ New Features
* SOLR-7241, SOLR-7263, SOLR-7279: More functionality moving the Admin UI to Angular JS
(Upayavira via Erick)
+* SOLR-7372: Limit memory consumed by LRUCache with a new 'maxRamMB' config parameter.
+ (yonik, shalin)
+
Bug Fixes
----------------------
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/LRUCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/LRUCache.java?rev=1672811&r1=1672810&r2=1672811&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/LRUCache.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/LRUCache.java Sat Apr 11 04:23:31 2015
@@ -17,21 +17,43 @@
package org.apache.solr.search;
+import java.util.Collection;
+import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.util.NumberUtils;
/**
*
*/
-public class LRUCache<K,V> extends SolrCacheBase implements SolrCache<K,V> {
+public class LRUCache<K,V> extends SolrCacheBase implements SolrCache<K,V>, Accountable {
+
+ static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(LRUCache.class);
+
+ /// Copied from Lucene's LRUQueryCache
+
+ // memory usage of a simple term query
+ static final long DEFAULT_RAM_BYTES_USED = 192;
+
+ static final long HASHTABLE_RAM_BYTES_PER_ENTRY =
+ 2 * RamUsageEstimator.NUM_BYTES_OBJECT_REF // key + value
+ * 2; // hash tables need to be oversized to avoid collisions, assume 2x capacity
+
+ static final long LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY =
+ HASHTABLE_RAM_BYTES_PER_ENTRY
+ + 2 * RamUsageEstimator.NUM_BYTES_OBJECT_REF; // previous & next references
+ /// End copied code
/* An instance of this class will be shared across multiple instances
* of an LRUCache at the same time. Make sure everything is thread safe.
@@ -41,6 +63,7 @@ public class LRUCache<K,V> extends SolrC
AtomicLong hits = new AtomicLong();
AtomicLong inserts = new AtomicLong();
AtomicLong evictions = new AtomicLong();
+ AtomicLong evictionsRamUsage = new AtomicLong();
}
private CumulativeStats stats;
@@ -51,12 +74,18 @@ public class LRUCache<K,V> extends SolrC
private long hits;
private long inserts;
private long evictions;
+ private long evictionsRamUsage;
private long warmupTime = 0;
private Map<K,V> map;
private String description="LRU Cache";
+ private long maxRamBytes = Long.MAX_VALUE;
+ // The synchronization used for the map will be used to update this,
+ // hence not an AtomicLong
+ private long ramBytesUsed = 0;
+
@Override
public Object init(Map args, Object persistence, CacheRegenerator regenerator) {
super.init(args, regenerator);
@@ -64,18 +93,49 @@ public class LRUCache<K,V> extends SolrC
final int limit = str==null ? 1024 : Integer.parseInt(str);
str = (String)args.get("initialSize");
final int initialSize = Math.min(str==null ? 1024 : Integer.parseInt(str), limit);
+ str = (String) args.get("maxRamMB");
+ final long maxRamBytes = this.maxRamBytes = str == null ? Long.MAX_VALUE : (long) (Double.parseDouble(str) * 1024L * 1024L);
description = generateDescription(limit, initialSize);
map = new LinkedHashMap<K,V>(initialSize, 0.75f, true) {
@Override
protected boolean removeEldestEntry(Map.Entry eldest) {
- if (size() > limit) {
- // increment evictions regardless of state.
- // this doesn't need to be synchronized because it will
- // only be called in the context of a higher level synchronized block.
- evictions++;
- stats.evictions.incrementAndGet();
- return true;
+ if (size() > limit || ramBytesUsed > maxRamBytes) {
+ if (maxRamBytes != Long.MAX_VALUE && ramBytesUsed > maxRamBytes) {
+ long bytesToDecrement = 0;
+
+ Iterator<Map.Entry<K, V>> iterator = entrySet().iterator();
+ do {
+ Map.Entry<K, V> entry = iterator.next();
+ if (entry.getKey() != null) {
+ if (entry.getKey() instanceof Accountable) {
+ bytesToDecrement += ((Accountable) entry.getKey()).ramBytesUsed();
+ } else {
+ bytesToDecrement += DEFAULT_RAM_BYTES_USED;
+ }
+ }
+ if (entry.getValue() != null) {
+ bytesToDecrement += ((Accountable) entry.getValue()).ramBytesUsed();
+ }
+ bytesToDecrement += LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY;
+ ramBytesUsed -= bytesToDecrement;
+ iterator.remove();
+ evictions++;
+ evictionsRamUsage++;
+ stats.evictions.incrementAndGet();
+ stats.evictionsRamUsage.incrementAndGet();
+ } while (iterator.hasNext() && ramBytesUsed > maxRamBytes);
+ // must return false according to javadocs of removeEldestEntry if we're modifying
+ // the map ourselves
+ return false;
+ } else {
+ // increment evictions regardless of state.
+ // this doesn't need to be synchronized because it will
+ // only be called in the context of a higher level synchronized block.
+ evictions++;
+ stats.evictions.incrementAndGet();
+ return true;
+ }
}
return false;
}
@@ -100,6 +160,9 @@ public class LRUCache<K,V> extends SolrC
if (isAutowarmingOn()) {
description += ", " + getAutowarmDescription();
}
+ if (maxRamBytes != Long.MAX_VALUE) {
+ description += ", maxRamMB=" + (maxRamBytes / 1024L / 1024L);
+ }
description += ')';
return description;
}
@@ -121,7 +184,42 @@ public class LRUCache<K,V> extends SolrC
// increment local inserts regardless of state???
// it does make it more consistent with the current size...
inserts++;
- return map.put(key,value);
+
+ // important to calc and add new ram bytes first so that removeEldestEntry can compare correctly
+ long keySize = DEFAULT_RAM_BYTES_USED;
+ if (maxRamBytes != Long.MAX_VALUE) {
+ if (key != null && key instanceof Accountable) {
+ keySize = ((Accountable) key).ramBytesUsed();
+ }
+ long valueSize = 0;
+ if (value != null) {
+ if (value instanceof Accountable) {
+ Accountable accountable = (Accountable) value;
+ valueSize = accountable.ramBytesUsed();
+ } else {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Cache: "
+ + getName() + " is configured with maxRamBytes=" + RamUsageEstimator.humanReadableUnits(maxRamBytes)
+ + " but its values do not implement org.apache.lucene.util.Accountable");
+ }
+ }
+ ramBytesUsed += keySize + valueSize + LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY;
+ }
+ V old = map.put(key, value);
+ if (maxRamBytes != Long.MAX_VALUE && old != null) {
+ long bytesToDecrement = ((Accountable) old).ramBytesUsed();
+ // the key existed in the map but we added its size before the put, so let's back out
+ bytesToDecrement += LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY;
+ if (key != null) {
+ if (key instanceof Accountable) {
+ Accountable aKey = (Accountable) key;
+ bytesToDecrement += aKey.ramBytesUsed();
+ } else {
+ bytesToDecrement += DEFAULT_RAM_BYTES_USED;
+ }
+ }
+ ramBytesUsed -= bytesToDecrement;
+ }
+ return old;
}
}
@@ -146,6 +244,7 @@ public class LRUCache<K,V> extends SolrC
public void clear() {
synchronized(map) {
map.clear();
+ ramBytesUsed = 0;
}
}
@@ -232,6 +331,11 @@ public class LRUCache<K,V> extends SolrC
lst.add("inserts", inserts);
lst.add("evictions", evictions);
lst.add("size", map.size());
+ if (maxRamBytes != Long.MAX_VALUE) {
+ lst.add("maxRamMB", maxRamBytes / 1024L / 1024L);
+ lst.add("ramBytesUsed", ramBytesUsed());
+ lst.add("evictionsRamUsage", evictionsRamUsage);
+ }
}
lst.add("warmupTime", warmupTime);
@@ -239,9 +343,12 @@ public class LRUCache<K,V> extends SolrC
long chits = stats.hits.get();
lst.add("cumulative_lookups", clookups);
lst.add("cumulative_hits", chits);
- lst.add("cumulative_hitratio", calcHitRatio(clookups,chits));
+ lst.add("cumulative_hitratio", calcHitRatio(clookups, chits));
lst.add("cumulative_inserts", stats.inserts.get());
lst.add("cumulative_evictions", stats.evictions.get());
+ if (maxRamBytes != Long.MAX_VALUE) {
+ lst.add("cumulative_evictionsRamUsage", stats.evictionsRamUsage.get());
+ }
return lst;
}
@@ -250,4 +357,22 @@ public class LRUCache<K,V> extends SolrC
public String toString() {
return name() + getStatistics().toString();
}
+
+ @Override
+ public long ramBytesUsed() {
+ synchronized (map) {
+ return BASE_RAM_BYTES_USED + ramBytesUsed;
+ }
+ }
+
+ @Override
+ public Collection<Accountable> getChildResources() {
+ if (maxRamBytes != Long.MAX_VALUE) {
+ synchronized (map) {
+ return Accountables.namedAccountables(getName(), (Map<?, ? extends Accountable>) map);
+ }
+ } else {
+ return Collections.emptyList();
+ }
+ }
}
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestLRUCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestLRUCache.java?rev=1672811&r1=1672810&r2=1672811&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestLRUCache.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestLRUCache.java Sat Apr 11 04:23:31 2015
@@ -22,7 +22,15 @@ import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.Parser;
+import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
/**
@@ -121,4 +129,64 @@ public class TestLRUCache extends Lucene
assertEquals(null, lruCacheNew.get(50));
lruCacheNew.close();
}
+
+ public void testMaxRamSize() throws Exception {
+ LRUCache<String, Accountable> accountableLRUCache = new LRUCache<>();
+ Map<String, String> params = new HashMap<>();
+ params.put("size", "5");
+ params.put("maxRamMB", "1");
+ CacheRegenerator cr = new NoOpRegenerator();
+ Object o = accountableLRUCache.init(params, null, cr);
+ long baseSize = accountableLRUCache.ramBytesUsed();
+ assertEquals(LRUCache.BASE_RAM_BYTES_USED, baseSize);
+ accountableLRUCache.put("1", new Accountable() {
+ @Override
+ public long ramBytesUsed() {
+ return 512 * 1024;
+ }
+ });
+ assertEquals(1, accountableLRUCache.size());
+ assertEquals(baseSize + 512 * 1024 + LRUCache.DEFAULT_RAM_BYTES_USED + LRUCache.LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY, accountableLRUCache.ramBytesUsed());
+ accountableLRUCache.put("2", new Accountable() {
+ @Override
+ public long ramBytesUsed() {
+ return 512 * 1024;
+ }
+ });
+ assertEquals(1, accountableLRUCache.size());
+ assertEquals(baseSize + 512 * 1024 + LRUCache.LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY + LRUCache.DEFAULT_RAM_BYTES_USED, accountableLRUCache.ramBytesUsed());
+ NamedList<Serializable> nl = accountableLRUCache.getStatistics();
+ assertEquals(1L, nl.get("evictions"));
+ assertEquals(1L, nl.get("evictionsRamUsage"));
+ accountableLRUCache.put("3", new Accountable() {
+ @Override
+ public long ramBytesUsed() {
+ return 1024;
+ }
+ });
+ nl = accountableLRUCache.getStatistics();
+ assertEquals(1L, nl.get("evictions"));
+ assertEquals(1L, nl.get("evictionsRamUsage"));
+ assertEquals(2L, accountableLRUCache.size());
+ assertEquals(baseSize + 513 * 1024 + LRUCache.LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY * 2 + LRUCache.DEFAULT_RAM_BYTES_USED * 2, accountableLRUCache.ramBytesUsed());
+
+ accountableLRUCache.clear();
+ assertEquals(RamUsageEstimator.shallowSizeOfInstance(LRUCache.class), accountableLRUCache.ramBytesUsed());
+ }
+
+ public void testNonAccountableValues() throws Exception {
+ LRUCache<String, String> cache = new LRUCache<>();
+ Map<String, String> params = new HashMap<>();
+ params.put("size", "5");
+ params.put("maxRamMB", "1");
+ CacheRegenerator cr = new NoOpRegenerator();
+ Object o = cache.init(params, null, cr);
+
+ try {
+ cache.put("1", "1");
+ fail("Adding a non-accountable value to a cache configured with maxRamBytes should have failed");
+ } catch (Exception e) {
+ assertEquals(e.getClass(), SolrException.class);
+ }
+ }
}
Modified: lucene/dev/trunk/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml?rev=1672811&r1=1672810&r2=1672811&view=diff
==============================================================================
--- lucene/dev/trunk/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml Sat Apr 11 04:23:31 2015
@@ -246,10 +246,13 @@
autowarmCount="0"/>
<!-- Query Result Cache
-
- Caches results of searches - ordered lists of document ids
- (DocList) based on a query, a sort, and the range of documents requested.
- -->
+
+ Caches results of searches - ordered lists of document ids
+ (DocList) based on a query, a sort, and the range of documents requested.
+ Additional supported parameter by LRUCache:
+ maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
+ to occupy
+ -->
<queryResultCache class="solr.LRUCache"
size="512"
initialSize="512"
Modified: lucene/dev/trunk/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml?rev=1672811&r1=1672810&r2=1672811&view=diff
==============================================================================
--- lucene/dev/trunk/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml Sat Apr 11 04:23:31 2015
@@ -477,7 +477,7 @@
initialSize - the initial capacity (number of entries) of
the cache. (see java.util.HashMap)
autowarmCount - the number of entries to prepopulate from
- and old cache.
+ and old cache.
-->
<filterCache class="solr.FastLRUCache"
size="512"
@@ -487,7 +487,10 @@
<!-- Query Result Cache
Caches results of searches - ordered lists of document ids
- (DocList) based on a query, a sort, and the range of documents requested.
+ (DocList) based on a query, a sort, and the range of documents requested.
+ Additional supported parameter by LRUCache:
+ maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
+ to occupy
-->
<queryResultCache class="solr.LRUCache"
size="512"
@@ -498,7 +501,7 @@
Caches Lucene Document objects (the stored fields for each
document). Since Lucene internal document ids are transient,
- this cache will not be autowarmed.
+ this cache will not be autowarmed.
-->
<documentCache class="solr.LRUCache"
size="512"
Modified: lucene/dev/trunk/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml?rev=1672811&r1=1672810&r2=1672811&view=diff
==============================================================================
--- lucene/dev/trunk/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml Sat Apr 11 04:23:31 2015
@@ -498,10 +498,13 @@
autowarmCount="0"/>
<!-- Query Result Cache
-
- Caches results of searches - ordered lists of document ids
- (DocList) based on a query, a sort, and the range of documents requested.
- -->
+
+ Caches results of searches - ordered lists of document ids
+ (DocList) based on a query, a sort, and the range of documents requested.
+ Additional supported parameter by LRUCache:
+ maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
+ to occupy
+ -->
<queryResultCache class="solr.LRUCache"
size="512"
initialSize="512"