You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by no...@apache.org on 2022/09/07 11:39:12 UTC
[solr] branch main updated: SOLR-16328 : intern() strings in DocCollection to reduce memory footp… (#965)
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 1b7ff717b19 SOLR-16328 : intern() strings in DocCollection to reduce memory footp… (#965)
1b7ff717b19 is described below
commit 1b7ff717b191502dde2476cf49dcd2ecf134d0a9
Author: Noble Paul <no...@users.noreply.github.com>
AuthorDate: Wed Sep 7 21:39:08 2022 +1000
SOLR-16328 : intern() strings in DocCollection to reduce memory footp… (#965)
---
solr/CHANGES.txt | 2 ++
.../java/org/apache/solr/core/CoreContainer.java | 34 ++++++++++++++++++++++
.../org/apache/solr/common/cloud/ClusterState.java | 16 +++++++++-
3 files changed, 51 insertions(+), 1 deletion(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index f6f59a16594..9113045788c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -92,6 +92,8 @@ Optimizations
* SOLR-16336: avoid fetching solrconfig.xml & schema.xml for already cached schema and config (noble)
+* SOLR-16328: intern() strings in DocCollection to reduce memory footprint (noble)
+
Bug Fixes
---------------------
* SOLR-15918: Skip repetitive parent znode creation on config set upload (Mike Drob)
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index a59b3c0ed81..5e5bda2ae1f 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -29,6 +29,7 @@ import static org.apache.solr.common.params.CommonParams.ZK_STATUS_PATH;
import static org.apache.solr.core.CorePropertiesLocator.PROPERTIES_FILENAME;
import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
+import com.github.benmanes.caffeine.cache.Interner;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
@@ -56,6 +57,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeoutException;
+import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
@@ -89,6 +91,7 @@ import org.apache.solr.common.AlreadyClosedException;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.Aliases;
+import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Replica.State;
@@ -150,6 +153,8 @@ import org.apache.solr.util.RefCounted;
import org.apache.solr.util.StartupLoggingUtils;
import org.apache.solr.util.stats.MetricUtils;
import org.apache.zookeeper.KeeperException;
+import org.noggit.JSONParser;
+import org.noggit.ObjectBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -383,6 +388,7 @@ public class CoreContainer {
if (null != this.cfg.getBooleanQueryMaxClauseCount()) {
IndexSearcher.setMaxClauseCount(this.cfg.getBooleanQueryMaxClauseCount());
}
+ setWeakStringInterner();
this.coresLocator = locator;
this.containerProperties = new Properties(config.getSolrProperties());
this.asyncSolrCoreLoad = asyncSolrCoreLoad;
@@ -2424,6 +2430,34 @@ public class CoreContainer {
public void runAsync(Runnable r) {
coreContainerAsyncTaskExecutor.submit(r);
}
+
+ public static void setWeakStringInterner() {
+ boolean enable = "true".equals(System.getProperty("solr.use.str.intern", "true"));
+ if (!enable) return;
+ Interner<String> interner = Interner.newWeakInterner();
+ ClusterState.setStrInternerParser(
+ new Function<>() {
+ @Override
+ public ObjectBuilder apply(JSONParser p) {
+ try {
+ return new ObjectBuilder(p) {
+ @Override
+ public void addKeyVal(Object map, Object key, Object val) throws IOException {
+ if (key != null) {
+ key = interner.intern(key.toString());
+ }
+ if (val instanceof String) {
+ val = interner.intern((String) val);
+ }
+ super.addKeyVal(map, key, val);
+ }
+ };
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ }
}
class CloserThread extends Thread {
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
index d54446afa0d..7d7e3c30a12 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.common.cloud;
+import static org.apache.solr.common.util.Utils.STANDARDOBJBUILDER;
+
import java.lang.invoke.MethodHandles;
import java.util.Collection;
import java.util.Collections;
@@ -28,6 +30,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
+import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.apache.solr.common.SolrException;
@@ -35,7 +38,9 @@ import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.DocCollection.CollectionStateProps;
import org.apache.solr.common.cloud.Replica.ReplicaStateProps;
import org.apache.solr.common.util.Utils;
+import org.noggit.JSONParser;
import org.noggit.JSONWriter;
+import org.noggit.ObjectBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -227,7 +232,8 @@ public class ClusterState implements JSONWriter.Writable {
return new ClusterState(liveNodes, Collections.<String, DocCollection>emptyMap());
}
@SuppressWarnings({"unchecked"})
- Map<String, Object> stateMap = (Map<String, Object>) Utils.fromJSON(bytes);
+ Map<String, Object> stateMap =
+ (Map<String, Object>) Utils.fromJSON(bytes, 0, bytes.length, STR_INTERNER_OBJ_BUILDER);
return createFromCollectionMap(version, stateMap, liveNodes);
}
@@ -477,4 +483,12 @@ public class ClusterState implements JSONWriter.Writable {
return perReplicaStates;
}
}
+
+ private static volatile Function<JSONParser, ObjectBuilder> STR_INTERNER_OBJ_BUILDER =
+ STANDARDOBJBUILDER;
+
+ public static void setStrInternerParser(Function<JSONParser, ObjectBuilder> fun) {
+ if (fun == null) return;
+ STR_INTERNER_OBJ_BUILDER = fun;
+ }
}