You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by no...@apache.org on 2022/09/07 11:47:41 UTC
[solr] branch branch_9x updated: SOLR-16328: intern() strings in DocCollection to reduce memory footprint
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 977a8464173 SOLR-16328: intern() strings in DocCollection to reduce memory footprint
977a8464173 is described below
commit 977a8464173fc9a8a23297c515a43ecd0d1f6e5e
Author: Noble Paul <no...@users.noreply.github.com>
AuthorDate: Wed Sep 7 21:39:08 2022 +1000
SOLR-16328: intern() strings in DocCollection to reduce memory footprint
---
solr/CHANGES.txt | 2 +
.../java/org/apache/solr/core/CoreContainer.java | 54 ++++++++++++++++++++++
.../org/apache/solr/common/cloud/ClusterState.java | 16 ++++++-
3 files changed, 71 insertions(+), 1 deletion(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 29093056f23..00ae9c8cd1a 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -70,6 +70,8 @@ Optimizations
* SOLR-16336: avoid fetching solrconfig.xml & schema.xml for already cached schema and config (noble)
+* SOLR-16328: intern() strings in DocCollection to reduce memory footprint (noble)
+
Bug Fixes
---------------------
* SOLR-15918: Skip repetitive parent znode creation on config set upload (Mike Drob)
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 3c2aba578e3..5e5bda2ae1f 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -29,6 +29,7 @@ import static org.apache.solr.common.params.CommonParams.ZK_STATUS_PATH;
import static org.apache.solr.core.CorePropertiesLocator.PROPERTIES_FILENAME;
import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
+import com.github.benmanes.caffeine.cache.Interner;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
@@ -56,6 +57,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeoutException;
+import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
@@ -88,6 +90,8 @@ import org.apache.solr.cluster.placement.impl.PlacementPluginFactoryLoader;
import org.apache.solr.common.AlreadyClosedException;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.Aliases;
+import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Replica.State;
@@ -149,6 +153,8 @@ import org.apache.solr.util.RefCounted;
import org.apache.solr.util.StartupLoggingUtils;
import org.apache.solr.util.stats.MetricUtils;
import org.apache.zookeeper.KeeperException;
+import org.noggit.JSONParser;
+import org.noggit.ObjectBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -382,6 +388,7 @@ public class CoreContainer {
if (null != this.cfg.getBooleanQueryMaxClauseCount()) {
IndexSearcher.setMaxClauseCount(this.cfg.getBooleanQueryMaxClauseCount());
}
+ setWeakStringInterner();
this.coresLocator = locator;
this.containerProperties = new Properties(config.getSolrProperties());
this.asyncSolrCoreLoad = asyncSolrCoreLoad;
@@ -2320,6 +2327,25 @@ public class CoreContainer {
return status;
}
+ /**
+ * Retrieve the aliases from zookeeper. This is typically cached and does not hit zookeeper after
+ * the first use.
+ *
+ * @return an immutable instance of {@code Aliases} accurate as of at the time this method is
+ * invoked, less any zookeeper update lag.
+ * @throws RuntimeException if invoked on a {@code CoreContainer} where {@link
+ * #isZooKeeperAware()} returns false
+ */
+ public Aliases getAliases() {
+ if (isZooKeeperAware()) {
+ return getZkController().getZkStateReader().getAliases();
+ } else {
+ // fail fast because it's programmer error, but give slightly more info than NPE.
+ throw new IllegalStateException(
+ "Aliases don't exist in a non-cloud context, check isZookeeperAware() before calling this method.");
+ }
+ }
+
// Occasionally we need to access the transient cache handler in places other than coreContainer.
public TransientSolrCoreCache getTransientCache() {
return solrCores.getTransientCacheHandler();
@@ -2404,6 +2430,34 @@ public class CoreContainer {
public void runAsync(Runnable r) {
coreContainerAsyncTaskExecutor.submit(r);
}
+
+ public static void setWeakStringInterner() {
+ boolean enable = "true".equals(System.getProperty("solr.use.str.intern", "true"));
+ if (!enable) return;
+ Interner<String> interner = Interner.newWeakInterner();
+ ClusterState.setStrInternerParser(
+ new Function<>() {
+ @Override
+ public ObjectBuilder apply(JSONParser p) {
+ try {
+ return new ObjectBuilder(p) {
+ @Override
+ public void addKeyVal(Object map, Object key, Object val) throws IOException {
+ if (key != null) {
+ key = interner.intern(key.toString());
+ }
+ if (val instanceof String) {
+ val = interner.intern((String) val);
+ }
+ super.addKeyVal(map, key, val);
+ }
+ };
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ }
}
class CloserThread extends Thread {
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
index d54446afa0d..7d7e3c30a12 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.common.cloud;
+import static org.apache.solr.common.util.Utils.STANDARDOBJBUILDER;
+
import java.lang.invoke.MethodHandles;
import java.util.Collection;
import java.util.Collections;
@@ -28,6 +30,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
+import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.apache.solr.common.SolrException;
@@ -35,7 +38,9 @@ import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.DocCollection.CollectionStateProps;
import org.apache.solr.common.cloud.Replica.ReplicaStateProps;
import org.apache.solr.common.util.Utils;
+import org.noggit.JSONParser;
import org.noggit.JSONWriter;
+import org.noggit.ObjectBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -227,7 +232,8 @@ public class ClusterState implements JSONWriter.Writable {
return new ClusterState(liveNodes, Collections.<String, DocCollection>emptyMap());
}
@SuppressWarnings({"unchecked"})
- Map<String, Object> stateMap = (Map<String, Object>) Utils.fromJSON(bytes);
+ Map<String, Object> stateMap =
+ (Map<String, Object>) Utils.fromJSON(bytes, 0, bytes.length, STR_INTERNER_OBJ_BUILDER);
return createFromCollectionMap(version, stateMap, liveNodes);
}
@@ -477,4 +483,12 @@ public class ClusterState implements JSONWriter.Writable {
return perReplicaStates;
}
}
+
+ private static volatile Function<JSONParser, ObjectBuilder> STR_INTERNER_OBJ_BUILDER =
+ STANDARDOBJBUILDER;
+
+ public static void setStrInternerParser(Function<JSONParser, ObjectBuilder> fun) {
+ if (fun == null) return;
+ STR_INTERNER_OBJ_BUILDER = fun;
+ }
}