You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2020/09/26 13:32:20 UTC
[lucene-solr] branch jira/solr14899 created (now ec9731f)
This is an automated email from the ASF dual-hosted git repository.
noble pushed a change to branch jira/solr14899
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
at ec9731f SOLR-14899: Make caches load from packages
This branch includes the following new commits:
new ec9731f SOLR-14899: Make caches load from packages
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[lucene-solr] 01/01: SOLR-14899: Make caches load from packages
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch jira/solr14899
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit ec9731f40db0f790769c77ebdda7fd884e400890
Author: noblepaul <no...@gmail.com>
AuthorDate: Sat Sep 26 23:31:45 2020 +1000
SOLR-14899: Make caches load from packages
---
.../src/java/org/apache/solr/core/SolrConfig.java | 2 +-
.../src/java/org/apache/solr/core/SolrCore.java | 40 +++--
.../org/apache/solr/core/SolrResourceLoader.java | 26 +--
.../org/apache/solr/pkg/PackagePluginHolder.java | 7 +-
.../java/org/apache/solr/search/CacheConfig.java | 32 ++--
.../org/apache/solr/search/DelegatingCache.java | 176 +++++++++++++++++++++
6 files changed, 250 insertions(+), 33 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
index bc1f055..bcc8bcb 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
@@ -239,7 +239,7 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable {
args.put("size", "10000");
args.put("initialSize", "10");
args.put("showItems", "-1");
- conf = new CacheConfig(CaffeineCache.class, args, null);
+ conf = new CacheConfig(CaffeineCache.class, args, null, this);
}
fieldValueCacheConfig = conf;
useColdSearcher = getBool("query/useColdSearcher", false);
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 955635a..1b8fe9c 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -57,7 +57,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
-import java.util.function.Consumer;
+import java.util.function.Function;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Timer;
@@ -281,7 +281,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
private ExecutorService coreAsyncTaskExecutor = ExecutorUtil.newMDCAwareCachedThreadPool("Core Async Task");
- public final SolrCore.Provider coreProvider;
+ private SolrCore.Provider coreProvider;
/**
* The SolrResourceLoader used to load all resources for this core.
@@ -951,7 +951,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
this.configSet = configSet;
this.coreDescriptor = Objects.requireNonNull(coreDescriptor, "coreDescriptor cannot be null");
setName(coreDescriptor.getName());
- coreProvider = new Provider(coreContainer, getName(), uniqueId);
+ coreProvider = new Provider(this);
this.solrConfig = configSet.getSolrConfig();
this.resourceLoader = configSet.getSolrConfig().getResourceLoader();
@@ -1085,6 +1085,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
} finally {
// allow firstSearcher events to fire and make sure it is released
latch.countDown();
+ if(coreProvider !=null) coreProvider.coreUnderConstruction = null;
}
assert ObjectReleaseTracker.track(this);
@@ -3237,6 +3238,9 @@ public final class SolrCore implements SolrInfoBean, Closeable {
return blobRef;
}
+ public SolrCore.Provider getCoreProvider(){
+ return coreProvider;
+ }
/**
* Run an arbitrary task in it's own thread. This is an expert option and is
* a method you should use with great care. It would be bad to run something that never stopped
@@ -3268,17 +3272,33 @@ public final class SolrCore implements SolrInfoBean, Closeable {
private final CoreContainer coreContainer;
private final String coreName;
private final UUID coreId;
+ private SolrCore coreUnderConstruction;
- public Provider(CoreContainer coreContainer, String coreName, UUID coreId) {
- this.coreContainer = coreContainer;
- this.coreName = coreName;
- this.coreId = coreId;
+ public Provider(SolrCore core) {
+ this.coreContainer = core.getCoreContainer();
+ this.coreName = core.getName();
+ this.coreId = core.uniqueId;
+ this.coreUnderConstruction = core;
+ }
+
+ public String getCoreName() {
+ return coreName;
}
- public void withCore(Consumer<SolrCore> r) {
+ public UUID getCoreId() {
+ return coreId;
+ }
+
+ public <T> T withCore(Function<SolrCore, T> r) {
+ if(coreUnderConstruction != null) {
+ //this core in under construction
+ return r.apply (coreUnderConstruction);
+ }
try(SolrCore core = coreContainer.getCore(coreName, coreId)) {
- if(core == null) return;
- r.accept(core);
+ if(core == null) {
+ return null;
+ }
+ return r.apply(core);
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
index 532f1dd..7ce7f70 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
@@ -40,7 +40,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -93,8 +92,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
private String name = "";
protected URLClassLoader classLoader;
private final Path instanceDir;
- private String coreName;
- private UUID coreId;
+ private SolrCore.Provider coreProvider;
private SolrConfig config;
private CoreContainer coreContainer;
private PackageListeningClassLoader schemaLoader ;
@@ -205,6 +203,15 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
}
}
+ public void setCoreProvider(SolrCore.Provider coreProvider) {
+ if(coreProvider != null) return;// it can be set only once
+ this.coreProvider = coreProvider;
+ }
+
+ public SolrCore.Provider getCoreProvider(){
+ return coreProvider;
+ }
+
/**
* Reloads all Lucene SPI implementations using the new classloader.
* This method must be called after {@link #addToClassLoader(List)}
@@ -653,9 +660,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
* Tell all {@link SolrCoreAware} instances about the SolrCore
*/
public void inform(SolrCore core) {
- this.coreName = core.getName();
this.config = core.getSolrConfig();
- this.coreId = core.uniqueId;
this.coreContainer = core.getCoreContainer();
if(getSchemaLoader() != null) core.getPackageListeners().addListener(schemaLoader);
@@ -841,12 +846,11 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
if (getSolrConfig() == null) return null;
return getSolrConfig().maxPackageVersion(pkg);
}, () -> {
- if(getCoreContainer() == null || config == null || coreName == null || coreId==null) return;
- try (SolrCore c = getCoreContainer().getCore(coreName, coreId)) {
- if (c != null) {
- c.fetchLatestSchema();
- }
- }
+ if(coreProvider == null) return;
+ coreProvider.withCore(c -> {
+ c.fetchLatestSchema();
+ return null;
+ });
});
}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
index 3a2bf4d..42f1456 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
@@ -45,7 +45,7 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
public PackagePluginHolder(PluginInfo info, SolrCore core, SolrConfig.SolrPluginInfo pluginMeta) {
super(info);
- this.coreProvider = core.coreProvider;
+ this.coreProvider = core.getCoreProvider();
this.pluginMeta = pluginMeta;
this.info = info;
@@ -63,7 +63,10 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
@Override
public void changed(PackageLoader.Package pkg, Ctx ctx) {
- coreProvider.withCore(c -> reload(pkg, c));
+ coreProvider.withCore(c -> {
+ reload(pkg, c);
+ return null;
+ });
}
diff --git a/solr/core/src/java/org/apache/solr/search/CacheConfig.java b/solr/core/src/java/org/apache/solr/search/CacheConfig.java
index 6470915..528a099 100644
--- a/solr/core/src/java/org/apache/solr/search/CacheConfig.java
+++ b/solr/core/src/java/org/apache/solr/search/CacheConfig.java
@@ -23,10 +23,12 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.Supplier;
+import org.apache.solr.common.MapSerializable;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.common.MapSerializable;
+import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.util.DOMUtil;
@@ -50,23 +52,26 @@ public class CacheConfig implements MapSerializable{
private String nodeName;
@SuppressWarnings({"rawtypes"})
- private Class<? extends SolrCache> clazz;
+ private Supplier<Class<? extends SolrCache>> clazz;
private Map<String,String> args;
private CacheRegenerator regenerator;
- private String cacheImpl;
+ private PluginInfo.ClassName cacheImpl;
private Object[] persistence = new Object[1];
private String regenImpl;
+ private SolrConfig solrConfig;
public CacheConfig() {}
@SuppressWarnings({"rawtypes"})
- public CacheConfig(Class<? extends SolrCache> clazz, Map<String,String> args, CacheRegenerator regenerator) {
- this.clazz = clazz;
+ public CacheConfig(Class<? extends SolrCache> clazz, Map<String,String> args, CacheRegenerator regenerator, SolrConfig solrConfig) {
+ this.clazz = () -> clazz;
+ this.cacheImpl = new PluginInfo.ClassName(clazz.getName());
this.args = args;
this.regenerator = regenerator;
+ this.solrConfig = solrConfig;
}
public CacheRegenerator getRegenerator() {
@@ -132,10 +137,10 @@ public class CacheConfig implements MapSerializable{
}
SolrResourceLoader loader = solrConfig.getResourceLoader();
- config.cacheImpl = config.args.get("class");
- if(config.cacheImpl == null) config.cacheImpl = "solr.CaffeineCache";
+ config.solrConfig = solrConfig;
+ config.cacheImpl = new PluginInfo.ClassName(config.args.getOrDefault("class", CaffeineCache.class.getName()));
config.regenImpl = config.args.get("regenerator");
- config.clazz = loader.findClass(config.cacheImpl, SolrCache.class);
+ config.clazz = () -> loader.findClass(config.cacheImpl.className, SolrCache.class);
if (config.regenImpl != null) {
config.regenerator = loader.newInstance(config.regenImpl, CacheRegenerator.class);
}
@@ -145,8 +150,17 @@ public class CacheConfig implements MapSerializable{
@SuppressWarnings({"rawtypes"})
public SolrCache newInstance() {
+ if(cacheImpl.pkg == null) {
+ return getSolrCache(clazz);
+ } else {
+ return new DelegatingCache(this, cacheImpl, solrConfig.getResourceLoader().getCoreProvider());
+ }
+ }
+ @SuppressWarnings({"rawtypes"})
+ public SolrCache getSolrCache(Supplier<Class <? extends SolrCache>> classSupplier ) {
try {
- SolrCache cache = clazz.getConstructor().newInstance();
+ Class clazz = classSupplier.get();
+ SolrCache cache = (SolrCache) clazz.getConstructor().newInstance();
persistence[0] = cache.init(args, persistence[0], regenerator);
return cache;
} catch (Exception e) {
diff --git a/solr/core/src/java/org/apache/solr/search/DelegatingCache.java b/solr/core/src/java/org/apache/solr/search/DelegatingCache.java
new file mode 100644
index 0000000..017a77e
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/DelegatingCache.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.core.PluginInfo;
+import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.metrics.SolrMetricsContext;
+import org.apache.solr.pkg.PackageLoader;
+import org.apache.solr.pkg.PackagePluginHolder;
+
+public class DelegatingCache<K, V> implements SolrCache<K, V> {
+
+ private static final SolrConfig.SolrPluginInfo CACHE_PLUGIN_INFO = SolrConfig.classVsSolrPluginInfo.get(SolrCache.class.getName());
+ private volatile SolrCache<K, V> delegate;
+ final CacheConfig cacheConfig;
+
+ private SolrMetricsContext solrMetricsContext;
+ private String scope;
+
+ public DelegatingCache(CacheConfig cacheConfig, PluginInfo.ClassName cName, SolrCore.Provider coreProvider) {
+ this.cacheConfig = cacheConfig;
+ PackagePluginHolder<SolrCache<K, V>> cacheHolder = coreProvider
+ .withCore(core -> new PackagePluginHolder<>(new PluginInfo(CACHE_PLUGIN_INFO.tag, cacheConfig.toMap(new HashMap<>())), core, CACHE_PLUGIN_INFO) {
+
+ @Override
+ @SuppressWarnings("unchecked")
+ protected Object initNewInstance(PackageLoader.Package.Version newest, SolrCore core) {
+ SolrCache newCache = cacheConfig.getSolrCache(() -> newest.getLoader().findClass(cName.className, SolrCache.class));
+ if (solrMetricsContext != null && scope != null) {
+ newCache.initializeMetrics(solrMetricsContext, scope);
+ }
+ SolrCache<K, V> oldCache = delegate;
+ newCache.setState(oldCache.getState());
+ delegate = newCache;
+ oldCache.clear();
+ return newCache;
+ }
+ });
+ if(cacheHolder == null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No core available");
+ }
+
+ }
+
+
+ @Override
+ public Object init(Map args, Object persistence, CacheRegenerator regenerator) {
+ return null;
+ }
+
+ @Override
+ public String name() {
+ return delegate.name();
+ }
+
+ @Override
+ public int size() {
+ return delegate.size();
+ }
+
+ @Override
+ public V put(K key, V value) {
+ return delegate.put(key, value);
+ }
+
+ @Override
+ public V get(K key) {
+ return delegate.get(key);
+ }
+
+ @Override
+ public V remove(K key) {
+ return delegate.remove(key);
+ }
+
+ @Override
+ public V computeIfAbsent(K key, Function<? super K, ? extends V> mappingFunction) {
+ return delegate.computeIfAbsent(key, mappingFunction);
+ }
+
+ @Override
+ public void clear() {
+ delegate.clear();
+
+ }
+
+ @Override
+ public void setState(State state) {
+ delegate.setState(state);
+ }
+
+ @Override
+ public State getState() {
+ return delegate.getState();
+ }
+
+ @Override
+ public void warm(SolrIndexSearcher searcher, SolrCache<K, V> old) {
+ if (old instanceof DelegatingCache) {
+ delegate.warm(searcher, ((DelegatingCache<K, V>) old).delegate);
+ } else {
+ delegate.warm(searcher, old);
+ }
+
+ }
+
+ @Override
+ public int getMaxSize() {
+ return delegate.getMaxSize();
+ }
+
+ @Override
+ public void setMaxSize(int maxSize) {
+ delegate.setMaxSize(maxSize);
+ }
+
+ @Override
+ public int getMaxRamMB() {
+ return delegate.getMaxRamMB();
+ }
+
+ @Override
+ public void setMaxRamMB(int maxRamMB) {
+ delegate.setMaxRamMB(maxRamMB);
+ }
+
+ @Override
+ public String getName() {
+ return delegate.getName();
+ }
+
+ @Override
+ public String getDescription() {
+ return delegate.getDescription();
+ }
+
+ @Override
+ public Category getCategory() {
+ return delegate.getCategory();
+ }
+
+
+
+ @Override
+ public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
+ delegate.initializeMetrics(parentContext, scope);
+ this.solrMetricsContext = parentContext;
+ this.scope = scope;
+ }
+
+ @Override
+ public SolrMetricsContext getSolrMetricsContext() {
+ return delegate.getSolrMetricsContext();
+ }
+}