You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2020/09/26 13:32:21 UTC
[lucene-solr] 01/01: SOLR-14899: Make caches load from packages
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch jira/solr14899
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit ec9731f40db0f790769c77ebdda7fd884e400890
Author: noblepaul <no...@gmail.com>
AuthorDate: Sat Sep 26 23:31:45 2020 +1000
SOLR-14899: Make caches load from packages
---
.../src/java/org/apache/solr/core/SolrConfig.java | 2 +-
.../src/java/org/apache/solr/core/SolrCore.java | 40 +++--
.../org/apache/solr/core/SolrResourceLoader.java | 26 +--
.../org/apache/solr/pkg/PackagePluginHolder.java | 7 +-
.../java/org/apache/solr/search/CacheConfig.java | 32 ++--
.../org/apache/solr/search/DelegatingCache.java | 176 +++++++++++++++++++++
6 files changed, 250 insertions(+), 33 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
index bc1f055..bcc8bcb 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
@@ -239,7 +239,7 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable {
args.put("size", "10000");
args.put("initialSize", "10");
args.put("showItems", "-1");
- conf = new CacheConfig(CaffeineCache.class, args, null);
+ conf = new CacheConfig(CaffeineCache.class, args, null, this);
}
fieldValueCacheConfig = conf;
useColdSearcher = getBool("query/useColdSearcher", false);
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 955635a..1b8fe9c 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -57,7 +57,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
-import java.util.function.Consumer;
+import java.util.function.Function;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Timer;
@@ -281,7 +281,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
private ExecutorService coreAsyncTaskExecutor = ExecutorUtil.newMDCAwareCachedThreadPool("Core Async Task");
- public final SolrCore.Provider coreProvider;
+ private SolrCore.Provider coreProvider;
/**
* The SolrResourceLoader used to load all resources for this core.
@@ -951,7 +951,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
this.configSet = configSet;
this.coreDescriptor = Objects.requireNonNull(coreDescriptor, "coreDescriptor cannot be null");
setName(coreDescriptor.getName());
- coreProvider = new Provider(coreContainer, getName(), uniqueId);
+ coreProvider = new Provider(this);
this.solrConfig = configSet.getSolrConfig();
this.resourceLoader = configSet.getSolrConfig().getResourceLoader();
@@ -1085,6 +1085,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
} finally {
// allow firstSearcher events to fire and make sure it is released
latch.countDown();
+ if(coreProvider !=null) coreProvider.coreUnderConstruction = null;
}
assert ObjectReleaseTracker.track(this);
@@ -3237,6 +3238,9 @@ public final class SolrCore implements SolrInfoBean, Closeable {
return blobRef;
}
+ public SolrCore.Provider getCoreProvider(){
+ return coreProvider;
+ }
/**
* Run an arbitrary task in it's own thread. This is an expert option and is
* a method you should use with great care. It would be bad to run something that never stopped
@@ -3268,17 +3272,33 @@ public final class SolrCore implements SolrInfoBean, Closeable {
private final CoreContainer coreContainer;
private final String coreName;
private final UUID coreId;
+ private SolrCore coreUnderConstruction;
- public Provider(CoreContainer coreContainer, String coreName, UUID coreId) {
- this.coreContainer = coreContainer;
- this.coreName = coreName;
- this.coreId = coreId;
+ public Provider(SolrCore core) {
+ this.coreContainer = core.getCoreContainer();
+ this.coreName = core.getName();
+ this.coreId = core.uniqueId;
+ this.coreUnderConstruction = core;
+ }
+
+ public String getCoreName() {
+ return coreName;
}
- public void withCore(Consumer<SolrCore> r) {
+ public UUID getCoreId() {
+ return coreId;
+ }
+
+ public <T> T withCore(Function<SolrCore, T> r) {
+ if(coreUnderConstruction != null) {
+ //this core in under construction
+ return r.apply (coreUnderConstruction);
+ }
try(SolrCore core = coreContainer.getCore(coreName, coreId)) {
- if(core == null) return;
- r.accept(core);
+ if(core == null) {
+ return null;
+ }
+ return r.apply(core);
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
index 532f1dd..7ce7f70 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
@@ -40,7 +40,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -93,8 +92,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
private String name = "";
protected URLClassLoader classLoader;
private final Path instanceDir;
- private String coreName;
- private UUID coreId;
+ private SolrCore.Provider coreProvider;
private SolrConfig config;
private CoreContainer coreContainer;
private PackageListeningClassLoader schemaLoader ;
@@ -205,6 +203,15 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
}
}
+ public void setCoreProvider(SolrCore.Provider coreProvider) {
+ if(coreProvider != null) return;// it can be set only once
+ this.coreProvider = coreProvider;
+ }
+
+ public SolrCore.Provider getCoreProvider(){
+ return coreProvider;
+ }
+
/**
* Reloads all Lucene SPI implementations using the new classloader.
* This method must be called after {@link #addToClassLoader(List)}
@@ -653,9 +660,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
* Tell all {@link SolrCoreAware} instances about the SolrCore
*/
public void inform(SolrCore core) {
- this.coreName = core.getName();
this.config = core.getSolrConfig();
- this.coreId = core.uniqueId;
this.coreContainer = core.getCoreContainer();
if(getSchemaLoader() != null) core.getPackageListeners().addListener(schemaLoader);
@@ -841,12 +846,11 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
if (getSolrConfig() == null) return null;
return getSolrConfig().maxPackageVersion(pkg);
}, () -> {
- if(getCoreContainer() == null || config == null || coreName == null || coreId==null) return;
- try (SolrCore c = getCoreContainer().getCore(coreName, coreId)) {
- if (c != null) {
- c.fetchLatestSchema();
- }
- }
+ if(coreProvider == null) return;
+ coreProvider.withCore(c -> {
+ c.fetchLatestSchema();
+ return null;
+ });
});
}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
index 3a2bf4d..42f1456 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
@@ -45,7 +45,7 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
public PackagePluginHolder(PluginInfo info, SolrCore core, SolrConfig.SolrPluginInfo pluginMeta) {
super(info);
- this.coreProvider = core.coreProvider;
+ this.coreProvider = core.getCoreProvider();
this.pluginMeta = pluginMeta;
this.info = info;
@@ -63,7 +63,10 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
@Override
public void changed(PackageLoader.Package pkg, Ctx ctx) {
- coreProvider.withCore(c -> reload(pkg, c));
+ coreProvider.withCore(c -> {
+ reload(pkg, c);
+ return null;
+ });
}
diff --git a/solr/core/src/java/org/apache/solr/search/CacheConfig.java b/solr/core/src/java/org/apache/solr/search/CacheConfig.java
index 6470915..528a099 100644
--- a/solr/core/src/java/org/apache/solr/search/CacheConfig.java
+++ b/solr/core/src/java/org/apache/solr/search/CacheConfig.java
@@ -23,10 +23,12 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.Supplier;
+import org.apache.solr.common.MapSerializable;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.common.MapSerializable;
+import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.util.DOMUtil;
@@ -50,23 +52,26 @@ public class CacheConfig implements MapSerializable{
private String nodeName;
@SuppressWarnings({"rawtypes"})
- private Class<? extends SolrCache> clazz;
+ private Supplier<Class<? extends SolrCache>> clazz;
private Map<String,String> args;
private CacheRegenerator regenerator;
- private String cacheImpl;
+ private PluginInfo.ClassName cacheImpl;
private Object[] persistence = new Object[1];
private String regenImpl;
+ private SolrConfig solrConfig;
public CacheConfig() {}
@SuppressWarnings({"rawtypes"})
- public CacheConfig(Class<? extends SolrCache> clazz, Map<String,String> args, CacheRegenerator regenerator) {
- this.clazz = clazz;
+ public CacheConfig(Class<? extends SolrCache> clazz, Map<String,String> args, CacheRegenerator regenerator, SolrConfig solrConfig) {
+ this.clazz = () -> clazz;
+ this.cacheImpl = new PluginInfo.ClassName(clazz.getName());
this.args = args;
this.regenerator = regenerator;
+ this.solrConfig = solrConfig;
}
public CacheRegenerator getRegenerator() {
@@ -132,10 +137,10 @@ public class CacheConfig implements MapSerializable{
}
SolrResourceLoader loader = solrConfig.getResourceLoader();
- config.cacheImpl = config.args.get("class");
- if(config.cacheImpl == null) config.cacheImpl = "solr.CaffeineCache";
+ config.solrConfig = solrConfig;
+ config.cacheImpl = new PluginInfo.ClassName(config.args.getOrDefault("class", CaffeineCache.class.getName()));
config.regenImpl = config.args.get("regenerator");
- config.clazz = loader.findClass(config.cacheImpl, SolrCache.class);
+ config.clazz = () -> loader.findClass(config.cacheImpl.className, SolrCache.class);
if (config.regenImpl != null) {
config.regenerator = loader.newInstance(config.regenImpl, CacheRegenerator.class);
}
@@ -145,8 +150,17 @@ public class CacheConfig implements MapSerializable{
@SuppressWarnings({"rawtypes"})
public SolrCache newInstance() {
+ if(cacheImpl.pkg == null) {
+ return getSolrCache(clazz);
+ } else {
+ return new DelegatingCache(this, cacheImpl, solrConfig.getResourceLoader().getCoreProvider());
+ }
+ }
+ @SuppressWarnings({"rawtypes"})
+ public SolrCache getSolrCache(Supplier<Class <? extends SolrCache>> classSupplier ) {
try {
- SolrCache cache = clazz.getConstructor().newInstance();
+ Class clazz = classSupplier.get();
+ SolrCache cache = (SolrCache) clazz.getConstructor().newInstance();
persistence[0] = cache.init(args, persistence[0], regenerator);
return cache;
} catch (Exception e) {
diff --git a/solr/core/src/java/org/apache/solr/search/DelegatingCache.java b/solr/core/src/java/org/apache/solr/search/DelegatingCache.java
new file mode 100644
index 0000000..017a77e
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/DelegatingCache.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.core.PluginInfo;
+import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.metrics.SolrMetricsContext;
+import org.apache.solr.pkg.PackageLoader;
+import org.apache.solr.pkg.PackagePluginHolder;
+
+public class DelegatingCache<K, V> implements SolrCache<K, V> {
+
+ private static final SolrConfig.SolrPluginInfo CACHE_PLUGIN_INFO = SolrConfig.classVsSolrPluginInfo.get(SolrCache.class.getName());
+ private volatile SolrCache<K, V> delegate;
+ final CacheConfig cacheConfig;
+
+ private SolrMetricsContext solrMetricsContext;
+ private String scope;
+
+ public DelegatingCache(CacheConfig cacheConfig, PluginInfo.ClassName cName, SolrCore.Provider coreProvider) {
+ this.cacheConfig = cacheConfig;
+ PackagePluginHolder<SolrCache<K, V>> cacheHolder = coreProvider
+ .withCore(core -> new PackagePluginHolder<>(new PluginInfo(CACHE_PLUGIN_INFO.tag, cacheConfig.toMap(new HashMap<>())), core, CACHE_PLUGIN_INFO) {
+
+ @Override
+ @SuppressWarnings("unchecked")
+ protected Object initNewInstance(PackageLoader.Package.Version newest, SolrCore core) {
+ SolrCache newCache = cacheConfig.getSolrCache(() -> newest.getLoader().findClass(cName.className, SolrCache.class));
+ if (solrMetricsContext != null && scope != null) {
+ newCache.initializeMetrics(solrMetricsContext, scope);
+ }
+ SolrCache<K, V> oldCache = delegate;
+ newCache.setState(oldCache.getState());
+ delegate = newCache;
+ oldCache.clear();
+ return newCache;
+ }
+ });
+ if(cacheHolder == null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No core available");
+ }
+
+ }
+
+
+ @Override
+ public Object init(Map args, Object persistence, CacheRegenerator regenerator) {
+ return null;
+ }
+
+ @Override
+ public String name() {
+ return delegate.name();
+ }
+
+ @Override
+ public int size() {
+ return delegate.size();
+ }
+
+ @Override
+ public V put(K key, V value) {
+ return delegate.put(key, value);
+ }
+
+ @Override
+ public V get(K key) {
+ return delegate.get(key);
+ }
+
+ @Override
+ public V remove(K key) {
+ return delegate.remove(key);
+ }
+
+ @Override
+ public V computeIfAbsent(K key, Function<? super K, ? extends V> mappingFunction) {
+ return delegate.computeIfAbsent(key, mappingFunction);
+ }
+
+ @Override
+ public void clear() {
+ delegate.clear();
+
+ }
+
+ @Override
+ public void setState(State state) {
+ delegate.setState(state);
+ }
+
+ @Override
+ public State getState() {
+ return delegate.getState();
+ }
+
+ @Override
+ public void warm(SolrIndexSearcher searcher, SolrCache<K, V> old) {
+ if (old instanceof DelegatingCache) {
+ delegate.warm(searcher, ((DelegatingCache<K, V>) old).delegate);
+ } else {
+ delegate.warm(searcher, old);
+ }
+
+ }
+
+ @Override
+ public int getMaxSize() {
+ return delegate.getMaxSize();
+ }
+
+ @Override
+ public void setMaxSize(int maxSize) {
+ delegate.setMaxSize(maxSize);
+ }
+
+ @Override
+ public int getMaxRamMB() {
+ return delegate.getMaxRamMB();
+ }
+
+ @Override
+ public void setMaxRamMB(int maxRamMB) {
+ delegate.setMaxRamMB(maxRamMB);
+ }
+
+ @Override
+ public String getName() {
+ return delegate.getName();
+ }
+
+ @Override
+ public String getDescription() {
+ return delegate.getDescription();
+ }
+
+ @Override
+ public Category getCategory() {
+ return delegate.getCategory();
+ }
+
+
+
+ @Override
+ public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
+ delegate.initializeMetrics(parentContext, scope);
+ this.solrMetricsContext = parentContext;
+ this.scope = scope;
+ }
+
+ @Override
+ public SolrMetricsContext getSolrMetricsContext() {
+ return delegate.getSolrMetricsContext();
+ }
+}