You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/03/10 09:56:26 UTC
[lucene] 30/42: merging with master
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch jira/solr14155-1
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 768c4f5d83ac4e48a2a25fbdbce5e317a00c582f
Author: noblepaul <no...@gmail.com>
AuthorDate: Tue Sep 8 14:06:31 2020 +1000
merging with master
---
.../java/org/apache/solr/core/CoreContainer.java | 1 -
.../org/apache/solr/core/SolrResourceLoader.java | 84 +++++++++++++++++-----
.../org/apache/solr/handler/StreamHandler.java | 2 +-
.../java/org/apache/solr/pkg/PackageListeners.java | 8 +++
.../solr/pkg/PackageListeningClassLoader.java | 5 ++
.../org/apache/solr/pkg/PackagePluginHolder.java | 68 ++++++++++--------
6 files changed, 120 insertions(+), 48 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index ed49cce..0453c61 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -1402,7 +1402,6 @@ public class CoreContainer {
return core.getDirectoryFactory().isSharedStorage();
} else {
ConfigSet configSet = coreConfigService.loadConfigSet(cd);
- configSet.getSolrConfig().getResourceLoader().core = core;
return DirectoryFactory.loadDirectoryFactory(configSet.getSolrConfig(), this, null).isSharedStorage();
}
}
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
index 082fe2a..96b8974 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
@@ -44,8 +44,10 @@ import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.SolrClassLoader;
import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.handler.component.ShardHandlerFactory;
+import org.apache.solr.pkg.PackageListeningClassLoader;
import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.QueryResponseWriter;
@@ -62,15 +64,14 @@ import org.slf4j.LoggerFactory;
/**
* @since solr 1.3
*/
-public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassLoader {
+public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassLoader, SolrCoreAware {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final String base = "org.apache.solr";
private static final String[] packages = {
"", "analysis.", "schema.", "handler.", "handler.tagger.", "search.", "update.", "core.", "response.", "request.",
- "update.processor.", "util.", "spelling.", "handler.component.", "handler.dataimport.",
- "spelling.suggest.", "spelling.suggest.fst.", "rest.schema.analysis.", "security.", "handler.admin.",
- "cloud.autoscaling."
+ "update.processor.", "util.", "spelling.", "handler.component.",
+ "spelling.suggest.", "spelling.suggest.fst.", "rest.schema.analysis.", "security.", "handler.admin."
};
private static final Charset UTF_8 = StandardCharsets.UTF_8;
@@ -78,12 +79,14 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
private String name = "";
protected URLClassLoader classLoader;
private final Path instanceDir;
+ private String coreName;
+ private UUID coreId;
+ private SolrConfig config;
+ private CoreContainer coreContainer;
+ private PackageListeningClassLoader schemaLoader ;
+
+ private PackageListeningClassLoader coreReloadingClassLoader ;
- /**
- * this is set by the {@link SolrCore}
- * This could be null if the core is not yet initialized
- */
- SolrCore core;
private final List<SolrCoreAware> waitingForCore = Collections.synchronizedList(new ArrayList<SolrCoreAware>());
private final List<SolrInfoBean> infoMBeans = Collections.synchronizedList(new ArrayList<SolrInfoBean>());
@@ -107,6 +110,33 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
return managedResourceRegistry;
}
+ private PackageListeningClassLoader createSchemaLoader() {
+ CoreContainer cc = getCoreContainer();
+ if (cc == null) {
+ //corecontainer not available . can't load from packages
+ return null;
+ }
+ return new PackageListeningClassLoader(cc, this, pkg -> {
+ if (getSolrConfig() == null) return null;
+ return getSolrConfig().maxPackageVersion(pkg);
+ }, () -> {
+ if(getCoreContainer() == null || config == null || coreName == null || coreId==null) return;
+ try (SolrCore c = getCoreContainer().getCore(coreName, coreId)) {
+ if (c != null) {
+ c.fetchLatestSchema();
+ }
+ }
+ });
+ }
+
+
+ public SolrClassLoader getSchemaLoader() {
+ if (schemaLoader == null) {
+ schemaLoader = createSchemaLoader();
+ }
+ return schemaLoader;
+ }
+
public SolrResourceLoader() {
this(SolrPaths.locateSolrHome(), null);
}
@@ -207,11 +237,6 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
TokenizerFactory.reloadTokenizers(this.classLoader);
}
- public SolrCore getCore(){
- return core;
- }
-
-
private static URLClassLoader addURLsToClassLoader(final URLClassLoader oldLoader, List<URL> urls) {
if (urls.size() == 0) {
return oldLoader;
@@ -617,7 +642,22 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
/**
* Tell all {@link SolrCoreAware} instances about the SolrCore
*/
+ @Override
public void inform(SolrCore core) {
+ this.coreName = core.getName();
+ this.config = core.getSolrConfig();
+ this.coreId = core.uniqueId;
+ this.coreContainer = core.getCoreContainer();
+ this.coreReloadingClassLoader = new PackageListeningClassLoader(core.getCoreContainer(),
+ this, s -> config.maxPackageVersion(s), null){
+ @Override
+ protected void doReloadAction(Ctx ctx) {
+ coreContainer.reload(coreName, coreId, true);
+ }
+ };
+ core.getPackageListeners().addListener(coreReloadingClassLoader, true);
+ if(getSchemaLoader() != null) core.getPackageListeners().addListener(schemaLoader);
+
// make a copy to avoid potential deadlock of a callback calling newInstance and trying to
// add something to waitingForCore.
SolrCoreAware[] arr;
@@ -756,6 +796,14 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, builder.toString());
}
+ public CoreContainer getCoreContainer(){
+ return coreContainer;
+ }
+
+ public SolrConfig getSolrConfig() {
+ return config;
+
+ }
@Override
public void close() throws IOException {
IOUtils.close(classLoader);
@@ -780,12 +828,12 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
private <T> T _classLookup(PluginInfo info, Function<PackageLoader.Package.Version, T> fun, boolean registerCoreReloadListener ) {
PluginInfo.ClassName cName = info.cName;
if (registerCoreReloadListener) {
- if (getCore() == null) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "SolrCore not set");
+ if (coreReloadingClassLoader == null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Core not set");
}
- return fun.apply(getCore().getCoreReloadingClassLoader().findPackageVersion(cName, true));
+ return fun.apply(coreReloadingClassLoader.findPackageVersion(cName, true));
} else {
- return fun.apply(getCore().getCoreReloadingClassLoader().findPackageVersion(cName, false));
+ return fun.apply(coreReloadingClassLoader.findPackageVersion(cName, false));
}
}
diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
index 0877c54..5b9cb27 100644
--- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
@@ -158,7 +158,7 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
}
@Override
- protected Object initNewInstance(PackageLoader.Package.Version newest) {
+ protected Object initNewInstance(PackageLoader.Package.Version newest, SolrCore core) {
return clazz = newest.getLoader().findClass(pluginInfo.className, Expressible.class);
}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java
index 7403baa..25c6e4f 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java
@@ -47,6 +47,14 @@ public class PackageListeners {
public synchronized void addListener(Listener listener) {
listeners.add(new SoftReference<>(listener));
+ }
+
+ public synchronized void addListener(Listener listener, boolean addFirst) {
+ if(addFirst) {
+ listeners.add(0, new SoftReference<>(listener));
+ } else {
+ addListener(listener);
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java
index 09f8871..af46d5b 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java
@@ -158,6 +158,11 @@ public class PackageListeningClassLoader implements SolrClassLoader , PackageLis
//no need to update
return;
}
+ doReloadAction(ctx);
+ }
+
+ protected void doReloadAction(Ctx ctx) {
+ if(onReload == null) return;
ctx.runLater(null, onReload);
}
}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
index c1cd645..0714543 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
@@ -20,10 +20,12 @@ package org.apache.solr.pkg;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Collections;
+import java.util.UUID;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
+import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.PluginBag;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrConfig;
@@ -38,7 +40,9 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String LATEST = "$LATEST";
- private final SolrCore core;
+ private final String coreName;
+ private final UUID coreId;
+ private final CoreContainer coreContainer;
private final SolrConfig.SolrPluginInfo pluginMeta;
private PackageLoader.Package.Version pkgVersion;
private PluginInfo info;
@@ -46,7 +50,9 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
public PackagePluginHolder(PluginInfo info, SolrCore core, SolrConfig.SolrPluginInfo pluginMeta) {
super(info);
- this.core = core;
+ this.coreContainer = core.getCoreContainer();
+ this.coreName = core.getName();
+ this.coreId = core.uniqueId;
this.pluginMeta = pluginMeta;
this.info = info;
@@ -91,44 +97,50 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
}
private synchronized void reload(PackageLoader.Package pkg) {
- String lessThan = core.getSolrConfig().maxPackageVersion(info.pkgName);
- PackageLoader.Package.Version newest = pkg.getLatest(lessThan);
- if (newest == null) {
- log.error("No latest version available for package : {}", pkg.name());
- return;
- }
- if (lessThan != null) {
- PackageLoader.Package.Version pkgLatest = pkg.getLatest();
- if (pkgLatest != newest) {
- if (log.isInfoEnabled()) {
- log.info("Using version :{}. latest is {}, params.json has config {} : {}", newest.getVersion(), pkgLatest.getVersion(), pkg.name(), lessThan);
- }
+ try (SolrCore core = coreContainer.getCore(coreName, coreId)) {
+ if (core == null) {
+ //this core got reloaded. do not do anything
+ return;
}
- }
+ String lessThan = core.getSolrConfig().maxPackageVersion(info.pkgName);
- if (pkgVersion != null) {
- if (newest == pkgVersion) {
- //I'm already using the latest classloader in the package. nothing to do
+ PackageLoader.Package.Version newest = pkg.getLatest(lessThan);
+ if (newest == null) {
+ log.error("No latest version available for package : {}", pkg.name());
return;
}
- }
+ if (lessThan != null) {
+ PackageLoader.Package.Version pkgLatest = pkg.getLatest();
+ if (pkgLatest != newest) {
+ if (log.isInfoEnabled()) {
+ log.info("Using version :{}. latest is {}, params.json has config {} : {}", newest.getVersion(), pkgLatest.getVersion(), pkg.name(), lessThan);
+ }
+ }
+ }
- if (log.isInfoEnabled()) {
- log.info("loading plugin: {} -> {} using package {}:{}",
- pluginInfo.type, pluginInfo.name, pkg.name(), newest.getVersion());
- }
+ if (pkgVersion != null) {
+ if (newest == pkgVersion) {
+ //I'm already using the latest classloader in the package. nothing to do
+ return;
+ }
+ }
- initNewInstance(newest);
- pkgVersion = newest;
+ if (log.isInfoEnabled()) {
+ log.info("loading plugin: {} -> {} using package {}:{}",
+ pluginInfo.type, pluginInfo.name, pkg.name(), newest.getVersion());
+ }
+ initNewInstance(newest, core);
+ pkgVersion = newest;
+ }
}
@SuppressWarnings({"unchecked"})
- protected Object initNewInstance(PackageLoader.Package.Version newest) {
+ protected Object initNewInstance(PackageLoader.Package.Version newest, SolrCore core) {
Object instance = SolrCore.createInstance(pluginInfo.className,
pluginMeta.clazz, pluginMeta.getCleanTag(), core, newest.getLoader());
PluginBag.initInstance(instance, pluginInfo);
- handleAwareCallbacks(newest.getLoader(), instance);
+ handleAwareCallbacks(newest.getLoader(), instance, core);
T old = inst;
inst = (T) instance;
if (old instanceof AutoCloseable) {
@@ -142,7 +154,7 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
return inst;
}
- private void handleAwareCallbacks(SolrResourceLoader loader, Object instance) {
+ private void handleAwareCallbacks(SolrResourceLoader loader, Object instance, SolrCore core) {
if (instance instanceof SolrCoreAware) {
SolrCoreAware coreAware = (SolrCoreAware) instance;
if (!core.getResourceLoader().addToCoreAware(coreAware)) {