You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/03/10 09:58:54 UTC
[lucene] 39/49: SOLR-14155: Load all other SolrCore plugins from
packages (#1666)
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch jira/solr-13105-toMerge
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 3d01944de08dcffed35c810d4bceb7e46ec49e74
Author: Noble Paul <no...@users.noreply.github.com>
AuthorDate: Wed Jan 13 22:28:01 2021 +1100
SOLR-14155: Load all other SolrCore plugins from packages (#1666)
---
solr/CHANGES.txt | 2 +
.../org/apache/solr/core/DirectoryFactory.java | 2 +-
.../src/java/org/apache/solr/core/PluginBag.java | 1 -
.../src/java/org/apache/solr/core/SolrCore.java | 18 ++--
.../org/apache/solr/core/SolrResourceLoader.java | 70 ++++++++++++++--
.../org/apache/solr/handler/SolrConfigHandler.java | 29 ++++---
.../solr/handler/component/SearchHandler.java | 26 +++---
.../java/org/apache/solr/pkg/PackageListeners.java | 15 +++-
.../solr/pkg/PackageListeningClassLoader.java | 73 +++++++++++------
.../org/apache/solr/pkg/PackagePluginHolder.java | 16 +++-
.../java/org/apache/solr/search/CacheConfig.java | 44 +++++++---
.../java/org/apache/solr/update/UpdateHandler.java | 13 +--
.../solr/configsets/conf2/conf/schema.xml | 43 ++++++++++
.../solr/configsets/conf2/conf/solrconfig.xml | 68 ++++++++++++++++
.../src/test/org/apache/solr/pkg/TestPackages.java | 95 +++++++++++++++++-----
15 files changed, 403 insertions(+), 112 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index a62ca52..7c8a26f 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -249,6 +249,8 @@ Improvements
* SOLR-15040: Improvements to postlogs timestamp handling (Joel Bernstein)
+* SOLR-14155: Load all other SolrCore plugins from packages (noble)
+
Optimizations
---------------------
* SOLR-14975: Optimize CoreContainer.getAllCoreNames, getLoadedCoreNames and getCoreDescriptors. (Bruno Roustant)
diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
index 5736692..619e7c9 100644
--- a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
@@ -419,7 +419,7 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
final DirectoryFactory dirFactory;
if (info != null) {
log.debug(info.className);
- dirFactory = config.getResourceLoader().newInstance(info.className, DirectoryFactory.class);
+ dirFactory = config.getResourceLoader().newInstance (info, DirectoryFactory.class, true);
// allow DirectoryFactory instances to access the CoreContainer
dirFactory.initCoreContainer(cc);
dirFactory.init(info.initArgs);
diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java
index bdafa72..427cd7e 100644
--- a/solr/core/src/java/org/apache/solr/core/PluginBag.java
+++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java
@@ -396,7 +396,6 @@ public class PluginBag<T> implements AutoCloseable {
if (pluginInfo != null) return pluginInfo.className;
return null;
}
-
public PluginInfo getPluginInfo() {
return pluginInfo;
}
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index a388bf3..c01f4c4 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -755,7 +755,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
IndexReaderFactory indexReaderFactory;
PluginInfo info = solrConfig.getPluginInfo(IndexReaderFactory.class.getName());
if (info != null) {
- indexReaderFactory = resourceLoader.newInstance(info.className, IndexReaderFactory.class);
+ indexReaderFactory = resourceLoader.newInstance(info, IndexReaderFactory.class, true);
indexReaderFactory.init(info.initArgs);
} else {
indexReaderFactory = new StandardIndexReaderFactory();
@@ -955,6 +955,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
this.solrConfig = configSet.getSolrConfig();
this.resourceLoader = configSet.getSolrConfig().getResourceLoader();
+ this.resourceLoader.initCore(this);
IndexSchema schema = configSet.getIndexSchema();
this.configSetProperties = configSet.getProperties();
@@ -1241,7 +1242,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
private void checkVersionFieldExistsInSchema(IndexSchema schema, CoreDescriptor coreDescriptor) {
if (null != coreDescriptor.getCloudDescriptor()) {
- // we are evidently running in cloud mode.
+ // we are evidently running in cloud mode.
//
// In cloud mode, version field is required for correct consistency
// ideally this check would be more fine grained, and individual features
@@ -1405,7 +1406,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
final PluginInfo info = solrConfig.getPluginInfo(CodecFactory.class.getName());
final CodecFactory factory;
if (info != null) {
- factory = resourceLoader.newInstance(info.className, CodecFactory.class);
+ factory = resourceLoader.newInstance( info, CodecFactory.class, true);
factory.init(info.initArgs);
} else {
factory = new CodecFactory() {
@@ -1443,8 +1444,8 @@ public final class SolrCore implements SolrInfoBean, Closeable {
final StatsCache cache;
PluginInfo pluginInfo = solrConfig.getPluginInfo(StatsCache.class.getName());
if (pluginInfo != null && pluginInfo.className != null && pluginInfo.className.length() > 0) {
- cache = createInitInstance(pluginInfo, StatsCache.class, null,
- LocalStatsCache.class.getName());
+ cache = resourceLoader.newInstance( pluginInfo, StatsCache.class, true);
+ initPlugin(pluginInfo ,cache);
if (log.isDebugEnabled()) {
log.debug("Using statsCache impl: {}", cache.getClass().getName());
}
@@ -2134,7 +2135,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
newReader = currentReader;
}
- // for now, turn off caches if this is for a realtime reader
+ // for now, turn off caches if this is for a realtime reader
// (caches take a little while to instantiate)
final boolean useCaches = !realtime;
final String newName = realtime ? "realtime" : "main";
@@ -3177,7 +3178,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
public void cleanupOldIndexDirectories(boolean reload) {
final DirectoryFactory myDirFactory = getDirectoryFactory();
final String myDataDir = getDataDir();
- final String myIndexDir = getNewIndexDir(); // ensure the latest replicated index is protected
+ final String myIndexDir = getNewIndexDir(); // ensure the latest replicated index is protected
final String coreName = getName();
if (myDirFactory != null && myDataDir != null && myIndexDir != null) {
Thread cleanupThread = new Thread(() -> {
@@ -3280,6 +3281,9 @@ public final class SolrCore implements SolrInfoBean, Closeable {
this.coreName = coreName;
this.coreId = coreId;
}
+ public void reload() {
+ coreContainer.reload(coreName, coreId);
+ }
public void withCore(Consumer<SolrCore> r) {
try(SolrCore core = coreContainer.getCore(coreName, coreId)) {
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
index fa183c6..288259a 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
@@ -42,6 +42,7 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -64,6 +65,7 @@ import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.handler.component.ShardHandlerFactory;
import org.apache.solr.logging.DeprecationLog;
import org.apache.solr.pkg.PackageListeningClassLoader;
+import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.rest.RestManager;
@@ -100,6 +102,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
private CoreContainer coreContainer;
private PackageListeningClassLoader schemaLoader ;
+ private PackageListeningClassLoader coreReloadingClassLoader ;
private final List<SolrCoreAware> waitingForCore = Collections.synchronizedList(new ArrayList<SolrCoreAware>());
private final List<SolrInfoBean> infoMBeans = Collections.synchronizedList(new ArrayList<SolrInfoBean>());
private final List<ResourceLoaderAware> waitingForResources = Collections.synchronizedList(new ArrayList<ResourceLoaderAware>());
@@ -109,7 +112,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
// Provide a registry so that managed resources can register themselves while the XML configuration
// documents are being parsed ... after all are registered, they are asked by the RestManager to
// initialize themselves. This two-step process is required because not all resources are available
- // (such as the SolrZkClient) when XML docs are being parsed.
+ // (such as the SolrZkClient) when XML docs are being parsed.
private RestManager.Registry managedResourceRegistry;
/** @see #reloadLuceneSPI() */
@@ -165,7 +168,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
if (instanceDir == null) {
throw new NullPointerException("SolrResourceLoader instanceDir must be non-null");
}
-
+
this.instanceDir = instanceDir;
log.debug("new SolrResourceLoader for directory: '{}'", this.instanceDir);
@@ -645,15 +648,29 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
}
}
+ void initCore(SolrCore core) {
+ this.coreName = core.getName();
+ this.config = core.getSolrConfig();
+ this.coreId = core.uniqueId;
+ this.coreContainer = core.getCoreContainer();
+ SolrCore.Provider coreProvider = core.coreProvider;
+
+ this.coreReloadingClassLoader = new PackageListeningClassLoader(core.getCoreContainer(),
+ this, s -> config.maxPackageVersion(s), null){
+ @Override
+ protected void doReloadAction(Ctx ctx) {
+ log.info("Core reloading classloader issued reload for: {}/{} ", coreName, coreId);
+ coreProvider.reload();
+ }
+ };
+ core.getPackageListeners().addListener(coreReloadingClassLoader, true);
+
+ }
/**
* Tell all {@link SolrCoreAware} instances about the SolrCore
*/
public void inform(SolrCore core) {
- this.coreName = core.getName();
- this.config = core.getSolrConfig();
- this.coreId = core.uniqueId;
- this.coreContainer = core.getCoreContainer();
if(getSchemaLoader() != null) core.getPackageListeners().addListener(schemaLoader);
// make a copy to avoid potential deadlock of a callback calling newInstance and trying to
@@ -834,6 +851,47 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
public List<SolrInfoBean> getInfoMBeans() {
return Collections.unmodifiableList(infoMBeans);
}
+ /**
+ * Load a class using an appropriate {@link SolrResourceLoader} depending of the package on that class
+ * @param registerCoreReloadListener register a listener for the package and reload the core if the package is changed.
+ * Use this sparingly. This will result in core reloads across all the cores in
+ * all collections using this configset
+ */
+ public <T> Class<? extends T> findClass( PluginInfo info, Class<T> type, boolean registerCoreReloadListener) {
+ if(info.cName.pkg == null) return findClass(info.className, type);
+ return _classLookup(info,
+ (Function<PackageLoader.Package.Version, Class<? extends T>>) ver -> ver.getLoader().findClass(info.cName.className, type), registerCoreReloadListener);
+
+ }
+
+
+ private <T> T _classLookup(PluginInfo info, Function<PackageLoader.Package.Version, T> fun, boolean registerCoreReloadListener ) {
+ PluginInfo.ClassName cName = info.cName;
+ if (registerCoreReloadListener) {
+ if (coreReloadingClassLoader == null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Core not set");
+ }
+ return fun.apply(coreReloadingClassLoader.findPackageVersion(cName, true));
+ } else {
+ return fun.apply(coreReloadingClassLoader.findPackageVersion(cName, false));
+ }
+ }
+
+ /**
+ *Create a n instance of a class using an appropriate {@link SolrResourceLoader} depending on the package of that class
+ * @param registerCoreReloadListener register a listener for the package and reload the core if the package is changed.
+ * Use this sparingly. This will result in core reloads across all the cores in
+ * all collections using this configset
+ */
+ public <T> T newInstance(PluginInfo info, Class<T> type, boolean registerCoreReloadListener) {
+ if(info.cName.pkg == null) {
+ return newInstance(info.cName.className == null?
+ type.getName():
+ info.cName.className ,
+ type);
+ }
+ return _classLookup( info, version -> version.getLoader().newInstance(info.cName.className, type), registerCoreReloadListener);
+ }
private PackageListeningClassLoader createSchemaLoader() {
CoreContainer cc = getCoreContainer();
diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
index 0b97896..db8ec17 100644
--- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
@@ -48,6 +48,7 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.io.stream.expr.Expressible;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.ZkSolrResourceLoader;
+import org.apache.solr.common.MapSerializable;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
@@ -69,6 +70,7 @@ import org.apache.solr.core.RequestParams;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.pkg.PackageAPI;
import org.apache.solr.pkg.PackageListeners;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
@@ -250,25 +252,28 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
String componentName = req.getParams().get("componentName");
if (componentName != null) {
@SuppressWarnings({"rawtypes"})
- Map map = (Map) val.get(parts.get(1));
- if (map != null) {
- Object o = map.get(componentName);
- val.put(parts.get(1), makeMap(componentName, o));
+ Map pluginNameVsPluginInfo = (Map) val.get(parts.get(1));
+ if (pluginNameVsPluginInfo != null) {
+ @SuppressWarnings({"rawtypes"})
+ Object o = pluginNameVsPluginInfo instanceof MapSerializable ?
+ pluginNameVsPluginInfo:
+ pluginNameVsPluginInfo.get(componentName);
+ @SuppressWarnings({"rawtypes"})
+ Map pluginInfo = o instanceof MapSerializable? ((MapSerializable) o).toMap(new LinkedHashMap<>()): (Map) o;
+ val.put(parts.get(1),pluginNameVsPluginInfo instanceof PluginInfo? pluginInfo : makeMap(componentName, pluginInfo));
if (req.getParams().getBool("meta", false)) {
// meta=true is asking for the package info of the plugin
// We go through all the listeners and see if there is one registered for this plugin
List<PackageListeners.Listener> listeners = req.getCore().getPackageListeners().getListeners();
for (PackageListeners.Listener listener :
listeners) {
- PluginInfo info = listener.pluginInfo();
- if(info == null) continue;
- if (info.type.equals(parts.get(1)) && info.name.equals(componentName)) {
- if (o instanceof Map) {
- @SuppressWarnings({"rawtypes"})
- Map m1 = (Map) o;
- m1.put("_packageinfo_", listener.getPackageVersion(info.cName));
+ Map<String, PackageAPI.PkgVersion> infos = listener.packageDetails();
+ if(infos == null || infos.isEmpty()) continue;
+ infos.forEach((s, mapWriter) -> {
+ if(s.equals(pluginInfo.get("class"))) {
+ (pluginInfo).put("_packageinfo_", mapWriter);
}
- }
+ });
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
index f5c2474..c0c536f 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
@@ -16,16 +16,6 @@
*/
package org.apache.solr.handler.component;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.lucene.index.ExitableDirectoryReader;
import org.apache.lucene.search.TotalHits;
@@ -44,6 +34,7 @@ import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.pkg.PackageAPI;
import org.apache.solr.pkg.PackageListeners;
import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.request.SolrQueryRequest;
@@ -61,10 +52,13 @@ import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.common.params.CommonParams.DISTRIB;
-import static org.apache.solr.common.params.CommonParams.FAILURE;
-import static org.apache.solr.common.params.CommonParams.PATH;
-import static org.apache.solr.common.params.CommonParams.STATUS;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.lang.invoke.MethodHandles;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.apache.solr.common.params.CommonParams.*;
/**
@@ -162,8 +156,8 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware,
}
@Override
- public PluginInfo pluginInfo() {
- return null;
+ public Map<String , PackageAPI.PkgVersion> packageDetails() {
+ return Collections.emptyMap();
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java
index 1895b6d..1281d99 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java
@@ -47,6 +47,14 @@ public class PackageListeners {
public synchronized void addListener(Listener listener) {
listeners.add(new SoftReference<>(listener));
+ }
+
+ public synchronized void addListener(Listener listener, boolean addFirst) {
+ if(addFirst) {
+ listeners.add(0, new SoftReference<>(listener));
+ } else {
+ addListener(listener);
+ }
}
@@ -71,7 +79,7 @@ public class PackageListeners {
invokeListeners(pkgInfo, ctx);
}
} finally {
- ctx.runLaterTasks(core::runAsync);
+ ctx.runLaterTasks(r -> core.getCoreContainer().runAsync(r));
MDCLoggingContext.clear();
}
}
@@ -102,7 +110,10 @@ public class PackageListeners {
/**Name of the package or null to listen to all package changes */
String packageName();
- PluginInfo pluginInfo();
+ /** fetch the package versions of class names
+ *
+ */
+ Map<String, PackageAPI.PkgVersion> packageDetails();
/**A callback when the package is updated */
void changed(PackageLoader.Package pkg, Ctx ctx);
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java
index bbdf66b..14617aa 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java
@@ -26,9 +26,8 @@ import org.apache.solr.common.cloud.SolrClassLoader;
import org.apache.solr.core.SolrResourceLoader;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
/**
* A {@link SolrClassLoader} that is designed to listen to a set of packages.
@@ -37,22 +36,29 @@ import java.util.function.Function;
* */
public class PackageListeningClassLoader implements SolrClassLoader , PackageListeners.Listener {
private final CoreContainer coreContainer;
- private final SolrResourceLoader coreResourceLoader;
+ private final SolrClassLoader fallbackClassLoader;
private final Function<String, String> pkgVersionSupplier;
/** package name and the versions that we are tracking
*/
- private Map<String ,PackageAPI.PkgVersion> packageVersions = new HashMap<>(1);
+ private Map<String ,PackageAPI.PkgVersion> packageVersions = new ConcurrentHashMap<>(1);
+ private Map<String, String> classNameVsPackageName = new ConcurrentHashMap<>();
private final Runnable onReload;
+ /**
+ * @param fallbackClassLoader The {@link SolrClassLoader} to use if no package is specified
+ * @param pkgVersionSupplier Get the version configured for a given package
+ * @param onReload The callback function that should be run if a package is updated
+ */
public PackageListeningClassLoader(CoreContainer coreContainer,
- SolrResourceLoader coreResourceLoader,
+ SolrClassLoader fallbackClassLoader,
Function<String, String> pkgVersionSupplier,
Runnable onReload) {
this.coreContainer = coreContainer;
- this.coreResourceLoader = coreResourceLoader;
+ this.fallbackClassLoader = fallbackClassLoader;
this.pkgVersionSupplier = pkgVersionSupplier;
this.onReload = () -> {
- packageVersions = new HashMap<>();
+ packageVersions = new ConcurrentHashMap<>();
+ classNameVsPackageName = new ConcurrentHashMap<>();
onReload.run();
};
}
@@ -61,18 +67,28 @@ public class PackageListeningClassLoader implements SolrClassLoader , PackageLis
@Override
public <T> T newInstance(String cname, Class<T> expectedType, String... subpackages) {
PluginInfo.ClassName cName = new PluginInfo.ClassName(cname);
- if(cName.pkg == null){
- return coreResourceLoader.newInstance(cname, expectedType, subpackages);
+ if(cName.pkg == null) {
+ return fallbackClassLoader.newInstance(cname, expectedType, subpackages);
} else {
- PackageLoader.Package.Version version = findPkgVersion(cName);
- return applyResourceLoaderAware(version, version.getLoader().newInstance(cName.className, expectedType, subpackages));
-
+ PackageLoader.Package.Version version = findPackageVersion(cName, true);
+ T obj = version.getLoader().newInstance(cName.className, expectedType, subpackages);
+ classNameVsPackageName.put(cName.original, cName.pkg);
+ return applyResourceLoaderAware(version, obj);
}
}
- private PackageLoader.Package.Version findPkgVersion(PluginInfo.ClassName cName) {
+
+ /**
+ * This looks up for package and also listens for that package if required
+ * @param cName The class name
+ */
+ public PackageLoader.Package.Version findPackageVersion(PluginInfo.ClassName cName, boolean registerListener) {
PackageLoader.Package.Version theVersion = coreContainer.getPackageLoader().getPackage(cName.pkg).getLatest(pkgVersionSupplier.apply(cName.pkg));
- packageVersions.put(cName.pkg, theVersion.getPkgVersion());
+ if(registerListener) {
+ classNameVsPackageName.put(cName.original, cName.pkg);
+ PackageAPI.PkgVersion pkgVersion = theVersion.getPkgVersion();
+ if(pkgVersion !=null) packageVersions.put(cName.pkg, pkgVersion);
+ }
return theVersion;
}
@@ -101,10 +117,12 @@ public class PackageListeningClassLoader implements SolrClassLoader , PackageLis
public <T> T newInstance(String cname, Class<T> expectedType, String[] subPackages, Class[] params, Object[] args) {
PluginInfo.ClassName cName = new PluginInfo.ClassName(cname);
if (cName.pkg == null) {
- return coreResourceLoader.newInstance(cname, expectedType, subPackages, params, args);
+ return fallbackClassLoader.newInstance(cname, expectedType, subPackages, params, args);
} else {
- PackageLoader.Package.Version version = findPkgVersion(cName);
- return applyResourceLoaderAware(version, version.getLoader().newInstance(cName.className, expectedType, subPackages, params, args));
+ PackageLoader.Package.Version version = findPackageVersion(cName, true);
+ T obj = version.getLoader().newInstance(cName.className, expectedType, subPackages, params, args);
+ classNameVsPackageName.put(cName.original, cName.pkg);
+ return applyResourceLoaderAware(version, obj);
}
}
@@ -112,10 +130,12 @@ public class PackageListeningClassLoader implements SolrClassLoader , PackageLis
public <T> Class<? extends T> findClass(String cname, Class<T> expectedType) {
PluginInfo.ClassName cName = new PluginInfo.ClassName(cname);
if (cName.pkg == null) {
- return coreResourceLoader.findClass(cname, expectedType);
+ return fallbackClassLoader.findClass(cname, expectedType);
} else {
- PackageLoader.Package.Version version = findPkgVersion(cName);
- return version.getLoader().findClass(cName.className, expectedType);
+ PackageLoader.Package.Version version = findPackageVersion(cName, true);
+ Class<? extends T> klas = version.getLoader().findClass(cName.className, expectedType);
+ classNameVsPackageName.put(cName.original, cName.pkg);
+ return klas;
}
}
@@ -126,8 +146,10 @@ public class PackageListeningClassLoader implements SolrClassLoader , PackageLis
}
@Override
- public PluginInfo pluginInfo() {
- return null;
+ public Map<String, PackageAPI.PkgVersion> packageDetails() {
+ Map<String, PackageAPI.PkgVersion> result = new LinkedHashMap<>();
+ classNameVsPackageName.forEach((k, v) -> result.put(k, packageVersions.get(v)));
+ return result;
}
@Override
@@ -146,6 +168,11 @@ public class PackageListeningClassLoader implements SolrClassLoader , PackageLis
//no need to update
return;
}
+ doReloadAction(ctx);
+ }
+
+ protected void doReloadAction(Ctx ctx) {
+ if(onReload == null) return;
ctx.runLater(null, onReload);
}
}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
index 332de74..79626de 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
@@ -19,6 +19,8 @@ package org.apache.solr.pkg;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.Collections;
+import java.util.Map;
import org.apache.lucene.util.ResourceLoaderAware;
import org.apache.solr.common.MapWriter;
@@ -57,8 +59,8 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
}
@Override
- public PluginInfo pluginInfo() {
- return info;
+ public Map<String, PackageAPI.PkgVersion> packageDetails() {
+ return Collections.singletonMap(info.cName.original, pkgVersion.getPkgVersion());
}
@Override
@@ -75,6 +77,12 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
});
}
+ public static <T> PluginBag.PluginHolder<T> createHolder(T inst, Class<T> type) {
+ SolrConfig.SolrPluginInfo plugin = SolrConfig.classVsSolrPluginInfo.get(type.getName());
+ PluginInfo info = new PluginInfo(plugin.tag, Collections.singletonMap("class", inst.getClass().getName()));
+ return new PluginBag.PluginHolder<T>(info,inst);
+ }
+
public static <T> PluginBag.PluginHolder<T> createHolder(PluginInfo info, SolrCore core, Class<T> type, String msg) {
if(info.cName.pkg == null) {
return new PluginBag.PluginHolder<T>(info, core.createInitInstance(info, type,msg, null));
@@ -108,7 +116,7 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
if (log.isInfoEnabled()) {
log.info("loading plugin: {} -> {} using package {}:{}",
- pluginInfo.type, pluginInfo.name, pkg.name(), newest.getVersion());
+ pluginInfo.type, pluginInfo.name, pkg.name(), newest.getVersion());
}
initNewInstance(newest, core);
@@ -119,7 +127,7 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
@SuppressWarnings({"unchecked"})
protected Object initNewInstance(PackageLoader.Package.Version newest, SolrCore core) {
Object instance = SolrCore.createInstance(pluginInfo.className,
- pluginMeta.clazz, pluginMeta.getCleanTag(), core, newest.getLoader());
+ pluginMeta.clazz, pluginMeta.getCleanTag(), core, newest.getLoader());
PluginBag.initInstance(instance, pluginInfo);
handleAwareCallbacks(newest.getLoader(), instance, core);
T old = inst;
diff --git a/solr/core/src/java/org/apache/solr/search/CacheConfig.java b/solr/core/src/java/org/apache/solr/search/CacheConfig.java
index d8c41c6..1520f80 100644
--- a/solr/core/src/java/org/apache/solr/search/CacheConfig.java
+++ b/solr/core/src/java/org/apache/solr/search/CacheConfig.java
@@ -23,11 +23,15 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.Supplier;
import org.apache.solr.common.MapSerializable;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.DOMUtil;
import org.apache.solr.common.util.StrUtils;
+
+import org.apache.solr.core.PluginInfo;
+
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrResourceLoader;
import org.slf4j.Logger;
@@ -46,11 +50,17 @@ import static org.apache.solr.common.params.CommonParams.NAME;
*/
public class CacheConfig implements MapSerializable{
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
+
private String nodeName;
+
+ /**
+ * When this object is created, the core is not yet available . So, if the class is to be
+ * loaded from a package we should have a corresponding core
+ *
+ */
@SuppressWarnings({"rawtypes"})
- private Class<? extends SolrCache> clazz;
+ private Supplier<Class<? extends SolrCache>> clazz;
private Map<String,String> args;
private CacheRegenerator regenerator;
@@ -64,9 +74,10 @@ public class CacheConfig implements MapSerializable{
@SuppressWarnings({"rawtypes"})
public CacheConfig(Class<? extends SolrCache> clazz, Map<String,String> args, CacheRegenerator regenerator) {
- this.clazz = clazz;
+ this.clazz = () -> clazz;
this.args = args;
this.regenerator = regenerator;
+ this.nodeName = args.get(NAME);
}
public CacheRegenerator getRegenerator() {
@@ -85,7 +96,7 @@ public class CacheConfig implements MapSerializable{
Node node = nodes.item(i);
if ("true".equals(DOMUtil.getAttrOrDefault(node, "enabled", "true"))) {
CacheConfig config = getConfig(solrConfig, node.getNodeName(),
- DOMUtil.toMap(node.getAttributes()), configPath);
+ DOMUtil.toMap(node.getAttributes()), configPath);
result.put(config.args.get(NAME), config);
}
}
@@ -133,20 +144,32 @@ public class CacheConfig implements MapSerializable{
SolrResourceLoader loader = solrConfig.getResourceLoader();
config.cacheImpl = config.args.get("class");
- if(config.cacheImpl == null) config.cacheImpl = "solr.CaffeineCache";
+ if (config.cacheImpl == null) config.cacheImpl = "solr.CaffeineCache";
+ config.clazz = new Supplier<>() {
+ @SuppressWarnings("rawtypes")
+ Class<SolrCache> loadedClass;
+
+ @Override
+ @SuppressWarnings("rawtypes")
+ public Class<? extends SolrCache> get() {
+ if (loadedClass != null) return loadedClass;
+ return loadedClass = (Class<SolrCache>) loader.findClass(
+ new PluginInfo("cache", Collections.singletonMap("class", config.cacheImpl)),
+ SolrCache.class, true);
+ }
+ };
config.regenImpl = config.args.get("regenerator");
- config.clazz = loader.findClass(config.cacheImpl, SolrCache.class);
if (config.regenImpl != null) {
config.regenerator = loader.newInstance(config.regenImpl, CacheRegenerator.class);
}
-
+
return config;
}
@SuppressWarnings({"rawtypes"})
public SolrCache newInstance() {
try {
- SolrCache cache = clazz.getConstructor().newInstance();
+ SolrCache cache = clazz.get().getConstructor().newInstance();
persistence[0] = cache.init(args, persistence[0], regenerator);
return cache;
} catch (Exception e) {
@@ -158,11 +181,8 @@ public class CacheConfig implements MapSerializable{
}
@Override
- @SuppressWarnings({"unchecked"})
public Map<String, Object> toMap(Map<String, Object> map) {
- @SuppressWarnings({"rawtypes"})
- Map result = Collections.unmodifiableMap(args);
- return result;
+ return new HashMap<>(args);
}
public String getNodeName() {
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateHandler.java b/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
index 14b78af..ade9240 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
@@ -20,12 +20,7 @@ import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Vector;
-import org.apache.solr.core.DirectoryFactory;
-import org.apache.solr.core.HdfsDirectoryFactory;
-import org.apache.solr.core.PluginInfo;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.core.SolrEventListener;
-import org.apache.solr.core.SolrInfoBean;
+import org.apache.solr.core.*;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
@@ -108,7 +103,7 @@ public abstract class UpdateHandler implements SolrInfoBean {
public UpdateHandler(SolrCore core) {
this(core, null);
}
-
+
public UpdateHandler(SolrCore core, UpdateLog updateLog) {
this.core=core;
idField = core.getLatestSchema().getUniqueKeyField();
@@ -123,8 +118,8 @@ public abstract class UpdateHandler implements SolrInfoBean {
if (dirFactory instanceof HdfsDirectoryFactory) {
ulog = new HdfsUpdateLog(((HdfsDirectoryFactory)dirFactory).getConfDir());
} else {
- String className = ulogPluginInfo.className == null ? UpdateLog.class.getName() : ulogPluginInfo.className;
- ulog = core.getResourceLoader().newInstance(className, UpdateLog.class);
+ ulog = ulogPluginInfo.className == null ? new UpdateLog():
+ core.getResourceLoader().newInstance(ulogPluginInfo, UpdateLog.class, true);
}
if (!core.isReloaded() && !dirFactory.isPersistent()) {
diff --git a/solr/core/src/test-files/solr/configsets/conf2/conf/schema.xml b/solr/core/src/test-files/solr/configsets/conf2/conf/schema.xml
new file mode 100644
index 0000000..85f7ed3
--- /dev/null
+++ b/solr/core/src/test-files/solr/configsets/conf2/conf/schema.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<schema name="minimal" version="1.1">
+ <fieldType name="string" class="solr.StrField"/>
+ <fieldType name="int" class="${solr.tests.IntegerFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="long" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <dynamicField name="*" type="string" indexed="true" stored="true"/>
+ <!-- for versioning -->
+ <field name="_version_" type="long" indexed="true" stored="true"/>
+ <field name="_root_" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="id" type="string" indexed="true" stored="true"/>
+ <dynamicField name="*_s" type="string" indexed="true" stored="true" />
+ <uniqueKey>id</uniqueKey>
+ <query>
+ <filterCache
+ size="0"
+ initialSize="0"
+ autowarmCount="0"/>
+ <queryResultCache
+ size="0"
+ initialSize="0"
+ autowarmCount="0"/>
+ <documentCache
+ size="0"
+ initialSize="0"
+ autowarmCount="0"/>
+ </query>
+</schema>
diff --git a/solr/core/src/test-files/solr/configsets/conf2/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/conf2/conf/solrconfig.xml
new file mode 100644
index 0000000..e69d16f
--- /dev/null
+++ b/solr/core/src/test-files/solr/configsets/conf2/conf/solrconfig.xml
@@ -0,0 +1,68 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Minimal solrconfig.xml with /select, /admin and /update only -->
+
+<config>
+
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <directoryFactory name="DirectoryFactory"
+ class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+ <commitWithin>
+ <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
+ </commitWithin>
+ <updateLog class="${solr.ulog:solr.UpdateLog}"></updateLog>
+ </updateHandler>
+
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="indent">true</str>
+ <str name="df">text</str>
+ </lst>
+
+ </requestHandler>
+ <query>
+ <filterCache
+ class = "mypkg:org.apache.solr.search.CaffeineCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0" />
+
+ <queryResultCache
+ size="512"
+ initialSize="512"
+ autowarmCount="0" />
+
+ <documentCache
+ size="512"
+ initialSize="512"
+ autowarmCount="0" />
+ </query>
+ <indexConfig>
+ <mergeScheduler class="${solr.mscheduler:org.apache.lucene.index.ConcurrentMergeScheduler}"/>
+ </indexConfig>
+</config>
+
diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
index bea5d0c..2e908ef 100644
--- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
@@ -17,17 +17,6 @@
package org.apache.solr.pkg;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
-
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory;
import org.apache.lucene.analysis.pattern.PatternReplaceCharFilterFactory;
@@ -40,11 +29,7 @@ import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.GenericSolrRequest;
-import org.apache.solr.client.solrj.request.RequestWriter;
-import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.client.solrj.request.V2Request;
+import org.apache.solr.client.solrj.request.*;
import org.apache.solr.client.solrj.request.beans.Package;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.util.ClientUtils;
@@ -75,13 +60,22 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH;
import static org.apache.solr.common.params.CommonParams.JAVABIN;
import static org.apache.solr.common.params.CommonParams.WT;
import static org.apache.solr.core.TestSolrConfigHandler.getFileContent;
-import static org.apache.solr.filestore.TestDistribPackageStore.checkAllNodesForFile;
-import static org.apache.solr.filestore.TestDistribPackageStore.readFile;
-import static org.apache.solr.filestore.TestDistribPackageStore.uploadKey;
+import static org.apache.solr.filestore.TestDistribPackageStore.*;
@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG")
public class TestPackages extends SolrCloudTestCase {
@@ -102,6 +96,69 @@ public class TestPackages extends SolrCloudTestCase {
@JsonProperty("class")
public String klass;
}
+
+
+ public void testCoreReloadingPlugin() throws Exception {
+ MiniSolrCloudCluster cluster =
+ configureCluster(4)
+ .withJettyConfig(jetty -> jetty.enableV2(true))
+ .addConfig("conf", configset("conf2"))
+ .configure();
+ try {
+ String FILE1 = "/mypkg/runtimelibs.jar";
+ String COLLECTION_NAME = "testCoreReloadingPluginColl";
+ byte[] derFile = readFile("cryptokeys/pub_key512.der");
+ uploadKey(derFile, PackageStoreAPI.KEYS_DIR+"/pub_key512.der", cluster);
+ postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1,
+ "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ==");
+
+ Package.AddVersion add = new Package.AddVersion();
+ add.version = "1.0";
+ add.pkg = "mypkg";
+ add.files = Arrays.asList(new String[]{FILE1});
+ V2Request req = new V2Request.Builder("/cluster/package")
+ .forceV2(true)
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload(Collections.singletonMap("add", add))
+ .build();
+
+ req.process(cluster.getSolrClient());
+ TestDistribPackageStore.assertResponseValues(10,
+ () -> new V2Request.Builder("/cluster/package").
+ withMethod(SolrRequest.METHOD.GET)
+ .build().process(cluster.getSolrClient()),
+ Utils.makeMap(
+ ":result:packages:mypkg[0]:version", "1.0",
+ ":result:packages:mypkg[0]:files[0]", FILE1
+ ));
+
+ CollectionAdminRequest
+ .createCollection(COLLECTION_NAME, "conf", 2, 2)
+ .process(cluster.getSolrClient());
+ cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4);
+
+ verifyComponent(cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", add.pkg, add.version);
+
+
+ add.version = "2.0";
+ req.process(cluster.getSolrClient());
+ TestDistribPackageStore.assertResponseValues(10,
+ () -> new V2Request.Builder("/cluster/package").
+ withMethod(SolrRequest.METHOD.GET)
+ .build().process(cluster.getSolrClient()),
+ Utils.makeMap(
+ ":result:packages:mypkg[1]:version", "2.0",
+ ":result:packages:mypkg[1]:files[0]", FILE1
+ ));
+ new UpdateRequest().commit(cluster.getSolrClient(), COLLECTION_NAME);
+
+ verifyComponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "query", "filterCache",
+ "mypkg", "2.0" );
+ } finally {
+ cluster.shutdown();
+ }
+ }
@Test
@SuppressWarnings({"unchecked"})
public void testPluginLoading() throws Exception {