You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2019/10/12 06:17:35 UTC
[lucene-solr] branch jira/SOLR-13822 created (now 08f9e29)
This is an automated email from the ASF dual-hosted git repository.
noble pushed a change to branch jira/SOLR-13822
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
at 08f9e29 SOLR-13822: Isolated class loading from packages
This branch includes the following new commits:
new 08f9e29 SOLR-13822: Isolated class loading from packages
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[lucene-solr] 01/01: SOLR-13822: Isolated class loading from
packages
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch jira/SOLR-13822
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 08f9e293ca3db6115f4732f76fb32ad69713a070
Author: noble <no...@apache.org>
AuthorDate: Sat Oct 12 17:16:43 2019 +1100
SOLR-13822: Isolated class loading from packages
---
.../java/org/apache/solr/core/CoreContainer.java | 10 +
.../src/java/org/apache/solr/core/PluginBag.java | 15 +-
.../src/java/org/apache/solr/core/PluginInfo.java | 39 +-
.../src/java/org/apache/solr/core/SolrCore.java | 18 +-
.../org/apache/solr/core/SolrResourceLoader.java | 333 ++++++++++--------
.../org/apache/solr/handler/SolrConfigHandler.java | 23 +-
.../src/java/org/apache/solr/pkg/PackageAPI.java | 367 +++++++++++++++++++
.../java/org/apache/solr/pkg/PackageListeners.java | 89 +++++
.../java/org/apache/solr/pkg/PackageLoader.java | 254 +++++++++++++
.../org/apache/solr/pkg/PackagePluginHolder.java | 92 +++++
.../solr/security/PermissionNameProvider.java | 3 +
.../src/test/org/apache/solr/pkg/TestPackages.java | 391 +++++++++++++++++++++
.../org/apache/solr/common/cloud/SolrZkClient.java | 9 +-
.../apache/solr/common/cloud/ZkStateReader.java | 1 +
14 files changed, 1471 insertions(+), 173 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 054bd67..c32783c 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -100,6 +100,7 @@ import org.apache.solr.logging.MDCLoggingContext;
import org.apache.solr.metrics.SolrCoreMetricManager;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
+import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.search.SolrFieldCacheBean;
@@ -221,6 +222,7 @@ public class CoreContainer {
protected volatile AutoscalingHistoryHandler autoscalingHistoryHandler;
private PackageStoreAPI packageStoreAPI;
+ private PackageLoader packageLoader;
// Bits for the state variable.
@@ -580,6 +582,10 @@ public class CoreContainer {
return replayUpdatesExecutor;
}
+ public PackageLoader getPackageLoader() {
+ return packageLoader;
+ }
+
public PackageStoreAPI getPackageStoreAPI() {
return packageStoreAPI;
}
@@ -734,6 +740,10 @@ public class CoreContainer {
if (isZooKeeperAware()) {
metricManager.loadClusterReporters(metricReporters, this);
}
+ packageLoader = new PackageLoader(this);
+ containerHandlers.getApiBag().register(new AnnotatedApi(packageLoader.getPackageAPI().editAPI), Collections.EMPTY_MAP);
+ containerHandlers.getApiBag().register(new AnnotatedApi(packageLoader.getPackageAPI().readAPI), Collections.EMPTY_MAP);
+
// setup executor to load cores in parallel
ExecutorService coreLoadExecutor = MetricUtils.instrumentedExecutorService(
diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java
index 6088f52..bfdfa20 100644
--- a/solr/core/src/java/org/apache/solr/core/PluginBag.java
+++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java
@@ -43,6 +43,7 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.handler.component.SearchComponent;
+import org.apache.solr.pkg.PackagePluginHolder;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
@@ -97,7 +98,7 @@ public class PluginBag<T> implements AutoCloseable {
this(klass, core, false);
}
- static void initInstance(Object inst, PluginInfo info) {
+ public static void initInstance(Object inst, PluginInfo info) {
if (inst instanceof PluginInfoInitialized) {
((PluginInfoInitialized) inst).init(info);
} else if (inst instanceof NamedListInitializedPlugin) {
@@ -138,9 +139,13 @@ public class PluginBag<T> implements AutoCloseable {
log.debug("{} : '{}' created with startup=lazy ", meta.getCleanTag(), info.name);
return new LazyPluginHolder<T>(meta, info, core, core.getResourceLoader(), false);
} else {
- T inst = core.createInstance(info.className, (Class<T>) meta.clazz, meta.getCleanTag(), null, core.getResourceLoader());
- initInstance(inst, info);
- return new PluginHolder<>(info, inst);
+ if (info.pkgName != null) {
+ return new PackagePluginHolder<>(info, core, meta);
+ } else {
+ T inst = core.createInstance(info.className, (Class<T>) meta.clazz, meta.getCleanTag(), null, core.getResourceLoader(info.pkgName));
+ initInstance(inst, info);
+ return new PluginHolder<>(info, inst);
+ }
}
}
@@ -329,7 +334,7 @@ public class PluginBag<T> implements AutoCloseable {
* subclasses may choose to lazily load the plugin
*/
public static class PluginHolder<T> implements AutoCloseable {
- private T inst;
+ protected T inst;
protected final PluginInfo pluginInfo;
boolean registerAPI = false;
diff --git a/solr/core/src/java/org/apache/solr/core/PluginInfo.java b/solr/core/src/java/org/apache/solr/core/PluginInfo.java
index 1bc85ae..366309e 100644
--- a/solr/core/src/java/org/apache/solr/core/PluginInfo.java
+++ b/solr/core/src/java/org/apache/solr/core/PluginInfo.java
@@ -16,14 +16,20 @@
*/
package org.apache.solr.core;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.solr.common.MapSerializable;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.Pair;
import org.apache.solr.util.DOMUtil;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
-import java.util.*;
-
import static java.util.Arrays.asList;
import static java.util.Collections.unmodifiableList;
import static java.util.Collections.unmodifiableMap;
@@ -35,27 +41,46 @@ import static org.apache.solr.schema.FieldType.CLASS_NAME;
*
*/
public class PluginInfo implements MapSerializable {
- public final String name, className, type;
+ public final String name, className, type, pkgName;
public final NamedList initArgs;
public final Map<String, String> attributes;
public final List<PluginInfo> children;
private boolean isFromSolrConfig;
+
+
public PluginInfo(String type, Map<String, String> attrs, NamedList initArgs, List<PluginInfo> children) {
this.type = type;
this.name = attrs.get(NAME);
- this.className = attrs.get(CLASS_NAME);
+ Pair<String, String> parsed = parseClassName(attrs.get(CLASS_NAME));
+ this.className = parsed.second();
+ this.pkgName = parsed.first();
this.initArgs = initArgs;
attributes = unmodifiableMap(attrs);
this.children = children == null ? Collections.<PluginInfo>emptyList(): unmodifiableList(children);
isFromSolrConfig = false;
}
+ static Pair<String,String > parseClassName(String name) {
+ String pkgName = null;
+ String className = name;
+ if (name != null) {
+ int colonIdx = name.indexOf(':');
+ if (colonIdx > -1) {
+ pkgName = name.substring(0, colonIdx);
+ className = name.substring(colonIdx + 1);
+ }
+ }
+ return new Pair<>(pkgName, className);
+
+ }
public PluginInfo(Node node, String err, boolean requireName, boolean requireClass) {
type = node.getNodeName();
name = DOMUtil.getAttr(node, NAME, requireName ? err : null);
- className = DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null);
+ Pair<String, String> parsed = parseClassName(DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null));
+ className = parsed.second();
+ pkgName = parsed.first();
initArgs = DOMUtil.childNodesToNamedList(node);
attributes = unmodifiableMap(DOMUtil.toMap(node.getAttributes()));
children = loadSubPlugins(node);
@@ -85,7 +110,9 @@ public class PluginInfo implements MapSerializable {
}
this.type = type;
this.name = (String) m.get(NAME);
- this.className = (String) m.get(CLASS_NAME);
+ Pair<String, String> parsed = parseClassName((String) m.get(CLASS_NAME));
+ this.className = parsed.second();
+ this.pkgName = parsed.first();
attributes = unmodifiableMap(m);
this.children = Collections.<PluginInfo>emptyList();
isFromSolrConfig = true;
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 3e2fb1e..59c9a7a 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -110,6 +110,8 @@ import org.apache.solr.logging.MDCLoggingContext;
import org.apache.solr.metrics.SolrCoreMetricManager;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
+import org.apache.solr.pkg.PackageListeners;
+import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.BinaryResponseWriter;
@@ -237,6 +239,8 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
public volatile boolean indexEnabled = true;
public volatile boolean readOnly = false;
+ private PackageListeners packageListeners = new PackageListeners();
+
public Set<String> getMetricNames() {
return metricNames;
}
@@ -261,6 +265,10 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
return restManager;
}
+ public PackageListeners getPackageListeners() {
+ return packageListeners;
+ }
+
static int boolean_query_max_clause_count = Integer.MIN_VALUE;
private ExecutorService coreAsyncTaskExecutor = ExecutorUtil.newMDCAwareCachedThreadPool("Core Async Task");
@@ -274,6 +282,14 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
return resourceLoader;
}
+ public SolrResourceLoader getResourceLoader(String pkg) {
+ if (pkg == null) {
+ return resourceLoader;
+ }
+ PackageLoader.Package aPackage = coreContainer.getPackageLoader().getPackage(pkg);
+ return aPackage.getLatest().getLoader();
+ }
+
/**
* Gets the configuration resource name used by this core instance.
*
@@ -856,7 +872,7 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
public <T extends Object> T createInitInstance(PluginInfo info, Class<T> cast, String msg, String defClassName) {
if (info == null) return null;
- T o = createInstance(info.className == null ? defClassName : info.className, cast, msg, this, getResourceLoader());
+ T o = createInstance(info.className == null ? defClassName : info.className, cast, msg, this, getResourceLoader(info.pkgName));
if (o instanceof PluginInfoInitialized) {
((PluginInfoInitialized) o).init(info);
} else if (o instanceof NamedListInitializedPlugin) {
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
index 4132918..cc1ef7a 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
@@ -28,6 +28,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.Constructor;
+import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.charset.CharacterCodingException;
@@ -81,11 +82,11 @@ import org.slf4j.LoggerFactory;
/**
* @since solr 1.3
- */
-public class SolrResourceLoader implements ResourceLoader,Closeable
-{
+ */
+public class SolrResourceLoader implements ResourceLoader, Closeable {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private String name = "";
static final String project = "solr";
static final String base = "org.apache" + "." + project;
static final String[] packages = {
@@ -100,7 +101,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
protected URLClassLoader classLoader;
private final Path instanceDir;
private String dataDir;
-
+
private final List<SolrCoreAware> waitingForCore = Collections.synchronizedList(new ArrayList<SolrCoreAware>());
private final List<SolrInfoBean> infoMBeans = Collections.synchronizedList(new ArrayList<SolrInfoBean>());
private final List<ResourceLoaderAware> waitingForResources = Collections.synchronizedList(new ArrayList<ResourceLoaderAware>());
@@ -109,18 +110,18 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
private final Properties coreProperties;
private volatile boolean live;
-
+
// Provide a registry so that managed resources can register themselves while the XML configuration
// documents are being parsed ... after all are registered, they are asked by the RestManager to
// initialize themselves. This two-step process is required because not all resources are available
// (such as the SolrZkClient) when XML docs are being parsed.
private RestManager.Registry managedResourceRegistry;
-
+
public synchronized RestManager.Registry getManagedResourceRegistry() {
if (managedResourceRegistry == null) {
- managedResourceRegistry = new RestManager.Registry();
+ managedResourceRegistry = new RestManager.Registry();
}
- return managedResourceRegistry;
+ return managedResourceRegistry;
}
public SolrResourceLoader() {
@@ -134,11 +135,20 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
* found in the "lib/" directory in the specified instance directory.
* If the instance directory is not specified (=null), SolrResourceLoader#locateInstanceDir will provide one.
*/
- public SolrResourceLoader(Path instanceDir, ClassLoader parent)
- {
+ public SolrResourceLoader(Path instanceDir, ClassLoader parent) {
this(instanceDir, parent, null);
}
+ public SolrResourceLoader(String name, List<Path> classpath, Path instanceDir, ClassLoader parent) throws MalformedURLException {
+ this(instanceDir, parent);
+ this.name = name;
+ for (Path path : classpath) {
+ addToClassLoader(path.toUri().normalize().toURL());
+ }
+
+ }
+
+
public SolrResourceLoader(Path instanceDir) {
this(instanceDir, null, null);
}
@@ -157,7 +167,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
if (instanceDir == null) {
this.instanceDir = SolrResourceLoader.locateSolrHome().toAbsolutePath().normalize();
log.debug("new SolrResourceLoader for deduced Solr Home: '{}'", this.instanceDir);
- } else{
+ } else {
this.instanceDir = instanceDir.toAbsolutePath().normalize();
log.debug("new SolrResourceLoader for directory: '{}'", this.instanceDir);
}
@@ -167,7 +177,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
}
this.classLoader = URLClassLoader.newInstance(new URL[0], parent);
- /*
+ /*
* Skip the lib subdirectory when we are loading from the solr home.
* Otherwise load it, so core lib directories still get loaded.
* The default sharedLib will pick this up later, and if the user has
@@ -264,6 +274,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
/**
* Utility method to get the URLs of all paths under a given directory that match a filter
+ *
* @param libDir the root directory
* @param filter the filter
* @return all matching URLs
@@ -296,8 +307,9 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
/**
* Utility method to get the URLs of all paths under a given directory that match a regex
+ *
* @param libDir the root directory
- * @param regex the regex as a String
+ * @param regex the regex as a String
* @return all matching URLs
* @throws IOException on error
*/
@@ -310,15 +322,17 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
}
});
}
-
- /** Ensures a directory name always ends with a '/'. */
+
+ /**
+ * Ensures a directory name always ends with a '/'.
+ */
public static String normalizeDir(String path) {
- return ( path != null && (!(path.endsWith("/") || path.endsWith("\\"))) )? path + File.separator : path;
+ return (path != null && (!(path.endsWith("/") || path.endsWith("\\")))) ? path + File.separator : path;
}
-
+
public String[] listConfigDir() {
File configdir = new File(getConfigDir());
- if( configdir.exists() && configdir.isDirectory() ) {
+ if (configdir.exists() && configdir.isDirectory()) {
return configdir.list();
} else {
return new String[0];
@@ -328,8 +342,8 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
public String getConfigDir() {
return instanceDir.resolve("conf").toString();
}
-
- public String getDataDir() {
+
+ public String getDataDir() {
return dataDir;
}
@@ -341,23 +355,28 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
* EXPERT
* <p>
* The underlying class loader. Most applications will not need to use this.
+ *
* @return The {@link ClassLoader}
*/
public ClassLoader getClassLoader() {
return classLoader;
}
- /** Opens a schema resource by its name.
+ /**
+ * Opens a schema resource by its name.
* Override this method to customize loading schema resources.
- *@return the stream for the named schema
+ *
+ * @return the stream for the named schema
*/
public InputStream openSchema(String name) throws IOException {
return openResource(name);
}
-
- /** Opens a config resource by its name.
+
+ /**
+ * Opens a config resource by its name.
* Override this method to customize loading config resources.
- *@return the stream for the named configuration
+ *
+ * @return the stream for the named configuration
*/
public InputStream openConfig(String name) throws IOException {
return openResource(name);
@@ -372,14 +391,16 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
throw new IOException("File " + pathToCheck + " is outside resource loader dir " + instanceDir +
"; set -Dsolr.allow.unsafe.resourceloading=true to allow unsafe loading");
}
-
- /** Opens any resource by its name.
+
+ /**
+ * Opens any resource by its name.
* By default, this will look in multiple locations to load the resource:
* $configDir/$resource (if resource is not absolute)
* $CWD/$resource
* otherwise, it will look for it in any jar accessible through the class loader.
* Override this method to customize loading resources.
- *@return the stream for the named resource
+ *
+ * @return the stream for the named resource
*/
@Override
public InputStream openResource(String resource) throws IOException {
@@ -461,22 +482,22 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
* @throws IOException If there is a low-level I/O error.
*/
public List<String> getLines(String resource,
- String encoding) throws IOException {
+ String encoding) throws IOException {
return getLines(resource, Charset.forName(encoding));
}
- public List<String> getLines(String resource, Charset charset) throws IOException{
+ public List<String> getLines(String resource, Charset charset) throws IOException {
try {
return WordlistLoader.getLines(openResource(resource), charset);
} catch (CharacterCodingException ex) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Error loading resource (wrong encoding?): " + resource, ex);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Error loading resource (wrong encoding?): " + resource, ex);
}
}
/*
- * A static map of short class name to fully qualified class name
+ * A static map of short class name to fully qualified class name
*/
private static final Map<String, String> classNameCache = new ConcurrentHashMap<>();
@@ -486,14 +507,14 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
}
// Using this pattern, legacy analysis components from previous Solr versions are identified and delegated to SPI loader:
- private static final Pattern legacyAnalysisPattern =
- Pattern.compile("((\\Q"+base+".analysis.\\E)|(\\Q"+project+".\\E))([\\p{L}_$][\\p{L}\\p{N}_$]+?)(TokenFilter|Filter|Tokenizer|CharFilter)Factory");
+ private static final Pattern legacyAnalysisPattern =
+ Pattern.compile("((\\Q" + base + ".analysis.\\E)|(\\Q" + project + ".\\E))([\\p{L}_$][\\p{L}\\p{N}_$]+?)(TokenFilter|Filter|Tokenizer|CharFilter)Factory");
@Override
public <T> Class<? extends T> findClass(String cname, Class<T> expectedType) {
return findClass(cname, expectedType, empty);
}
-
+
/**
* This method loads a class either with its FQN or a short-name (solr.class-simplename or class-simplename).
* It tries to load the class with the name that is given first and if it fails, it tries all the known
@@ -501,25 +522,25 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
* for the same class faster. The caching is done only if the class is loaded by the webapp classloader and it
* is loaded using a shortname.
*
- * @param cname The name or the short name of the class.
+ * @param cname The name or the short name of the class.
* @param subpackages the packages to be tried if the cname starts with solr.
* @return the loaded class. An exception is thrown if it fails
*/
public <T> Class<? extends T> findClass(String cname, Class<T> expectedType, String... subpackages) {
if (subpackages == null || subpackages.length == 0 || subpackages == packages) {
subpackages = packages;
- String c = classNameCache.get(cname);
- if(c != null) {
+ String c = classNameCache.get(cname);
+ if (c != null) {
try {
return Class.forName(c, true, classLoader).asSubclass(expectedType);
} catch (ClassNotFoundException | ClassCastException e) {
// this can happen if the legacyAnalysisPattern below caches the wrong thing
- log.warn("Unable to load cached class, attempting lookup. name={} shortname={} reason={}", c, cname, e);
+ log.warn( name + " Unable to load cached class, attempting lookup. name={} shortname={} reason={}", c, cname, e);
classNameCache.remove(cname);
}
}
}
-
+
Class<? extends T> clazz = null;
try {
// first try legacy analysis patterns, now replaced by Lucene's Analysis package:
@@ -537,43 +558,43 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
} else {
log.warn("'{}' looks like an analysis factory, but caller requested different class type: {}", cname, expectedType.getName());
}
- } catch (IllegalArgumentException ex) {
+ } catch (IllegalArgumentException ex) {
// ok, we fall back to legacy loading
}
}
-
+
// first try cname == full name
try {
return clazz = Class.forName(cname, true, classLoader).asSubclass(expectedType);
} catch (ClassNotFoundException e) {
- String newName=cname;
+ String newName = cname;
if (newName.startsWith(project)) {
- newName = cname.substring(project.length()+1);
+ newName = cname.substring(project.length() + 1);
}
for (String subpackage : subpackages) {
try {
String name = base + '.' + subpackage + newName;
log.trace("Trying class name " + name);
- return clazz = Class.forName(name,true,classLoader).asSubclass(expectedType);
+ return clazz = Class.forName(name, true, classLoader).asSubclass(expectedType);
} catch (ClassNotFoundException e1) {
// ignore... assume first exception is best.
}
}
-
- throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Error loading class '" + cname + "'", e);
+
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, name +" Error loading class '" + cname + "'", e);
}
-
+
} finally {
if (clazz != null) {
//cache the shortname vs FQN if it is loaded by the webapp classloader and it is loaded
// using a shortname
if (clazz.getClassLoader() == SolrResourceLoader.class.getClassLoader() &&
- !cname.equals(clazz.getName()) &&
- (subpackages.length == 0 || subpackages == packages)) {
+ !cname.equals(clazz.getName()) &&
+ (subpackages.length == 0 || subpackages == packages)) {
//store in the cache
classNameCache.put(cname, clazz.getName());
}
-
+
// print warning if class is deprecated
if (clazz.isAnnotationPresent(Deprecated.class)) {
log.warn("Solr loaded a deprecated plugin/analysis class [{}]. Please consult documentation how to replace it accordingly.",
@@ -582,9 +603,9 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
}
}
}
-
+
static final String empty[] = new String[0];
-
+
@Override
public <T> T newInstance(String name, Class<T> expectedType) {
return newInstance(name, expectedType, empty);
@@ -593,33 +614,32 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
private static final Class[] NO_CLASSES = new Class[0];
private static final Object[] NO_OBJECTS = new Object[0];
- public <T> T newInstance(String cname, Class<T> expectedType, String ... subpackages) {
+ public <T> T newInstance(String cname, Class<T> expectedType, String... subpackages) {
return newInstance(cname, expectedType, subpackages, NO_CLASSES, NO_OBJECTS);
}
- public CoreAdminHandler newAdminHandlerInstance(final CoreContainer coreContainer, String cname, String ... subpackages) {
+ public CoreAdminHandler newAdminHandlerInstance(final CoreContainer coreContainer, String cname, String... subpackages) {
Class<? extends CoreAdminHandler> clazz = findClass(cname, CoreAdminHandler.class, subpackages);
- if( clazz == null ) {
- throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
- "Can not find class: "+cname + " in " + classLoader);
+ if (clazz == null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Can not find class: " + cname + " in " + classLoader);
}
-
+
CoreAdminHandler obj = null;
try {
Constructor<? extends CoreAdminHandler> ctor = clazz.getConstructor(CoreContainer.class);
obj = ctor.newInstance(coreContainer);
- }
- catch (Exception e) {
- throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
- "Error instantiating class: '" + clazz.getName()+"'", e);
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Error instantiating class: '" + clazz.getName() + "'", e);
}
if (!live) {
//TODO: Does SolrCoreAware make sense here since in a multi-core context
// which core are we talking about ?
- if( obj instanceof ResourceLoaderAware ) {
- assertAwareCompatibility( ResourceLoaderAware.class, obj );
- waitingForResources.add( (ResourceLoaderAware)obj );
+ if (obj instanceof ResourceLoaderAware) {
+ assertAwareCompatibility(ResourceLoaderAware.class, obj);
+ waitingForResources.add((ResourceLoaderAware) obj);
}
}
@@ -627,12 +647,11 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
}
-
- public <T> T newInstance(String cName, Class<T> expectedType, String [] subPackages, Class[] params, Object[] args){
+ public <T> T newInstance(String cName, Class<T> expectedType, String[] subPackages, Class[] params, Object[] args) {
Class<? extends T> clazz = findClass(cName, expectedType, subPackages);
- if( clazz == null ) {
- throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
- "Can not find class: "+cName + " in " + classLoader);
+ if (clazz == null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Can not find class: " + cName + " in " + classLoader);
}
T obj = null;
@@ -653,25 +672,25 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
}
} catch (Error err) {
- log.error("Loading Class " + cName + " ("+clazz.getName() + ") triggered serious java error: "
- + err.getClass().getName(), err);
+ log.error("Loading Class " + cName + " (" + clazz.getName() + ") triggered serious java error: "
+ + err.getClass().getName(), err);
throw err;
} catch (Exception e) {
- throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
- "Error instantiating class: '" + clazz.getName()+"'", e);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Error instantiating class: '" + clazz.getName() + "'", e);
}
if (!live) {
- if( obj instanceof SolrCoreAware ) {
- assertAwareCompatibility( SolrCoreAware.class, obj );
- waitingForCore.add( (SolrCoreAware)obj );
+ if (obj instanceof SolrCoreAware) {
+ assertAwareCompatibility(SolrCoreAware.class, obj);
+ waitingForCore.add((SolrCoreAware) obj);
}
- if( obj instanceof ResourceLoaderAware ) {
- assertAwareCompatibility( ResourceLoaderAware.class, obj );
- waitingForResources.add( (ResourceLoaderAware)obj );
+ if (obj instanceof ResourceLoaderAware) {
+ assertAwareCompatibility(ResourceLoaderAware.class, obj);
+ waitingForResources.add((ResourceLoaderAware) obj);
}
- if (obj instanceof SolrInfoBean){
+ if (obj instanceof SolrInfoBean) {
//TODO: Assert here?
infoMBeans.add((SolrInfoBean) obj);
}
@@ -680,12 +699,11 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
return obj;
}
-
+
/**
* Tell all {@link SolrCoreAware} instances about the SolrCore
*/
- public void inform(SolrCore core)
- {
+ public void inform(SolrCore core) {
this.dataDir = core.getDataDir();
// make a copy to avoid potential deadlock of a callback calling newInstance and trying to
@@ -698,22 +716,21 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
waitingForCore.clear();
}
- for( SolrCoreAware aware : arr) {
- aware.inform( core );
+ for (SolrCoreAware aware : arr) {
+ aware.inform(core);
}
}
// this is the last method to be called in SolrCore before the latch is released.
live = true;
}
-
+
/**
* Tell all {@link ResourceLoaderAware} instances about the loader
*/
- public void inform( ResourceLoader loader ) throws IOException
- {
+ public void inform(ResourceLoader loader) throws IOException {
- // make a copy to avoid potential deadlock of a callback adding to the list
+ // make a copy to avoid potential deadlock of a callback adding to the list
ResourceLoaderAware[] arr;
while (waitingForResources.size() > 0) {
@@ -722,7 +739,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
waitingForResources.clear();
}
- for( ResourceLoaderAware aware : arr) {
+ for (ResourceLoaderAware aware : arr) {
aware.inform(loader);
}
}
@@ -730,6 +747,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
/**
* Register any {@link SolrInfoBean}s
+ *
* @param infoRegistry The Info Registry
*/
public void inform(Map<String, SolrInfoBean> infoRegistry) {
@@ -755,7 +773,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
}
}
}
-
+
/**
* Determines the solrhome from the environment.
* Tries JNDI (java:comp/env/solr/home) then system property (solr.solr.home);
@@ -765,12 +783,13 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
/**
* Finds the solrhome based on looking up the value in one of three places:
* <ol>
- * <li>JNDI: via java:comp/env/solr/home</li>
- * <li>The system property solr.solr.home</li>
- * <li>Look in the current working directory for a solr/ directory</li>
+ * <li>JNDI: via java:comp/env/solr/home</li>
+ * <li>The system property solr.solr.home</li>
+ * <li>Look in the current working directory for a solr/ directory</li>
* </ol>
- *
+ * <p>
* The return value is normalized. Normalization essentially means it ends in a trailing slash.
+ *
* @return A normalized solrhome
* @see #normalizeDir(String)
*/
@@ -780,27 +799,27 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
// Try JNDI
try {
Context c = new InitialContext();
- home = (String)c.lookup("java:comp/env/"+project+"/home");
- logOnceInfo("home_using_jndi", "Using JNDI solr.home: "+home );
+ home = (String) c.lookup("java:comp/env/" + project + "/home");
+ logOnceInfo("home_using_jndi", "Using JNDI solr.home: " + home);
} catch (NoInitialContextException e) {
- log.debug("JNDI not configured for "+project+" (NoInitialContextEx)");
+ log.debug("JNDI not configured for " + project + " (NoInitialContextEx)");
} catch (NamingException e) {
- log.debug("No /"+project+"/home in JNDI");
- } catch( RuntimeException ex ) {
+ log.debug("No /" + project + "/home in JNDI");
+ } catch (RuntimeException ex) {
log.warn("Odd RuntimeException while testing for JNDI: " + ex.getMessage());
- }
-
+ }
+
// Now try system property
- if( home == null ) {
+ if (home == null) {
String prop = project + ".solr.home";
home = System.getProperty(prop);
- if( home != null ) {
- logOnceInfo("home_using_sysprop", "Using system property "+prop+": " + home );
+ if (home != null) {
+ logOnceInfo("home_using_sysprop", "Using system property " + prop + ": " + home);
}
}
-
+
// if all else fails, try
- if( home == null ) {
+ if (home == null) {
home = project + '/';
logOnceInfo("home_default", project + " home defaulted to '" + home + "' (could not find system property or JNDI)");
}
@@ -809,22 +828,23 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
/**
* Solr allows users to store arbitrary files in a special directory located directly under SOLR_HOME.
- *
+ * <p>
* This directory is generally created by each node on startup. Files located in this directory can then be
* manipulated using select Solr features (e.g. streaming expressions).
*/
public static final String USER_FILES_DIRECTORY = "userfiles";
+
public static void ensureUserFilesDataDir(Path solrHome) {
final Path userFilesPath = getUserFilesPath(solrHome);
final File userFilesDirectory = new File(userFilesPath.toString());
- if (! userFilesDirectory.exists()) {
+ if (!userFilesDirectory.exists()) {
try {
final boolean created = userFilesDirectory.mkdir();
- if (! created) {
+ if (!created) {
log.warn("Unable to create [{}] directory in SOLR_HOME [{}]. Features requiring this directory may fail.", USER_FILES_DIRECTORY, solrHome);
}
} catch (Exception e) {
- log.warn("Unable to create [" + USER_FILES_DIRECTORY + "] directory in SOLR_HOME [" + solrHome + "]. Features requiring this directory may fail.", e);
+ log.warn("Unable to create [" + USER_FILES_DIRECTORY + "] directory in SOLR_HOME [" + solrHome + "]. Features requiring this directory may fail.", e);
}
}
}
@@ -847,72 +867,73 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
public Path getInstancePath() {
return instanceDir;
}
-
+
/**
* Keep a list of classes that are allowed to implement each 'Aware' interface
*/
private static final Map<Class, Class[]> awareCompatibility;
+
static {
awareCompatibility = new HashMap<>();
- awareCompatibility.put(
- SolrCoreAware.class, new Class[] {
- // DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY
- // VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
- CodecFactory.class,
- DirectoryFactory.class,
- ManagedIndexSchemaFactory.class,
- QueryResponseWriter.class,
- SearchComponent.class,
- ShardHandlerFactory.class,
- SimilarityFactory.class,
- SolrRequestHandler.class,
- UpdateRequestProcessorFactory.class
- }
+ awareCompatibility.put(
+ SolrCoreAware.class, new Class[]{
+ // DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY
+ // VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
+ CodecFactory.class,
+ DirectoryFactory.class,
+ ManagedIndexSchemaFactory.class,
+ QueryResponseWriter.class,
+ SearchComponent.class,
+ ShardHandlerFactory.class,
+ SimilarityFactory.class,
+ SolrRequestHandler.class,
+ UpdateRequestProcessorFactory.class
+ }
);
awareCompatibility.put(
- ResourceLoaderAware.class, new Class[] {
- // DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY
- // VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
- CharFilterFactory.class,
- TokenFilterFactory.class,
- TokenizerFactory.class,
- QParserPlugin.class,
- FieldType.class
- }
+ ResourceLoaderAware.class, new Class[]{
+ // DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY
+ // VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
+ CharFilterFactory.class,
+ TokenFilterFactory.class,
+ TokenizerFactory.class,
+ QParserPlugin.class,
+ FieldType.class
+ }
);
}
/**
* Utility function to throw an exception if the class is invalid
*/
- static void assertAwareCompatibility( Class aware, Object obj )
- {
- Class[] valid = awareCompatibility.get( aware );
- if( valid == null ) {
- throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
- "Unknown Aware interface: "+aware );
- }
- for( Class v : valid ) {
- if( v.isInstance( obj ) ) {
+ static void assertAwareCompatibility(Class aware, Object obj) {
+ Class[] valid = awareCompatibility.get(aware);
+ if (valid == null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Unknown Aware interface: " + aware);
+ }
+ for (Class v : valid) {
+ if (v.isInstance(obj)) {
return;
}
}
StringBuilder builder = new StringBuilder();
- builder.append( "Invalid 'Aware' object: " ).append( obj );
- builder.append( " -- ").append( aware.getName() );
- builder.append( " must be an instance of: " );
- for( Class v : valid ) {
- builder.append( "[" ).append( v.getName() ).append( "] ") ;
+ builder.append("Invalid 'Aware' object: ").append(obj);
+ builder.append(" -- ").append(aware.getName());
+ builder.append(" must be an instance of: ");
+ for (Class v : valid) {
+ builder.append("[").append(v.getName()).append("] ");
}
- throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, builder.toString() );
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, builder.toString());
}
@Override
public void close() throws IOException {
IOUtils.close(classLoader);
}
- public List<SolrInfoBean> getInfoMBeans(){
+
+ public List<SolrInfoBean> getInfoMBeans() {
return Collections.unmodifiableList(infoMBeans);
}
@@ -922,8 +943,8 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
File confFile = new File(loader.getConfigDir(), resourceName);
try {
File parentDir = confFile.getParentFile();
- if ( ! parentDir.isDirectory()) {
- if ( ! parentDir.mkdirs()) {
+ if (!parentDir.isDirectory()) {
+ if (!parentDir.mkdirs()) {
final String msg = "Can't create managed schema directory " + parentDir.getAbsolutePath();
log.error(msg);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg);
diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
index 11c6404..2085221 100644
--- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
@@ -68,6 +68,7 @@ import org.apache.solr.core.RequestParams;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.pkg.PackageListeners;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
@@ -245,8 +246,22 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
if (componentName != null) {
Map map = (Map) val.get(parts.get(1));
if (map != null) {
- val.put(parts.get(1), makeMap(componentName, map.get(componentName)));
+ Object o = map.get(componentName);
+ val.put(parts.get(1), makeMap(componentName, o));
+ if(req.getParams().getBool("meta", false)){
+ for (PackageListeners.Listener listener :
+ req.getCore().getPackageListeners().getListeners()) {
+ PluginInfo info = listener.pluginInfo();
+ if(info.type.equals(parts.get(1)) && info.name.equals(componentName)){
+ if (o instanceof Map) {
+ Map m1 = (Map) o;
+ m1.put("_packageinfo_", listener.getPackageVersion());
+ }
+ }
+ }
+ }
}
+
}
resp.add("config", val);
@@ -488,6 +503,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
}
List errs = CommandOperation.captureErrors(ops);
if (!errs.isEmpty()) {
+ log.error("ERRROR:" +Utils.toJSONString(errs));
throw new ApiBag.ExceptionWithErrObject(SolrException.ErrorCode.BAD_REQUEST,"error processing commands", errs);
}
@@ -495,7 +511,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
if (loader instanceof ZkSolrResourceLoader) {
int latestVersion = ZkController.persistConfigResourceToZooKeeper((ZkSolrResourceLoader) loader, overlay.getZnodeVersion(),
ConfigOverlay.RESOURCE_NAME, overlay.toByteArray(), true);
- log.info("Executed config commands successfully and persisted to ZK {}", ops);
+ log.debug("Executed config commands successfully and persisted to ZK {}", ops);
waitForAllReplicasState(req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName(),
req.getCore().getCoreContainer().getZkController(),
ConfigOverlay.NAME,
@@ -503,7 +519,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
} else {
SolrResourceLoader.persistConfLocally(loader, ConfigOverlay.RESOURCE_NAME, overlay.toByteArray());
req.getCore().getCoreContainer().reload(req.getCore().getName());
- log.info("Executed config commands successfully and persited to File System {}", ops);
+ log.debug("Executed config commands successfully and persited to File System {}", ops);
}
}
@@ -570,6 +586,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
try {
req.getCore().createInitInstance(new PluginInfo(SolrRequestHandler.TYPE, op.getDataMap()), expected, clz, "");
} catch (Exception e) {
+ log.error("Error checking plugin : ",e);
op.addError(e.getMessage());
return false;
}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
new file mode 100644
index 0000000..0267f37
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
@@ -0,0 +1,367 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.pkg;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.solr.api.Command;
+import org.apache.solr.api.EndPoint;
+import org.apache.solr.api.PayloadObj;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.cloud.ZooKeeperException;
+import org.apache.solr.common.util.CommandOperation;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.filestore.PackageStoreAPI;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.util.ReflectMapWriter;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH;
+import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM;
+import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM;
+
+public class PackageAPI {
+ public static final String PACKAGES = "packages";
+ public final boolean enablePackages = Boolean.parseBoolean(System.getProperty("enable.packages", "false"));
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ final CoreContainer coreContainer;
+ private ObjectMapper mapper = new ObjectMapper();
+ private final PackageLoader packageLoader;
+ Packages pkgs;
+
+ public final Edit editAPI = new Edit();
+ public final Read readAPI = new Read();
+
+ public PackageAPI(CoreContainer coreContainer, PackageLoader loader) {
+ this.coreContainer = coreContainer;
+ this.packageLoader = loader;
+ pkgs = new Packages();
+ SolrZkClient zkClient = coreContainer.getZkController().getZkClient();
+ try {
+ registerListener(zkClient);
+ } catch (KeeperException | InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ private void registerListener(SolrZkClient zkClient)
+ throws KeeperException, InterruptedException {
+ String path = SOLR_PKGS_PATH;
+ zkClient.exists(path,
+ new Watcher() {
+
+ @Override
+ public void process(WatchedEvent event) {
+ // session events are not change events, and do not remove the watcher
+ if (Event.EventType.None.equals(event.getType())) {
+ return;
+ }
+ try {
+ synchronized (this) {
+ log.debug("Updating [{}] ... ", path);
+
+ // remake watch
+ final Watcher thisWatch = this;
+ final Stat stat = new Stat();
+ final byte[] data = zkClient.getData(path, thisWatch, stat, true);
+ pkgs = readPkgsFromZk(data, stat);
+ packageLoader.refreshPackageConf();
+ }
+ } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) {
+ log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: [{}]", e.getMessage());
+ } catch (KeeperException e) {
+ log.error("A ZK error has occurred", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.warn("Interrupted", e);
+ }
+ }
+
+ }, true);
+ }
+
+
+ private Packages readPkgsFromZk(byte[] data, Stat stat) throws KeeperException, InterruptedException {
+
+ if (data == null || stat == null) {
+ stat = new Stat();
+ data = coreContainer.getZkController().getZkClient()
+ .getData(ZkStateReader.CLUSTER_PROPS, null, stat, true);
+
+ }
+ Packages packages = null;
+ if (data == null || data.length == 0) {
+ packages = new Packages();
+ } else {
+ try {
+ packages = mapper.readValue(data, Packages.class);
+ packages.znodeVersion = stat.getVersion();
+ } catch (IOException e) {
+ //invalid data in packages
+ //TODO handle properly;
+ return new Packages();
+ }
+ }
+ return packages;
+ }
+
+
+ public static class Packages implements ReflectMapWriter {
+ @JsonProperty
+ public int znodeVersion = -1;
+
+ @JsonProperty
+ public Map<String, List<PkgVersion>> packages = new LinkedHashMap<>();
+
+
+ public Packages copy() {
+ Packages p = new Packages();
+ p.znodeVersion = this.znodeVersion;
+ p.packages = new LinkedHashMap<>();
+ packages.forEach((s, versions) ->
+ p.packages.put(s, new ArrayList<>(versions)));
+ return p;
+ }
+ }
+
+ public static class PkgVersion implements ReflectMapWriter {
+
+ @JsonProperty
+ public String version;
+
+ @JsonProperty
+ public List<String> files;
+
+ public PkgVersion() {
+ }
+
+ public PkgVersion(AddVersion addVersion) {
+ this.version = addVersion.version;
+ this.files = addVersion.files;
+ }
+
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof PkgVersion) {
+ PkgVersion that = (PkgVersion) obj;
+ return Objects.equals(this.version, that.version)
+ && Objects.equals(this.files, that.files);
+
+ }
+ return false;
+ }
+ }
+
+
+ @EndPoint(method = SolrRequest.METHOD.POST,
+ path = "/cluster/package",
+ permission = PACKAGE_EDIT_PERM)
+ public class Edit {
+
+ @Command(name = "add")
+ public void add(SolrQueryRequest req, SolrQueryResponse rsp, PayloadObj<AddVersion> payload) {
+ if (!checkEnabled(payload)) return;
+ AddVersion add = payload.get();
+ if (add.files.isEmpty()) {
+ payload.addError("No files specified");
+ return;
+ }
+ PackageStoreAPI packageStoreAPI = coreContainer.getPackageStoreAPI();
+ packageStoreAPI.validateFiles(add.files, true, s -> payload.addError(s));
+ if (payload.hasError()) return;
+ Packages[] finalState = new Packages[1];
+ try {
+ coreContainer.getZkController().getZkClient().atomicUpdate(SOLR_PKGS_PATH, (stat, bytes) -> {
+ Packages packages = null;
+ try {
+ packages = bytes == null ? new Packages() : mapper.readValue(bytes, Packages.class);
+ packages = packages.copy();
+ } catch (IOException e) {
+ log.error("Error deserializing packages.json", e);
+ packages = new Packages();
+ }
+ packages.packages.computeIfAbsent(add.pkg, Utils.NEW_ARRAYLIST_FUN).add(new PkgVersion(add));
+ packages.znodeVersion = stat.getVersion() + 1;
+ finalState[0] = packages;
+ return Utils.toJSON(packages);
+ });
+ } catch (KeeperException | InterruptedException e) {
+ finalState[0] = null;
+ handleZkErr(e);
+ }
+ if (finalState[0] != null) {
+// succeeded in updating
+ pkgs = finalState[0];
+ notifyAllNodesToSync(pkgs.znodeVersion);
+ packageLoader.refreshPackageConf();
+ }
+
+ }
+
+ @Command(name = "delete")
+ public void del(SolrQueryRequest req, SolrQueryResponse rsp, PayloadObj<DelVersion> payload) {
+ if (!checkEnabled(payload)) return;
+ DelVersion delVersion = payload.get();
+ try {
+ coreContainer.getZkController().getZkClient().atomicUpdate(SOLR_PKGS_PATH, (stat, bytes) -> {
+ Packages packages = null;
+ try {
+ packages = mapper.readValue(bytes, Packages.class);
+ packages = packages.copy();
+ } catch (IOException e) {
+ packages = new Packages();
+ }
+
+ List<PkgVersion> versions = packages.packages.get(delVersion.pkg);
+ if (versions == null || versions.isEmpty()) {
+ payload.addError("No such package: " + delVersion.pkg);
+ return null;// no change
+ }
+ int idxToremove = -1;
+ for (int i = 0; i < versions.size(); i++) {
+ if (Objects.equals(versions.get(i).version, delVersion.version)) {
+ idxToremove = i;
+ break;
+ }
+ }
+ if (idxToremove == -1) {
+ payload.addError("No such version: " + delVersion.version);
+ return null;
+ }
+ versions.remove(idxToremove);
+ packages.znodeVersion = stat.getVersion() + 1;
+ return Utils.toJSON(packages);
+ });
+ } catch (KeeperException | InterruptedException e) {
+ handleZkErr(e);
+
+ }
+
+
+ }
+
+ }
+
+ private boolean checkEnabled(CommandOperation payload) {
+ if (!enablePackages) {
+ payload.addError("Package loading is not enabled , Start your nodes with -Denable.packages=true");
+ return false;
+ }
+ return true;
+ }
+
+ @EndPoint(
+ method = SolrRequest.METHOD.GET,
+ path = {"/cluster/package/",
+ "/cluster/package/{name}"},
+ permission = PACKAGE_READ_PERM
+ )
+ public class Read {
+ @Command()
+ public void get(SolrQueryRequest req, SolrQueryResponse rsp) {
+ int expectedVersion = req.getParams().getInt("expectedVersion", -1);
+ if (expectedVersion != -1) {
+ syncToVersion(expectedVersion);
+ }
+ String name = req.getPathTemplateValues().get("name");
+ if (name == null) {
+ rsp.add("result", pkgs);
+ } else {
+ rsp.add("result", Collections.singletonMap(name, pkgs.packages.get(name)));
+ }
+ }
+
+ private void syncToVersion(int expectedVersion) {
+ for (int i = 0; i < 10; i++) {
+ log.debug("my version is {} , and expected version {}", pkgs.znodeVersion, expectedVersion);
+ if (pkgs.znodeVersion >= expectedVersion) {
+ return;
+ }
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ }
+ try {
+ pkgs = readPkgsFromZk(null, null);
+ } catch (KeeperException | InterruptedException e) {
+ handleZkErr(e);
+
+ }
+
+ }
+
+ }
+
+
+ }
+
+ public static class AddVersion implements ReflectMapWriter {
+ @JsonProperty(value = "package", required = true)
+ public String pkg;
+ @JsonProperty(required = true)
+ public String version;
+ @JsonProperty(required = true)
+ public List<String> files;
+
+ }
+
+ public static class DelVersion implements ReflectMapWriter {
+ @JsonProperty(value = "package", required = true)
+ public String pkg;
+ @JsonProperty(required = true)
+ public String version;
+
+ }
+
+ void notifyAllNodesToSync(int expected) {
+ for (String s : coreContainer.getPackageStoreAPI().shuffledNodes()) {
+ Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(),
+ coreContainer.getZkController().zkStateReader.getBaseUrlForNodeName(s).replace("/solr", "/api") + "/cluster/package?wt=javabin&omitHeader=true&expectedVersion" + expected,
+ Utils.JAVABINCONSUMER);
+ }
+ }
+
+ public void handleZkErr(Exception e) {
+ log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e));
+ }
+
+
+}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java
new file mode 100644
index 0000000..c6ebae5
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.pkg;
+
+import java.lang.ref.WeakReference;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.solr.core.PluginInfo;
+
+public class PackageListeners {
+ // this registry only keeps a weak reference because it does not want to
+ // cause a memory leak if the listener forgets to unregister itself
+ private List<WeakReference<Listener>> listeners = new ArrayList<>();
+
+ public synchronized void addListener(Listener listener) {
+ listeners.add(new WeakReference<>(listener));
+
+ }
+
+ public synchronized void removeListener(Listener listener) {
+ Iterator<WeakReference<Listener>> it = listeners.iterator();
+ while (it.hasNext()) {
+ WeakReference<Listener> ref = it.next();
+ Listener pkgListener = ref.get();
+ if(pkgListener == null || pkgListener == listener){
+ it.remove();
+ }
+
+ }
+
+ }
+
+ synchronized void packagesUpdated(List<PackageLoader.Package> pkgs){
+ for (PackageLoader.Package pkgInfo : pkgs) {
+ invokeListeners(pkgInfo);
+ }
+ }
+
+ private synchronized void invokeListeners(PackageLoader.Package pkg) {
+ for (WeakReference<Listener> ref : listeners) {
+ Listener listener = ref.get();
+ if (listener != null && listener.packageName().equals(pkg.name())) {
+ listener.changed(pkg);
+ }
+ }
+ }
+
+ public List<Listener> getListeners(){
+ List<Listener> result = new ArrayList<>();
+ for (WeakReference<Listener> ref : listeners) {
+ Listener l = ref.get();
+ if(l != null){
+ result.add(l);
+ }
+
+ }
+ return result;
+ }
+
+
+
+ public interface Listener {
+ String packageName();
+
+ PluginInfo pluginInfo();
+
+ void changed(PackageLoader.Package pkg);
+
+ PackageLoader.Package.Version getPackageVersion();
+
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java
new file mode 100644
index 0000000..7efcc88
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.pkg;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.MalformedURLException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrResourceLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PackageLoader {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final CoreContainer coreContainer;
+ private final Map<String, Package> packageClassLoaders = new ConcurrentHashMap<>();
+
+ private PackageAPI.Packages myCopy;
+
+ private PackageAPI packageAPI;
+
+
+ public PackageLoader(CoreContainer coreContainer) {
+ this.coreContainer = coreContainer;
+ packageAPI = new PackageAPI(coreContainer, this);
+ myCopy = packageAPI.pkgs;
+
+ }
+
+ public PackageAPI getPackageAPI() {
+ return packageAPI;
+ }
+
+ public Package getPackage(String key) {
+ return packageClassLoaders.get(key);
+ }
+
+ public Map<String, Package> getPackages() {
+ return Collections.EMPTY_MAP;
+ }
+
+ public void refreshPackageConf() {
+ log.info("{} updated to version {}", ZkStateReader.SOLR_PKGS_PATH, packageAPI.pkgs.znodeVersion);
+
+ List<Package> updated = new ArrayList<>();
+ Map<String, List<PackageAPI.PkgVersion>> modified = getModified(myCopy, packageAPI.pkgs);
+ for (Map.Entry<String, List<PackageAPI.PkgVersion>> e : modified.entrySet()) {
+ if (e.getValue() != null) {
+ Package p = packageClassLoaders.get(e.getKey());
+ if (e.getValue() != null && p == null) {
+ packageClassLoaders.put(e.getKey(), p = new Package(e.getKey()));
+ }
+ p.updateVersions(e.getValue());
+ updated.add(p);
+ } else {
+ Package p = packageClassLoaders.remove(e.getKey());
+ if (p != null) {
+ //other classes are holding to a reference to this objecec
+ // they should know that this is removed
+ p.markDeleted();
+ }
+ }
+ }
+ for (SolrCore core : coreContainer.getCores()) {
+ core.getPackageListeners().packagesUpdated(updated);
+
+ }
+ }
+
+ public Map<String, List<PackageAPI.PkgVersion>> getModified(PackageAPI.Packages old, PackageAPI.Packages newPkgs) {
+ Map<String, List<PackageAPI.PkgVersion>> changed = new HashMap<>();
+ for (Map.Entry<String, List<PackageAPI.PkgVersion>> e : newPkgs.packages.entrySet()) {
+ List<PackageAPI.PkgVersion> versions = old.packages.get(e.getKey());
+ if (versions != null) {
+ if (!Objects.equals(e.getValue(), versions)) {
+ changed.put(e.getKey(), e.getValue());
+ }
+ } else {
+ changed.put(e.getKey(), e.getValue());
+ }
+ }
+ //some packages are deleted altogether
+ for (String s : old.packages.keySet()) {
+ if (!newPkgs.packages.keySet().contains(s)) {
+ changed.put(s, null);
+ }
+ }
+
+ return changed;
+
+ }
+
+ public SolrResourceLoader getResourceLoader(String pkg, String version) {
+ return null;
+ }
+
+
+ public class Package {
+ final String name;
+ final Map<String, Version> myVersions = new ConcurrentHashMap<>();
+ private List<String> sortedVersions = new CopyOnWriteArrayList<>();
+ String latest;
+ private boolean deleted;
+
+
+ public Package(String name) {
+ this.name = name;
+ }
+
+ public boolean isDeleted() {
+ return deleted;
+ }
+
+
+ private synchronized void updateVersions(List<PackageAPI.PkgVersion> modified) {
+ for (PackageAPI.PkgVersion v : modified) {
+ Version version = myVersions.get(v.version);
+ if (version == null) {
+ myVersions.put(v.version, new Version(this, v));
+ sortedVersions.add(v.version);
+ }
+ }
+
+ Set<String> newVersions = new HashSet<>();
+ for (PackageAPI.PkgVersion v : modified) {
+ newVersions.add(v.version);
+ }
+ for (String s : new HashSet<>(myVersions.keySet())) {
+ if (!newVersions.contains(s)) {
+ sortedVersions.remove(s);
+ myVersions.remove(s);
+ }
+ }
+
+ sortedVersions.sort(String::compareTo);
+ if (sortedVersions.size() > 0) {
+ latest = sortedVersions.get(sortedVersions.size() - 1);
+ } else {
+ latest = null;
+ }
+
+ }
+
+
+ public Version getLatest() {
+ return latest == null ? null : myVersions.get(latest);
+ }
+
+ public Version getLatest(String lessThan) {
+ String latest = null;
+ for (String v : (ArrayList<String>) new ArrayList(sortedVersions)) {
+ if (v.compareTo(lessThan) < 1) {
+ latest = v;
+ } else break;
+
+ }
+
+
+ return latest == null ? null : myVersions.get(latest);
+ }
+
+ public String name() {
+ return name;
+ }
+
+ private void markDeleted() {
+ deleted = true;
+ myVersions.clear();
+ sortedVersions.clear();
+ latest = null;
+
+ }
+
+ public class Version implements MapWriter {
+ private final Package parent;
+ private SolrResourceLoader loader;
+
+ private final PackageAPI.PkgVersion version;
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put("package", parent.name());
+ version.writeMap(ew);
+ }
+
+ public Version(Package parent, PackageAPI.PkgVersion v) {
+ this.parent = parent;
+ this.version = v;
+ List<Path> paths = new ArrayList<>();
+ for (String file : version.files) {
+ paths.add(coreContainer.getPackageStoreAPI().getPackageStore().getRealpath(file));
+ }
+
+ try {
+ loader = new SolrResourceLoader(
+ "PACKAGE_LOADER:"+ parent.name()+ ":"+ version,
+ paths,
+ coreContainer.getResourceLoader().getInstancePath(),
+ coreContainer.getResourceLoader().getClassLoader());
+ } catch (MalformedURLException e) {
+ log.error("Could not load classloader ", e);
+ }
+ }
+
+ public String getVersion() {
+ return version.version;
+ }
+
+ public Collection getFiles() {
+ return Collections.unmodifiableList(version.files);
+ }
+
+ public SolrResourceLoader getLoader() {
+ return loader;
+
+ }
+
+ }
+ }
+
+
+}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
new file mode 100644
index 0000000..f0364c4
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.pkg;
+
+import java.lang.invoke.MethodHandles;
+
+import org.apache.solr.core.PluginBag;
+import org.apache.solr.core.PluginInfo;
+import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrCore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final SolrCore core;
+ private final SolrConfig.SolrPluginInfo pluginMeta;
+ private PackageLoader.Package aPackage;
+ private PackageLoader.Package.Version pkgVersion;
+
+
+ public PackagePluginHolder(PluginInfo info, SolrCore core, SolrConfig.SolrPluginInfo pluginMeta) {
+ super(info);
+ this.core = core;
+ this.pluginMeta = pluginMeta;
+
+ reload(aPackage = core.getCoreContainer().getPackageLoader().getPackage(info.pkgName));
+ core.getPackageListeners().addListener(new PackageListeners.Listener() {
+ @Override
+ public String packageName() {
+ return info.pkgName;
+ }
+
+ @Override
+ public PluginInfo pluginInfo() {
+ return info;
+ }
+
+ @Override
+ public void changed(PackageLoader.Package pkg) {
+ reload(pkg);
+
+ }
+
+ @Override
+ public PackageLoader.Package.Version getPackageVersion() {
+ return pkgVersion;
+ }
+
+ });
+ }
+
+
+ private synchronized void reload(PackageLoader.Package pkg) {
+ if(pkgVersion != null && aPackage.getLatest() == pkgVersion ) return;
+
+ if (inst != null) log.info("reloading plugin {} ", pluginInfo.name);
+ PackageLoader.Package.Version newest = pkg.getLatest();
+ if(newest == null) return;
+ Object instance = SolrCore.createInstance(pluginInfo.className,
+ pluginMeta.clazz, pluginMeta.getCleanTag(), core, newest.getLoader());
+ PluginBag.initInstance(instance, pluginInfo);
+ T old = inst;
+ inst = (T) instance;
+ pkgVersion = newest;
+ if (old instanceof AutoCloseable) {
+ AutoCloseable closeable = (AutoCloseable) old;
+ try {
+ closeable.close();
+ } catch (Exception e) {
+ log.error("error closing plugin", e);
+ }
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
index a4c7c0d..b5d4099 100644
--- a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
+++ b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
@@ -53,6 +53,9 @@ public interface PermissionNameProvider {
METRICS_HISTORY_READ_PERM("metrics-history-read", null),
FILESTORE_READ_PERM("filestore-read", null),
FILESTORE_WRITE_PERM("filestore-write", null),
+ PACKAGE_EDIT_PERM("package-edit", null),
+ PACKAGE_READ_PERM("package-read", null),
+
ALL("all", unmodifiableSet(new HashSet<>(asList("*", null))))
;
final String name;
diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
new file mode 100644
index 0000000..5de69cf
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
@@ -0,0 +1,391 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.pkg;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.request.V2Request;
+import org.apache.solr.cloud.ConfigRequest;
+import org.apache.solr.cloud.MiniSolrCloudCluster;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.MapWriterMap;
+import org.apache.solr.common.NavigableObject;
+import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.filestore.TestDistribPackageStore;
+import org.apache.solr.util.LogLevel;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.data.Stat;
+import org.junit.Test;
+
+import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH;
+import static org.apache.solr.common.params.CommonParams.JAVABIN;
+import static org.apache.solr.common.params.CommonParams.WT;
+import static org.apache.solr.core.TestDynamicLoading.getFileContent;
+import static org.apache.solr.filestore.TestDistribPackageStore.readFile;
+
+@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG")
+public class TestPackages extends SolrCloudTestCase {
+
+ @Test
+ public void testPluginLoading() throws Exception {
+ System.setProperty("enable.packages", "true");
+ MiniSolrCloudCluster cluster =
+ configureCluster(4)
+ .withJettyConfig(jetty -> jetty.enableV2(true))
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ try {
+ String FILE1 = "/mypkg/runtimelibs.jar";
+ String FILE2 = "/mypkg/runtimelibs_v2.jar";
+ String COLLECTION_NAME = "testPluginLoadingColl";
+ byte[] derFile = readFile("cryptokeys/pub_key512.der");
+ cluster.getZkClient().makePath("/keys/exe", true);
+ cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
+ postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1,
+ "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ==");
+
+ PackageAPI.AddVersion add = new PackageAPI.AddVersion();
+ add.version = "1.0";
+ add.pkg = "mypkg";
+ add.files = Arrays.asList(new String[]{FILE1});
+ V2Request req = new V2Request.Builder("/cluster/package")
+ .forceV2(true)
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload(Collections.singletonMap("add", add))
+ .build();
+
+ req.process(cluster.getSolrClient());
+
+
+ CollectionAdminRequest
+ .createCollection(COLLECTION_NAME, "conf", 2, 2)
+ .setMaxShardsPerNode(100)
+ .process(cluster.getSolrClient());
+ cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4);
+
+ TestDistribPackageStore.assertResponseValues(10,
+ () -> new V2Request.Builder("/cluster/package").
+ withMethod(SolrRequest.METHOD.GET)
+ .build().process(cluster.getSolrClient()),
+ Utils.makeMap(
+ ":result:packages:mypkg[0]:version", "1.0",
+ ":result:packages:mypkg[0]:files[0]", FILE1
+ ));
+
+ String payload = "{\n" +
+ "'create-requesthandler' : { 'name' : '/runtime', 'class': 'mypkg:org.apache.solr.core.RuntimeLibReqHandler' }," +
+ "'create-searchcomponent' : { 'name' : 'get', 'class': 'mypkg:org.apache.solr.core.RuntimeLibSearchComponent' }," +
+ "'create-queryResponseWriter' : { 'name' : 'json1', 'class': 'mypkg:org.apache.solr.core.RuntimeLibResponseWriter' }" +
+ "}";
+ cluster.getSolrClient().request(new ConfigRequest(payload) {
+ @Override
+ public String getCollection() {
+ return COLLECTION_NAME;
+ }
+ });
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "queryResponseWriter", "json1",
+ "mypkg", "1.0" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "searchComponent", "get",
+ "mypkg", "1.0" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "requestHandler", "/runtime",
+ "mypkg", "1.0" );
+
+
+
+ executeReq( "/" + COLLECTION_NAME + "/runtime?wt=javabin", cluster.getRandomJetty(random()),
+ Utils.JAVABINCONSUMER,
+ Utils.makeMap("class", "org.apache.solr.core.RuntimeLibReqHandler"));
+
+ executeReq( "/" + COLLECTION_NAME + "/get?wt=json", cluster.getRandomJetty(random()),
+ Utils.JSONCONSUMER,
+ Utils.makeMap("class", "org.apache.solr.core.RuntimeLibSearchComponent",
+ "Version","1"));
+
+
+ executeReq( "/" + COLLECTION_NAME + "/runtime?wt=json1", cluster.getRandomJetty(random()),
+ Utils.JSONCONSUMER,
+ Utils.makeMap("wt", "org.apache.solr.core.RuntimeLibResponseWriter"));
+
+ //now upload the second jar
+ postFileAndWait(cluster, "runtimecode/runtimelibs_v2.jar.bin", FILE2,
+ "j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA==");
+
+ //add the version using package API
+ add.version = "1.1";
+ add.pkg = "mypkg";
+ add.files = Arrays.asList(new String[]{FILE2});
+ req.process(cluster.getSolrClient());
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "queryResponseWriter", "json1",
+ "mypkg", "1.1" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "searchComponent", "get",
+ "mypkg", "1.1" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "requestHandler", "/runtime",
+ "mypkg", "1.1" );
+
+ /*executeReq( "/" + COLLECTION_NAME + "/get?wt=json", cluster.getRandomJetty(random()),
+ Utils.JSONCONSUMER,
+ Utils.makeMap("class", "org.apache.solr.core.RuntimeLibSearchComponent",
+ "Version","2"));
+
+ PackageAPI.DelVersion delVersion = new PackageAPI.DelVersion();
+ delVersion.pkg = "mypkg";
+ delVersion.version = "1.1";
+ new V2Request.Builder("/cluster/package")
+ .withMethod(SolrRequest.METHOD.POST)
+ .forceV2(true)
+ .withPayload(delVersion)
+ .build()
+ .process(cluster.getSolrClient());
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "queryResponseWriter", "json1",
+ "mypkg", "1.0" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "searchComponent", "get",
+ "mypkg", "1.0" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "requestHandler", "/runtime",
+ "mypkg", "1.0" );
+
+*/
+ } finally {
+ cluster.shutdown();
+ }
+
+ }
+
+ private void executeReq(String uri, JettySolrRunner jetty, Utils.InputStreamConsumer parser, Map expected) throws Exception {
+ try(HttpSolrClient client = (HttpSolrClient) jetty.newClient()){
+ TestDistribPackageStore.assertResponseValues(10,
+ () -> {
+ Object o = Utils.executeGET(client.getHttpClient(),
+ jetty.getBaseUrl() + uri, parser);
+ if(o instanceof NavigableObject) return (NavigableObject) o;
+ if(o instanceof Map) return new MapWriterMap((Map) o);
+ throw new RuntimeException("Unknown response");
+ }, expected);
+
+ }
+ }
+
+ private void verifyCmponent(SolrClient client, String COLLECTION_NAME,
+ String componentType, String componentName, String pkg, String version) throws Exception {
+ SolrParams params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME,
+ WT, JAVABIN,
+ "componentName", componentName,
+ "meta", "true"));
+
+ String s = "queryResponseWriter";
+ GenericSolrRequest req1 = new GenericSolrRequest(SolrRequest.METHOD.GET,
+ "/config/" + componentType, params);
+ TestDistribPackageStore.assertResponseValues(10,
+ client,
+ req1, Utils.makeMap(
+ ":config:" + componentType + ":" + componentName + ":_packageinfo_:package", pkg,
+ ":config:" + componentType + ":" + componentName + ":_packageinfo_:version", version
+ ));
+ }
+
+ @Test
+ public void testAPI() throws Exception {
+ System.setProperty("enable.packages", "true");
+ MiniSolrCloudCluster cluster =
+ configureCluster(4)
+ .withJettyConfig(jetty -> jetty.enableV2(true))
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ try {
+ String errPath = "/error/details[0]/errorMessages[0]";
+ String FILE1 = "/mypkg/v.0.12/jar_a.jar";
+ String FILE2 = "/mypkg/v.0.12/jar_b.jar";
+ String FILE3 = "/mypkg/v.0.13/jar_a.jar";
+
+ PackageAPI.AddVersion add = new PackageAPI.AddVersion();
+ add.version = "0.12";
+ add.pkg = "test_pkg";
+ add.files = Arrays.asList(new String[]{FILE1, FILE2});
+ V2Request req = new V2Request.Builder("/cluster/package")
+ .forceV2(true)
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload(Collections.singletonMap("add", add))
+ .build();
+
+ //the files is not yet there. The command should fail with error saying "No such file"
+ expectError(req, cluster.getSolrClient(), errPath, "No such file :");
+
+
+ //post the jar file. No signature is sent
+ postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1, null);
+
+
+ add.files = Arrays.asList(new String[]{FILE1});
+ expectError(req, cluster.getSolrClient(), errPath,
+ FILE1 + " has no signature");
+ //now we upload the keys
+ byte[] derFile = readFile("cryptokeys/pub_key512.der");
+ cluster.getZkClient().makePath("/keys/exe", true);
+ cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
+ //and upload the same file with a different name but it has proper signature
+ postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE2,
+ "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ==");
+ // with correct signature
+ //after uploading the file, let's delete the keys to see if we get proper error message
+ cluster.getZkClient().delete("/keys/exe/pub_key512.der", -1, true);
+ add.files = Arrays.asList(new String[]{FILE2});
+ expectError(req, cluster.getSolrClient(), errPath,
+ "ZooKeeper does not have any public keys");
+
+ //Now lets' put the keys back
+ cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
+
+ //this time we have a file with proper signature, public keys are in ZK
+ // so the add {} command should succeed
+ req.process(cluster.getSolrClient());
+
+ //Now verify the data in ZK
+ TestDistribPackageStore.assertResponseValues(1,
+ () -> new MapWriterMap((Map) Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH,
+ null, new Stat(), true))),
+ Utils.makeMap(
+ ":packages:test_pkg[0]:version", "0.12",
+ ":packages:test_pkg[0]:files[0]", FILE1
+ ));
+
+ //post a new jar with a proper signature
+ postFileAndWait(cluster, "runtimecode/runtimelibs_v2.jar.bin", FILE3,
+ "j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA==");
+
+
+ //this time we are adding the second version of the package (0.13)
+ add.version = "0.13";
+ add.pkg = "test_pkg";
+ add.files = Arrays.asList(new String[]{FILE3});
+
+ //this request should succeed
+ req.process(cluster.getSolrClient());
+ //no verify the data (/packages.json) in ZK
+ TestDistribPackageStore.assertResponseValues(1,
+ () -> new MapWriterMap((Map) Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH,
+ null, new Stat(), true))),
+ Utils.makeMap(
+ ":packages:test_pkg[1]:version", "0.13",
+ ":packages:test_pkg[1]:files[0]", FILE3
+ ));
+
+ //Now we will just delete one version
+ PackageAPI.DelVersion delVersion = new PackageAPI.DelVersion();
+ delVersion.version = "0.1";//this version does not exist
+ delVersion.pkg = "test_pkg";
+ req = new V2Request.Builder("/cluster/package")
+ .forceV2(true)
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload(Collections.singletonMap("delete", delVersion))
+ .build();
+
+ //we are expecting an error
+ expectError(req, cluster.getSolrClient(), errPath, "No such version:");
+
+ delVersion.version = "0.12";//correct version. Should succeed
+ req.process(cluster.getSolrClient());
+ //Verify with ZK that the data is correcy
+ TestDistribPackageStore.assertResponseValues(1,
+ () -> new MapWriterMap((Map) Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH,
+ null, new Stat(), true))),
+ Utils.makeMap(
+ ":packages:test_pkg[0]:version", "0.13",
+ ":packages:test_pkg[0]:files[0]", FILE2
+ ));
+
+
+ //So far we have been verifying the details with ZK directly
+ //use the package read API to verify with each node that it has the correct data
+ for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
+ String path = jetty.getBaseUrl().toString().replace("/solr", "/api") + "/cluster/package?wt=javabin";
+ TestDistribPackageStore.assertResponseValues(10, new Callable<NavigableObject>() {
+ @Override
+ public NavigableObject call() throws Exception {
+ try (HttpSolrClient solrClient = (HttpSolrClient) jetty.newClient()) {
+ return (NavigableObject) Utils.executeGET(solrClient.getHttpClient(), path, Utils.JAVABINCONSUMER);
+ }
+ }
+ }, Utils.makeMap(
+ ":result:packages:test_pkg[0]:version", "0.13",
+ ":result:packages:test_pkg[0]:files[0]", FILE3
+ ));
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ static void postFileAndWait(MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception {
+ ByteBuffer fileContent = getFileContent(fname);
+ String sha512 = DigestUtils.sha512Hex(fileContent.array());
+
+ TestDistribPackageStore.postFile(cluster.getSolrClient(),
+ fileContent,
+ path, sig);// has file, but no signature
+
+ TestDistribPackageStore.waitForAllNodesHaveFile(cluster, path, Utils.makeMap(
+ ":files:" + path + ":sha512",
+ sha512
+ ), false);
+ }
+
+ private void expectError(V2Request req, SolrClient client, String errPath, String expectErrorMsg) throws IOException, SolrServerException {
+ try {
+ req.process(client);
+ fail("should have failed with message : " + expectErrorMsg);
+ } catch (BaseHttpSolrClient.RemoteExecutionException e) {
+ String msg = e.getMetaData()._getStr(errPath, "");
+ assertTrue("should have failed with message: " + expectErrorMsg + "actual message : " + msg,
+ msg.contains(expectErrorMsg)
+ );
+ }
+ }
+}
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
index 98f0d9a..b8105ab 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
@@ -34,6 +34,7 @@ import java.nio.file.Path;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.RejectedExecutionException;
+import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.regex.Pattern;
@@ -353,6 +354,10 @@ public class SolrZkClient implements Closeable {
}
public void atomicUpdate(String path, Function<byte[], byte[]> editor) throws KeeperException, InterruptedException {
+ atomicUpdate(path, (stat, bytes) -> editor.apply(bytes));
+ }
+
+ public void atomicUpdate(String path, BiFunction<Stat , byte[], byte[]> editor) throws KeeperException, InterruptedException {
for (; ; ) {
byte[] modified = null;
byte[] zkData = null;
@@ -360,7 +365,7 @@ public class SolrZkClient implements Closeable {
try {
if (exists(path, true)) {
zkData = getData(path, null, s, true);
- modified = editor.apply(zkData);
+ modified = editor.apply(s, zkData);
if (modified == null) {
//no change , no need to persist
return;
@@ -368,7 +373,7 @@ public class SolrZkClient implements Closeable {
setData(path, modified, s.getVersion(), true);
break;
} else {
- modified = editor.apply(null);
+ modified = editor.apply(s,null);
if (modified == null) {
//no change , no need to persist
return;
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index dcf7d9e..5dea5b0 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -118,6 +118,7 @@ public class ZkStateReader implements SolrCloseable {
public static final String SOLR_AUTOSCALING_TRIGGER_STATE_PATH = "/autoscaling/triggerState";
public static final String SOLR_AUTOSCALING_NODE_ADDED_PATH = "/autoscaling/nodeAdded";
public static final String SOLR_AUTOSCALING_NODE_LOST_PATH = "/autoscaling/nodeLost";
+ public static final String SOLR_PKGS_PATH = "/packages.json";
public static final String DEFAULT_SHARD_PREFERENCES = "defaultShardPreferences";
public static final String REPLICATION_FACTOR = "replicationFactor";