You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2019/11/06 08:45:24 UTC
[lucene-solr] branch branch_8x updated: SOLR-13822: Isolated
Classloading from packages (#997)
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new 37059eb SOLR-13822: Isolated Classloading from packages (#997)
37059eb is described below
commit 37059eb5949cca064df73c0486e088b9d201775f
Author: Noble Paul <no...@users.noreply.github.com>
AuthorDate: Wed Nov 6 19:45:14 2019 +1100
SOLR-13822: Isolated Classloading from packages (#997)
* SOLR-13821: A Package store to store and load package artifacts
* SOLR-13822: A Package management system with the following features. A packages.json in ZK to store
the configuration, APIs to read/edit them and isolated classloaders to load the classes from
those packages if the 'class' attribute is prefixed with `<package-name>:`
* SOLR-13841: Provide mappings for jackson annotation @JsonProperty to use Jackson deserializer
---
solr/CHANGES.txt | 10 +-
.../src/java/org/apache/solr/api/AnnotatedApi.java | 74 +--
.../java/org/apache/solr/core/CoreContainer.java | 16 +
.../src/java/org/apache/solr/core/PluginBag.java | 28 +-
.../src/java/org/apache/solr/core/PluginInfo.java | 44 +-
.../java/org/apache/solr/core/RequestParams.java | 9 +
.../src/java/org/apache/solr/core/SolrCore.java | 22 +-
.../org/apache/solr/core/SolrResourceLoader.java | 28 +-
.../apache/solr/filestore/DistribPackageStore.java | 73 ++-
.../org/apache/solr/filestore/PackageStore.java | 6 +-
.../org/apache/solr/filestore/PackageStoreAPI.java | 88 +++-
.../org/apache/solr/handler/SolrConfigHandler.java | 39 +-
.../solr/handler/component/SearchHandler.java | 60 ++-
.../src/java/org/apache/solr/pkg/PackageAPI.java | 388 ++++++++++++++++
.../java/org/apache/solr/pkg/PackageListeners.java | 111 +++++
.../java/org/apache/solr/pkg/PackageLoader.java | 300 +++++++++++++
.../org/apache/solr/pkg/PackagePluginHolder.java | 123 +++++
.../solr/security/PermissionNameProvider.java | 3 +
.../processor/UpdateRequestProcessorChain.java | 43 +-
.../src/java/org/apache/solr/util/CryptoKeys.java | 24 +-
.../solr/util/SolrJacksonAnnotationInspector.java | 89 ++++
.../solr/filestore/TestDistribPackageStore.java | 99 ++--
.../solr/handler/admin/TestApiFramework.java | 2 +-
.../src/test/org/apache/solr/pkg/TestPackages.java | 496 +++++++++++++++++++++
.../apache/solr/update/processor/RuntimeUrp.java | 2 +-
.../solr/util/TestSolrJacksonAnnotation.java | 82 ++++
.../solr/client/solrj/io/stream/ZplotStream.java | 2 +-
.../solr/client/solrj/request/beans/Package.java | 46 ++
.../client/solrj/request/beans/pakage-info.java | 23 +
.../solr/common/annotation/JsonProperty.java | 39 ++
.../org/apache/solr/common/cloud/SolrZkClient.java | 9 +-
.../apache/solr/common/cloud/ZkStateReader.java | 1 +
.../apache/solr/common/util/JsonSchemaCreator.java | 85 ++++
.../apache/solr/common}/util/ReflectMapWriter.java | 5 +-
.../org/apache/solr/client/solrj/io/TestLang.java | 15 +-
35 files changed, 2256 insertions(+), 228 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 01cf378..304c14e 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -37,12 +37,18 @@ Upgrade Notes
New Features
---------------------
+* SOLR-13821: A Package store to store and load package artifacts (noble, Ishan Chattopadhyaya)
+
+* SOLR-13822: A Package management system with the following features. A packages.json in ZK to store
+ the configuration, APIs to read/edit them and isolated classloaders to load the classes from
+ hose packages if the 'class' attribute is prefixed with `<package-name>:` (noble, Ishan Chattopadhyaya)
+
* SOLR-10786: Add DBSCAN clustering Streaming Evaluator (Joel Bernstein)
Improvements
---------------------
-*SOLR-13731: javabin must support a 1:1 mapping of the JSON update format (noble)
+* SOLR-13731: 'javabin' must support a 1:1 mapping of the JSON update format (noble)
* SOLR-13831: Support defining arbitrary autoscaling simulation scenarios. (ab)
@@ -85,6 +91,8 @@ Other Changes
* SOLR-12769: Fix incorrect documentation for 'delete' op in Request parameters API (Alexandre Rafalovitch, Munendra S N)
+* SOLR-13841: Provide mappings for jackson annotation @JsonProperty to use Jackson deserializer (noble)
+
* SOLR-13824: Strictly reject anything after JSON in most APIs (Mikhail Khludnev, Munendra S N)
================== 8.3.0 ==================
diff --git a/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java b/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java
index e9073ae..ac9fdd0 100644
--- a/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java
+++ b/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java
@@ -19,7 +19,6 @@ package org.apache.solr.api;
import java.lang.invoke.MethodHandles;
-import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
@@ -33,18 +32,19 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SpecProvider;
import org.apache.solr.common.util.CommandOperation;
+import org.apache.solr.common.util.JsonSchemaCreator;
import org.apache.solr.common.util.Utils;
import org.apache.solr.common.util.ValidatingJsonMap;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.security.AuthorizationContext;
import org.apache.solr.security.PermissionNameProvider;
+import org.apache.solr.util.SolrJacksonAnnotationInspector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,7 +61,7 @@ import org.slf4j.LoggerFactory;
public class AnnotatedApi extends Api implements PermissionNameProvider {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- public static final String ERR ="Error executing commands :";
+ public static final String ERR = "Error executing commands :";
private EndPoint endPoint;
private Map<String, Cmd> commands = new HashMap<>();
private final Api fallback;
@@ -137,7 +137,7 @@ public class AnnotatedApi extends Api implements PermissionNameProvider {
}
}
- List<CommandOperation> cmds = req.getCommands(false);
+ List<CommandOperation> cmds = req.getCommands(true);
boolean allExists = true;
for (CommandOperation cmd : cmds) {
if (!commands.containsKey(cmd.name)) {
@@ -161,8 +161,8 @@ public class AnnotatedApi extends Api implements PermissionNameProvider {
List<Map> errs = CommandOperation.captureErrors(cmds);
if (!errs.isEmpty()) {
- log.error(ERR+ Utils.toJSONString(errs));
- throw new ApiBag.ExceptionWithErrObject(SolrException.ErrorCode.BAD_REQUEST, ERR , errs);
+ log.error(ERR + Utils.toJSONString(errs));
+ throw new ApiBag.ExceptionWithErrObject(SolrException.ErrorCode.BAD_REQUEST, ERR, errs);
}
}
@@ -178,6 +178,7 @@ public class AnnotatedApi extends Api implements PermissionNameProvider {
Cmd(Command command, Object obj, Method method) {
+ mapper.setAnnotationIntrospector(SolrJacksonAnnotationInspector.INSTANCE);
if (Modifier.isPublic(method.getModifiers())) {
this.command = command;
this.obj = obj;
@@ -227,6 +228,7 @@ public class AnnotatedApi extends Api implements PermissionNameProvider {
}
if (isWrappedInPayloadObj) {
PayloadObj<Object> payloadObj = new PayloadObj<>(cmd.name, cmd.getCommandData(), o);
+ cmd = payloadObj;
method.invoke(obj, req, rsp, payloadObj);
} else {
method.invoke(obj, req, rsp, o);
@@ -239,71 +241,33 @@ public class AnnotatedApi extends Api implements PermissionNameProvider {
} catch (SolrException se) {
+ log.error("Error executing command ", se);
throw se;
} catch (InvocationTargetException ite) {
+ log.error("Error executing command ", ite);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, ite.getCause());
} catch (Exception e) {
+ log.error("Error executing command : ", e);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
}
- private static final Map<Class, String> primitives = new HashMap<>();
-
- static {
- primitives.put(String.class, "string");
- primitives.put(Integer.class, "integer");
- primitives.put(int.class, "integer");
- primitives.put(Float.class, "number");
- primitives.put(float.class, "number");
- primitives.put(Double.class, "number");
- primitives.put(double.class, "number");
- primitives.put(Boolean.class, "boolean");
- primitives.put(List.class, "array");
- }
-
-
public static Map<String, Object> createSchema(Method m) {
Type[] types = m.getGenericParameterTypes();
if (types.length == 3) {
- return createSchemaFromType(types[2]);
-
- }
- return null;
- }
-
- private static Map<String, Object> createSchemaFromType(Type t) {
- Map<String, Object> map = new LinkedHashMap<>();
- if (t instanceof ParameterizedType) {
- ParameterizedType typ = (ParameterizedType) t;
- if (typ.getRawType() == PayloadObj.class) {
- t = typ.getActualTypeArguments()[0];
+ Type t = types[2];
+ if (t instanceof ParameterizedType) {
+ ParameterizedType typ = (ParameterizedType) t;
+ if (typ.getRawType() == PayloadObj.class) {
+ t = typ.getActualTypeArguments()[0];
+ }
}
- }
+ return JsonSchemaCreator.getSchema(t);
- if (primitives.containsKey(t)) {
- map.put("type", primitives.get(t));
- } else if (t instanceof ParameterizedType && ((ParameterizedType) t).getRawType() == List.class) {
- Type typ = ((ParameterizedType) t).getActualTypeArguments()[0];
- map.put("type", "array");
- map.put("items", createSchemaFromType(typ));
- } else {
- createObjectSchema((Class) t, map);
- }
- return map;
- }
-
- private static void createObjectSchema(Class klas, Map<String, Object> map) {
- map.put("type", "object");
- Map<String, Object> props = new HashMap<>();
- map.put("properties", props);
- for (Field fld : klas.getDeclaredFields()) {
- JsonProperty p = fld.getAnnotation(JsonProperty.class);
- if (p == null) continue;
- props.put(p.value(), createSchemaFromType(fld.getGenericType()));
}
+ return null;
}
-
}
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 0d83982..3ff86d2 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -101,6 +101,7 @@ import org.apache.solr.metrics.SolrCoreMetricManager;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
+import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.search.SolrFieldCacheBean;
@@ -224,6 +225,7 @@ public class CoreContainer {
protected volatile AutoscalingHistoryHandler autoscalingHistoryHandler;
private PackageStoreAPI packageStoreAPI;
+ private PackageLoader packageLoader;
// Bits for the state variable.
@@ -583,6 +585,13 @@ public class CoreContainer {
return replayUpdatesExecutor;
}
+ public PackageLoader getPackageLoader() {
+ return packageLoader;
+ }
+
+ public PackageStoreAPI getPackageStoreAPI() {
+ return packageStoreAPI;
+ }
//-------------------------------------------------------------------
// Initialization / Cleanup
//-------------------------------------------------------------------
@@ -735,8 +744,12 @@ public class CoreContainer {
if (isZooKeeperAware()) {
metricManager.loadClusterReporters(metricReporters, this);
+ packageLoader = new PackageLoader(this);
+ containerHandlers.getApiBag().register(new AnnotatedApi(packageLoader.getPackageAPI().editAPI), Collections.EMPTY_MAP);
+ containerHandlers.getApiBag().register(new AnnotatedApi(packageLoader.getPackageAPI().readAPI), Collections.EMPTY_MAP);
}
+
// setup executor to load cores in parallel
ExecutorService coreLoadExecutor = MetricUtils.instrumentedExecutorService(
ExecutorUtil.newMDCAwareFixedThreadPool(
@@ -1049,6 +1062,9 @@ public class CoreContainer {
log.warn("Exception while closing auditlogger plugin.", e);
}
+ if(packageLoader != null){
+ org.apache.lucene.util.IOUtils.closeWhileHandlingException(packageLoader);
+ }
org.apache.lucene.util.IOUtils.closeWhileHandlingException(loader); // best effort
}
diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java
index fa2c3e3..f547d10 100644
--- a/solr/core/src/java/org/apache/solr/core/PluginBag.java
+++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java
@@ -29,6 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
@@ -43,8 +44,10 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.handler.component.SearchComponent;
+import org.apache.solr.pkg.PackagePluginHolder;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
+import org.apache.solr.update.processor.UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder;
import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
import org.apache.solr.util.CryptoKeys;
import org.apache.solr.util.SimplePostTool;
@@ -97,7 +100,7 @@ public class PluginBag<T> implements AutoCloseable {
this(klass, core, false);
}
- static void initInstance(Object inst, PluginInfo info) {
+ public static void initInstance(Object inst, PluginInfo info) {
if (inst instanceof PluginInfoInitialized) {
((PluginInfoInitialized) inst).init(info);
} else if (inst instanceof NamedListInitializedPlugin) {
@@ -138,14 +141,23 @@ public class PluginBag<T> implements AutoCloseable {
log.debug("{} : '{}' created with startup=lazy ", meta.getCleanTag(), info.name);
return new LazyPluginHolder<T>(meta, info, core, core.getResourceLoader(), false);
} else {
- T inst = core.createInstance(info.className, (Class<T>) meta.clazz, meta.getCleanTag(), null, core.getResourceLoader());
- initInstance(inst, info);
- return new PluginHolder<>(info, inst);
+ if (info.pkgName != null) {
+ PackagePluginHolder<T> holder = new PackagePluginHolder<>(info, core, meta);
+ return meta.clazz == UpdateRequestProcessorFactory.class ?
+ new PluginHolder(info, new LazyUpdateProcessorFactoryHolder(holder)) :
+ holder;
+ } else {
+ T inst = core.createInstance(info.className, (Class<T>) meta.clazz, meta.getCleanTag(), null, core.getResourceLoader(info.pkgName));
+ initInstance(inst, info);
+ return new PluginHolder<>(info, inst);
+ }
}
}
- /** make a plugin available in an alternate name. This is an internal API and not for public use
- * @param src key in which the plugin is already registered
+ /**
+ * make a plugin available in an alternate name. This is an internal API and not for public use
+ *
+ * @param src key in which the plugin is already registered
* @param target the new key in which the plugin should be aliased to. If target exists already, the alias fails
* @return flag if the operation is successful or not
*/
@@ -340,8 +352,8 @@ public class PluginBag<T> implements AutoCloseable {
* An indirect reference to a plugin. It just wraps a plugin instance.
* subclasses may choose to lazily load the plugin
*/
- public static class PluginHolder<T> implements AutoCloseable {
- private T inst;
+ public static class PluginHolder<T> implements Supplier<T>, AutoCloseable {
+ protected T inst;
protected final PluginInfo pluginInfo;
boolean registerAPI = false;
diff --git a/solr/core/src/java/org/apache/solr/core/PluginInfo.java b/solr/core/src/java/org/apache/solr/core/PluginInfo.java
index 1bc85ae..bb290e1 100644
--- a/solr/core/src/java/org/apache/solr/core/PluginInfo.java
+++ b/solr/core/src/java/org/apache/solr/core/PluginInfo.java
@@ -16,14 +16,20 @@
*/
package org.apache.solr.core;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.solr.common.MapSerializable;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.Pair;
import org.apache.solr.util.DOMUtil;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
-import java.util.*;
-
import static java.util.Arrays.asList;
import static java.util.Collections.unmodifiableList;
import static java.util.Collections.unmodifiableMap;
@@ -35,27 +41,51 @@ import static org.apache.solr.schema.FieldType.CLASS_NAME;
*
*/
public class PluginInfo implements MapSerializable {
- public final String name, className, type;
+ public final String name, className, type, pkgName;
public final NamedList initArgs;
public final Map<String, String> attributes;
public final List<PluginInfo> children;
private boolean isFromSolrConfig;
+
+
public PluginInfo(String type, Map<String, String> attrs, NamedList initArgs, List<PluginInfo> children) {
this.type = type;
this.name = attrs.get(NAME);
- this.className = attrs.get(CLASS_NAME);
+ Pair<String, String> parsed = parseClassName(attrs.get(CLASS_NAME));
+ this.className = parsed.second();
+ this.pkgName = parsed.first();
this.initArgs = initArgs;
attributes = unmodifiableMap(attrs);
this.children = children == null ? Collections.<PluginInfo>emptyList(): unmodifiableList(children);
isFromSolrConfig = false;
}
+ /** class names can be prefixed with package name e.g: my_package:my.pkg.Class
+ * This checks if it is a package name prefixed classname.
+ * the return value has first = package name & second = class name
+ */
+ static Pair<String,String > parseClassName(String name) {
+ String pkgName = null;
+ String className = name;
+ if (name != null) {
+ int colonIdx = name.indexOf(':');
+ if (colonIdx > -1) {
+ pkgName = name.substring(0, colonIdx);
+ className = name.substring(colonIdx + 1);
+ }
+ }
+ return new Pair<>(pkgName, className);
+
+ }
+
public PluginInfo(Node node, String err, boolean requireName, boolean requireClass) {
type = node.getNodeName();
name = DOMUtil.getAttr(node, NAME, requireName ? err : null);
- className = DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null);
+ Pair<String, String> parsed = parseClassName(DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null));
+ className = parsed.second();
+ pkgName = parsed.first();
initArgs = DOMUtil.childNodesToNamedList(node);
attributes = unmodifiableMap(DOMUtil.toMap(node.getAttributes()));
children = loadSubPlugins(node);
@@ -85,7 +115,9 @@ public class PluginInfo implements MapSerializable {
}
this.type = type;
this.name = (String) m.get(NAME);
- this.className = (String) m.get(CLASS_NAME);
+ Pair<String, String> parsed = parseClassName((String) m.get(CLASS_NAME));
+ this.className = parsed.second();
+ this.pkgName = parsed.first();
attributes = unmodifiableMap(m);
this.children = Collections.<PluginInfo>emptyList();
isFromSolrConfig = true;
diff --git a/solr/core/src/java/org/apache/solr/core/RequestParams.java b/solr/core/src/java/org/apache/solr/core/RequestParams.java
index 50a4fd0..d1f7f3d 100644
--- a/solr/core/src/java/org/apache/solr/core/RequestParams.java
+++ b/solr/core/src/java/org/apache/solr/core/RequestParams.java
@@ -250,9 +250,18 @@ public class RequestParams implements MapSerializable {
return m1;
}
+ /**
+ * @param type one of defaults, appends, invariants
+ */
public VersionedParams getParams(String type) {
return paramsMap.get(type);
}
+
+ /**get the raw map
+ */
+ public Map<String, Object> get() {
+ return defaults;
+ }
}
public static class VersionedParams extends MapSolrParams {
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index aff53a3..3c81eca 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -110,6 +110,8 @@ import org.apache.solr.logging.MDCLoggingContext;
import org.apache.solr.metrics.SolrCoreMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
+import org.apache.solr.pkg.PackageListeners;
+import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.BinaryResponseWriter;
@@ -238,6 +240,8 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
public volatile boolean indexEnabled = true;
public volatile boolean readOnly = false;
+ private PackageListeners packageListeners = new PackageListeners(this);
+
public Set<String> getMetricNames() {
return metricNames;
}
@@ -262,6 +266,10 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
return restManager;
}
+ public PackageListeners getPackageListeners() {
+ return packageListeners;
+ }
+
static int boolean_query_max_clause_count = Integer.MIN_VALUE;
private ExecutorService coreAsyncTaskExecutor = ExecutorUtil.newMDCAwareCachedThreadPool("Core Async Task");
@@ -275,6 +283,18 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
return resourceLoader;
}
+ /** Gets the SolrResourceLoader for a given package
+ * @param pkg The package name
+ */
+ public SolrResourceLoader getResourceLoader(String pkg) {
+ if (pkg == null) {
+ return resourceLoader;
+ }
+ PackageLoader.Package aPackage = coreContainer.getPackageLoader().getPackage(pkg);
+ PackageLoader.Package.Version latest = aPackage.getLatest();
+ return latest.getLoader();
+ }
+
/**
* Gets the configuration resource name used by this core instance.
*
@@ -857,7 +877,7 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
public <T extends Object> T createInitInstance(PluginInfo info, Class<T> cast, String msg, String defClassName) {
if (info == null) return null;
- T o = createInstance(info.className == null ? defClassName : info.className, cast, msg, this, getResourceLoader());
+ T o = createInstance(info.className == null ? defClassName : info.className, cast, msg, this, getResourceLoader(info.pkgName));
if (o instanceof PluginInfoInitialized) {
((PluginInfoInitialized) o).init(info);
} else if (o instanceof NamedListInitializedPlugin) {
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
index f27edbc..890e98a 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
@@ -28,6 +28,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.Constructor;
+import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.charset.CharacterCodingException;
@@ -52,6 +53,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.util.CharFilterFactory;
import org.apache.lucene.analysis.util.ResourceLoader;
@@ -95,7 +97,10 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
};
private static final java.lang.String SOLR_CORE_NAME = "solr.core.name";
private static Set<String> loggedOnce = new ConcurrentSkipListSet<>();
+ private static final Charset UTF_8 = StandardCharsets.UTF_8;
+
+ private String name = "";
protected URLClassLoader classLoader;
private final Path instanceDir;
private String dataDir;
@@ -103,7 +108,6 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
private final List<SolrCoreAware> waitingForCore = Collections.synchronizedList(new ArrayList<SolrCoreAware>());
private final List<SolrInfoBean> infoMBeans = Collections.synchronizedList(new ArrayList<SolrInfoBean>());
private final List<ResourceLoaderAware> waitingForResources = Collections.synchronizedList(new ArrayList<ResourceLoaderAware>());
- private static final Charset UTF_8 = StandardCharsets.UTF_8;
private final Properties coreProperties;
@@ -138,6 +142,16 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
this(instanceDir, parent, null);
}
+ public SolrResourceLoader(String name, List<Path> classpath, Path instanceDir, ClassLoader parent) throws MalformedURLException {
+ this(instanceDir, parent);
+ this.name = name;
+ for (Path path : classpath) {
+ addToClassLoader(path.toUri().normalize().toURL());
+ }
+
+ }
+
+
public SolrResourceLoader(Path instanceDir) {
this(instanceDir, null, null);
}
@@ -502,15 +516,15 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
public <T> Class<? extends T> findClass(String cname, Class<T> expectedType, String... subpackages) {
if (subpackages == null || subpackages.length == 0 || subpackages == packages) {
subpackages = packages;
- String c = classNameCache.get(cname);
- if(c != null) {
+ String c = classNameCache.get(cname);
+ if (c != null) {
try {
return Class.forName(c, true, classLoader).asSubclass(expectedType);
- } catch (ClassNotFoundException e) {
- //this is unlikely
- log.error("Unable to load cached class-name : "+ c +" for shortname : "+cname + e);
+ } catch (ClassNotFoundException | ClassCastException e) {
+ // this can happen if the legacyAnalysisPattern below caches the wrong thing
+ log.warn( name + " Unable to load cached class, attempting lookup. name={} shortname={} reason={}", c, cname, e);
+ classNameCache.remove(cname);
}
-
}
}
diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
index d4e1f77..cd653e7 100644
--- a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
+++ b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
@@ -29,12 +29,10 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Consumer;
import java.util.function.Predicate;
@@ -45,7 +43,6 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.BlobRepository;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.filestore.PackageStoreAPI.MetaData;
import org.apache.zookeeper.server.ByteBufferInputStream;
@@ -62,6 +59,7 @@ public class DistribPackageStore implements PackageStore {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final CoreContainer coreContainer;
private Map<String, FileInfo> tmpFiles = new ConcurrentHashMap<>();
+
public DistribPackageStore(CoreContainer coreContainer) {
this.coreContainer = coreContainer;
ensurePackageStoreDir(coreContainer.getResourceLoader().getInstancePath());
@@ -73,23 +71,13 @@ public class DistribPackageStore implements PackageStore {
}
- /**
- * get a list of nodes randomly shuffled
- * * @lucene.internal
- */
- public ArrayList<String> shuffledNodes() {
- Set<String> liveNodes = coreContainer.getZkController().getZkStateReader().getClusterState().getLiveNodes();
- ArrayList<String> l = new ArrayList(liveNodes);
- l.remove(myNode());
- Collections.shuffle(l, BlobRepository.RANDOM);
- return l;
- }
+
@Override
public Path getRealpath(String path) {
if (File.separatorChar == '\\') {
- path = path.replace('/' , File.separatorChar);
+ path = path.replace('/', File.separatorChar);
}
if (path.charAt(0) != File.separatorChar) {
path = File.separator + path;
@@ -182,7 +170,7 @@ public class DistribPackageStore implements PackageStore {
try {
IOUtils.deleteFilesIfExist(getRealpath(path), getRealpath(getMetaPath()));
} catch (IOException e) {
- log.error("Unable to delete files: "+path);
+ log.error("Unable to delete files: " + path);
}
}
@@ -224,8 +212,7 @@ public class DistribPackageStore implements PackageStore {
}
boolean fetchFromAnyNode() {
-
- ArrayList<String> l = shuffledNodes();
+ ArrayList<String> l = coreContainer.getPackageStoreAPI().shuffledNodes();
ZkStateReader stateReader = coreContainer.getZkController().getZkStateReader();
for (String liveNode : l) {
try {
@@ -273,17 +260,15 @@ public class DistribPackageStore implements PackageStore {
}
-
-
public FileDetails getDetails() {
- FileType type = getType(path);
+ FileType type = getType(path, false);
return new FileDetails() {
@Override
public MetaData getMetaData() {
try {
return readMetaData();
- } catch (Exception e){
+ } catch (Exception e) {
throw new RuntimeException(e);
}
}
@@ -307,18 +292,17 @@ public class DistribPackageStore implements PackageStore {
return;
}
ew.put("timestamp", getTimeStamp());
- metaData.writeMap(ew);
+ if(metaData != null)
+ metaData.writeMap(ew);
}
};
-
-
}
public void readData(Consumer<FileEntry> consumer) throws IOException {
MetaData meta = readMetaData();
try (InputStream is = new FileInputStream(realPath().toFile())) {
- consumer.accept(new FileEntry(null, meta,path ){
+ consumer.accept(new FileEntry(null, meta, path) {
@Override
public InputStream getInputStream() {
return is;
@@ -337,10 +321,10 @@ public class DistribPackageStore implements PackageStore {
byte[] bytes = baos.toByteArray();
info.persistToFile(entry.buf, ByteBuffer.wrap(bytes, 0, bytes.length));
tmpFiles.put(entry.getPath(), info);
- List<String> nodes = shuffledNodes();
+ List<String> nodes = coreContainer.getPackageStoreAPI().shuffledNodes();
int i = 0;
int FETCHFROM_SRC = 50;
- String myNodeName = myNode();
+ String myNodeName = coreContainer.getZkController().getNodeName();
try {
for (String node : nodes) {
String baseUrl = coreContainer.getZkController().getZkStateReader().getBaseUrlForNodeName(node);
@@ -370,36 +354,31 @@ public class DistribPackageStore implements PackageStore {
Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(), url, null);
} catch (Exception e) {
log.info("Node: " + node +
- " failed to respond for blob notification", e);
+ " failed to respond for file fetch notification", e);
//ignore the exception
// some nodes may be down or not responding
}
i++;
}
} finally {
- new Thread(() -> {
+ coreContainer.getUpdateShardHandler().getUpdateExecutor().submit(() -> {
try {
- // keep the jar in memory for 10 secs , so that
- //every node can download it from memory without the file system
Thread.sleep(10 * 1000);
- } catch (Exception e) {
- //don't care
} finally {
tmpFiles.remove(entry.getPath());
}
- }).start();
-
-
+ return null;
+ });
}
}
@Override
- public synchronized boolean fetch(String path, String from) {
+ public boolean fetch(String path, String from) {
if (path == null || path.isEmpty()) return false;
FileInfo f = new FileInfo(path);
try {
- if(f.exists(true, false)){
+ if (f.exists(true, false)) {
return true;
}
} catch (IOException e) {
@@ -419,7 +398,7 @@ public class DistribPackageStore implements PackageStore {
}
@Override
- public synchronized void get(String path, Consumer<FileEntry> consumer) throws IOException {
+ public void get(String path, Consumer<FileEntry> consumer, boolean fetchmissing) throws IOException {
File file = getRealpath(path).toFile();
String simpleName = file.getName();
if (isMetaDataFile(simpleName)) {
@@ -440,10 +419,10 @@ public class DistribPackageStore implements PackageStore {
@Override
- public synchronized List list(String path, Predicate<String> predicate) {
+ public List list(String path, Predicate<String> predicate) {
File file = getRealpath(path).toFile();
List<FileDetails> fileDetails = new ArrayList<>();
- FileType type = getType(path);
+ FileType type = getType(path, false);
if (type == FileType.DIRECTORY) {
file.list((dir, name) -> {
if (predicate == null || predicate.test(name)) {
@@ -455,7 +434,6 @@ public class DistribPackageStore implements PackageStore {
});
} else if (type == FileType.FILE) {
-
fileDetails.add(new FileInfo(path).getDetails());
}
@@ -464,9 +442,14 @@ public class DistribPackageStore implements PackageStore {
@Override
- public synchronized FileType getType(String path) {
+ public FileType getType(String path, boolean fetchMissing) {
File file = getRealpath(path).toFile();
- if (!file.exists()) return FileType.NOFILE;
+ if (!file.exists() && fetchMissing) {
+ if (fetch(path, null)) {
+ file = getRealpath(path).toFile();
+ }
+ if (!file.exists()) return FileType.NOFILE;
+ }
if (file.isDirectory()) return FileType.DIRECTORY;
return isMetaDataFile(file.getName()) ? FileType.METADATA : FileType.FILE;
}
diff --git a/solr/core/src/java/org/apache/solr/filestore/PackageStore.java b/solr/core/src/java/org/apache/solr/filestore/PackageStore.java
index b9be691..8ac2388 100644
--- a/solr/core/src/java/org/apache/solr/filestore/PackageStore.java
+++ b/solr/core/src/java/org/apache/solr/filestore/PackageStore.java
@@ -45,11 +45,11 @@ public interface PackageStore {
/**
* read file content from a given path
*/
- void get(String path, Consumer<FileEntry> filecontent) throws IOException;
+ void get(String path, Consumer<FileEntry> filecontent, boolean getMissing) throws IOException;
/**
* Fetch a resource from another node
- * internal
+ * internal API
*/
boolean fetch(String path, String from);
@@ -63,7 +63,7 @@ public interface PackageStore {
/**
* The type of the resource
*/
- FileType getType(String path);
+ FileType getType(String path, boolean fetchMissing);
public class FileEntry {
final ByteBuffer buf;
diff --git a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
index 71ee9d8..7e80b9a 100644
--- a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
+++ b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
@@ -18,13 +18,18 @@
package org.apache.solr.filestore;
import java.io.IOException;
+import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.solr.api.Command;
@@ -39,6 +44,7 @@ import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.BlobRepository;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
@@ -75,6 +81,50 @@ public class PackageStoreAPI {
return packageStore;
}
+ /**
+ * get a list of nodes randomly shuffled
+ * * @lucene.internal
+ */
+ public ArrayList<String> shuffledNodes() {
+ Set<String> liveNodes = coreContainer.getZkController().getZkStateReader().getClusterState().getLiveNodes();
+ ArrayList<String> l = new ArrayList(liveNodes);
+ l.remove(coreContainer.getZkController().getNodeName());
+ Collections.shuffle(l, BlobRepository.RANDOM);
+ return l;
+ }
+
+ public void validateFiles(List<String> files, boolean validateSignatures, Consumer<String> errs) {
+ for (String path : files) {
+ try {
+ PackageStore.FileType type = packageStore.getType(path, true);
+ if (type != PackageStore.FileType.FILE) {
+ errs.accept("No such file : " + path);
+ continue;
+ }
+
+ packageStore.get(path, entry -> {
+ if (entry.getMetaData().signatures == null ||
+ entry.getMetaData().signatures.isEmpty()) {
+ errs.accept(path + " has no signature");
+ return;
+ }
+ if (validateSignatures) {
+ try {
+ validate(entry.meta.signatures, entry);
+ } catch (SolrException e) {
+ log.error("error validating package artifact", e);
+ errs.accept(e.getMessage());
+ }
+ }
+ }, false);
+ } catch (Exception e) {
+ log.error("Error reading file ", e);
+ errs.accept("Error reading file " + path + " " + e.getMessage());
+ }
+ }
+
+ }
+
@EndPoint(
path = "/cluster/files/*",
method = SolrRequest.METHOD.PUT,
@@ -148,7 +198,7 @@ public class PackageStoreAPI {
cryptoKeys = new CryptoKeys(keys);
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Error parsing public keyts in ZooKeeper");
+ "Error parsing public keys in ZooKeeper");
}
for (String sig : sigs) {
if (cryptoKeys.verify(sig, buf) == null) {
@@ -187,7 +237,7 @@ public class PackageStoreAPI {
path = "";
}
- PackageStore.FileType typ = packageStore.getType(path);
+ PackageStore.FileType typ = packageStore.getType(path, false);
if (typ == PackageStore.FileType.NOFILE) {
rsp.add("files", Collections.singletonMap(path, null));
return;
@@ -221,7 +271,7 @@ public class PackageStoreAPI {
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reading file" + path);
}
- });
+ }, false);
});
}
@@ -270,4 +320,36 @@ public class PackageStoreAPI {
}
}
}
+
+ public void validate(List<String> sigs,
+ PackageStore.FileEntry entry) throws SolrException {
+ Map<String, byte[]> keys = CloudUtil.getTrustedKeys(
+ coreContainer.getZkController().getZkClient(), "exe");
+ if (keys == null || keys.isEmpty()) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "ZooKeeper does not have any public keys");
+ }
+ CryptoKeys cryptoKeys = null;
+ try {
+ cryptoKeys = new CryptoKeys(keys);
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Error parsing public keys in ZooKeeper");
+ }
+ for (String sig : sigs) {
+ Supplier<String> errMsg = () -> "Signature does not match any public key : " + sig;
+ if (entry.getBuffer() != null) {
+ if (cryptoKeys.verify(sig, entry.getBuffer()) == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get());
+ }
+ } else {
+ InputStream inputStream = entry.getInputStream();
+ if (cryptoKeys.verify(sig, inputStream) == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get());
+ }
+
+ }
+
+ }
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
index 5952d80..004da31 100644
--- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
@@ -68,6 +68,7 @@ import org.apache.solr.core.RequestParams;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.pkg.PackageListeners;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
@@ -150,7 +151,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
public static boolean getImmutable(SolrCore core) {
NamedList configSetProperties = core.getConfigSetProperties();
- if(configSetProperties == null) return false;
+ if (configSetProperties == null) return false;
Object immutable = configSetProperties.get(IMMUTABLE_CONFIGSET_ARG);
return immutable != null && Boolean.parseBoolean(immutable.toString());
}
@@ -245,10 +246,26 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
if (componentName != null) {
Map map = (Map) val.get(parts.get(1));
if (map != null) {
- val.put(parts.get(1), makeMap(componentName, map.get(componentName)));
+ Object o = map.get(componentName);
+ val.put(parts.get(1), makeMap(componentName, o));
+ if (req.getParams().getBool("meta", false)) {
+ // meta=true is asking for the package info of the plugin
+ // We go through all the listeners and see if there is one registered for this plugin
+ List<PackageListeners.Listener> listeners = req.getCore().getPackageListeners().getListeners();
+ for (PackageListeners.Listener listener :
+ listeners) {
+ PluginInfo info = listener.pluginInfo();
+ if(info == null) continue;
+ if (info.type.equals(parts.get(1)) && info.name.equals(componentName)) {
+ if (o instanceof Map) {
+ Map m1 = (Map) o;
+ m1.put("_packageinfo_", listener.getPackageVersion());
+ }
+ }
+ }
+ }
}
}
-
resp.add("config", val);
}
}
@@ -429,7 +446,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
List errs = CommandOperation.captureErrors(ops);
if (!errs.isEmpty()) {
- throw new ApiBag.ExceptionWithErrObject(SolrException.ErrorCode.BAD_REQUEST,"error processing params", errs);
+ throw new ApiBag.ExceptionWithErrObject(SolrException.ErrorCode.BAD_REQUEST, "error processing params", errs);
}
SolrResourceLoader loader = req.getCore().getResourceLoader();
@@ -492,14 +509,15 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
}
List errs = CommandOperation.captureErrors(ops);
if (!errs.isEmpty()) {
- throw new ApiBag.ExceptionWithErrObject(SolrException.ErrorCode.BAD_REQUEST,"error processing commands", errs);
+ log.error("ERROR:" + Utils.toJSONString(errs));
+ throw new ApiBag.ExceptionWithErrObject(SolrException.ErrorCode.BAD_REQUEST, "error processing commands", errs);
}
SolrResourceLoader loader = req.getCore().getResourceLoader();
if (loader instanceof ZkSolrResourceLoader) {
int latestVersion = ZkController.persistConfigResourceToZooKeeper((ZkSolrResourceLoader) loader, overlay.getZnodeVersion(),
ConfigOverlay.RESOURCE_NAME, overlay.toByteArray(), true);
- log.info("Executed config commands successfully and persisted to ZK {}", ops);
+ log.debug("Executed config commands successfully and persisted to ZK {}", ops);
waitForAllReplicasState(req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName(),
req.getCore().getCoreContainer().getZkController(),
ConfigOverlay.NAME,
@@ -530,8 +548,8 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
op.getMap(PluginInfo.INVARIANTS, null);
op.getMap(PluginInfo.APPENDS, null);
if (op.hasError()) return overlay;
- if(info.clazz == PluginBag.RuntimeLib.class) {
- if(!PluginBag.RuntimeLib.isEnabled()){
+ if (info.clazz == PluginBag.RuntimeLib.class) {
+ if (!PluginBag.RuntimeLib.isEnabled()) {
op.addError("Solr not started with -Denable.runtime.lib=true");
return overlay;
}
@@ -563,7 +581,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
private boolean pluginExists(SolrConfig.SolrPluginInfo info, ConfigOverlay overlay, String name) {
List<PluginInfo> l = req.getCore().getSolrConfig().getPluginInfos(info.clazz.getName());
- for (PluginInfo pluginInfo : l) if(name.equals( pluginInfo.name)) return true;
+ for (PluginInfo pluginInfo : l) if (name.equals(pluginInfo.name)) return true;
return overlay.getNamedPlugins(info.getCleanTag()).containsKey(name);
}
@@ -574,6 +592,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
try {
req.getCore().createInitInstance(new PluginInfo(SolrRequestHandler.TYPE, op.getDataMap()), expected, clz, "");
} catch (Exception e) {
+ log.error("Error checking plugin : ", e);
op.addError(e.getMessage());
return false;
}
@@ -679,7 +698,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
c == '_' ||
c == '-' ||
c == '.'
- ) continue;
+ ) continue;
else {
return formatString("''{0}'' name should only have chars [a-zA-Z_-.0-9] ", s);
}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
index 64b8c9a..f617fcb 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
@@ -40,6 +40,8 @@ import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.pkg.PackageListeners;
+import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SolrQueryTimeoutImpl;
@@ -58,11 +60,9 @@ import static org.apache.solr.common.params.CommonParams.PATH;
/**
- *
* Refer SOLR-281
- *
*/
-public class SearchHandler extends RequestHandlerBase implements SolrCoreAware , PluginInfoInitialized, PermissionNameProvider {
+public class SearchHandler extends RequestHandlerBase implements SolrCoreAware, PluginInfoInitialized, PermissionNameProvider {
static final String INIT_COMPONENTS = "components";
static final String INIT_FIRST_COMPONENTS = "first-components";
static final String INIT_LAST_COMPONENTS = "last-components";
@@ -70,22 +70,21 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware ,
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
protected volatile List<SearchComponent> components;
- private ShardHandlerFactory shardHandlerFactory ;
+ private ShardHandlerFactory shardHandlerFactory;
private PluginInfo shfInfo;
private SolrCore core;
- protected List<String> getDefaultComponents()
- {
+ protected List<String> getDefaultComponents() {
ArrayList<String> names = new ArrayList<>(8);
- names.add( QueryComponent.COMPONENT_NAME );
- names.add( FacetComponent.COMPONENT_NAME );
- names.add( FacetModule.COMPONENT_NAME );
- names.add( MoreLikeThisComponent.COMPONENT_NAME );
- names.add( HighlightComponent.COMPONENT_NAME );
- names.add( StatsComponent.COMPONENT_NAME );
- names.add( DebugComponent.COMPONENT_NAME );
- names.add( ExpandComponent.COMPONENT_NAME);
- names.add( TermsComponent.COMPONENT_NAME);
+ names.add(QueryComponent.COMPONENT_NAME);
+ names.add(FacetComponent.COMPONENT_NAME);
+ names.add(FacetModule.COMPONENT_NAME);
+ names.add(MoreLikeThisComponent.COMPONENT_NAME);
+ names.add(HighlightComponent.COMPONENT_NAME);
+ names.add(StatsComponent.COMPONENT_NAME);
+ names.add(DebugComponent.COMPONENT_NAME);
+ names.add(ExpandComponent.COMPONENT_NAME);
+ names.add(TermsComponent.COMPONENT_NAME);
return names;
}
@@ -94,7 +93,7 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware ,
public void init(PluginInfo info) {
init(info.initArgs);
for (PluginInfo child : info.children) {
- if("shardHandlerFactory".equals(child.type)){
+ if ("shardHandlerFactory".equals(child.type)) {
this.shfInfo = child;
break;
}
@@ -113,8 +112,7 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware ,
*/
@Override
@SuppressWarnings("unchecked")
- public void inform(SolrCore core)
- {
+ public void inform(SolrCore core) {
this.core = core;
List<String> c = (List<String>) initArgs.get(INIT_COMPONENTS);
Set<String> missing = new HashSet<>(core.getSearchComponents().checkContains(c));
@@ -143,6 +141,32 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware ,
});
}
+ if (core.getCoreContainer().isZooKeeperAware()) {
+ core.getPackageListeners().addListener(new PackageListeners.Listener() {
+ @Override
+ public String packageName() {
+ return null;
+ }
+
+ @Override
+ public PluginInfo pluginInfo() {
+ return null;
+ }
+
+ @Override
+ public void changed(PackageLoader.Package pkg) {
+ //we could optimize this by listening to only relevant packages,
+ // but it is not worth optimizing as these are lightweight objects
+ components = null;
+ }
+
+ @Override
+ public PackageLoader.Package.Version getPackageVersion() {
+ return null;
+ }
+ });
+ }
+
}
private void initComponents() {
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
new file mode 100644
index 0000000..bf29b64
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
@@ -0,0 +1,388 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.pkg;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.solr.api.Command;
+import org.apache.solr.api.EndPoint;
+import org.apache.solr.api.PayloadObj;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.request.beans.Package;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.annotation.JsonProperty;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.cloud.ZooKeeperException;
+import org.apache.solr.common.util.CommandOperation;
+import org.apache.solr.common.util.ReflectMapWriter;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.filestore.PackageStoreAPI;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.util.SolrJacksonAnnotationInspector;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH;
+import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM;
+import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM;
+
+/**This implements the public end points (/api/cluster/package) of package API.
+ *
+ */
+public class PackageAPI {
+ public static final String PACKAGES = "packages";
+ public final boolean enablePackages = Boolean.parseBoolean(System.getProperty("enable.packages", "false"));
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ final CoreContainer coreContainer;
+ private final ObjectMapper mapper = new ObjectMapper();
+ private final PackageLoader packageLoader;
+ Packages pkgs;
+
+ public final Edit editAPI = new Edit();
+ public final Read readAPI = new Read();
+
+ public PackageAPI(CoreContainer coreContainer, PackageLoader loader) {
+ this.coreContainer = coreContainer;
+ this.packageLoader = loader;
+ mapper.setAnnotationIntrospector(SolrJacksonAnnotationInspector.INSTANCE);
+ pkgs = new Packages();
+ SolrZkClient zkClient = coreContainer.getZkController().getZkClient();
+ try {
+ registerListener(zkClient);
+ } catch (KeeperException | InterruptedException e) {
+ SolrZkClient.checkInterrupted(e);
+ }
+ }
+
+ private void registerListener(SolrZkClient zkClient)
+ throws KeeperException, InterruptedException {
+ String path = SOLR_PKGS_PATH;
+ zkClient.exists(path,
+ new Watcher() {
+
+ @Override
+ public void process(WatchedEvent event) {
+ // session events are not change events, and do not remove the watcher
+ if (Event.EventType.None.equals(event.getType())) {
+ return;
+ }
+ try {
+ synchronized (this) {
+ log.debug("Updating [{}] ... ", path);
+
+ // remake watch
+ final Watcher thisWatch = this;
+ final Stat stat = new Stat();
+ final byte[] data = zkClient.getData(path, thisWatch, stat, true);
+ pkgs = readPkgsFromZk(data, stat);
+ packageLoader.refreshPackageConf();
+ }
+ } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) {
+ log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: [{}]", e.getMessage());
+ } catch (KeeperException e) {
+ log.error("A ZK error has occurred", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.warn("Interrupted", e);
+ }
+ }
+
+ }, true);
+ }
+
+
+ private Packages readPkgsFromZk(byte[] data, Stat stat) throws KeeperException, InterruptedException {
+
+ if (data == null || stat == null) {
+ stat = new Stat();
+ data = coreContainer.getZkController().getZkClient()
+ .getData(ZkStateReader.CLUSTER_PROPS, null, stat, true);
+
+ }
+ Packages packages = null;
+ if (data == null || data.length == 0) {
+ packages = new Packages();
+ } else {
+ try {
+ packages = mapper.readValue(data, Packages.class);
+ packages.znodeVersion = stat.getVersion();
+ } catch (IOException e) {
+ //invalid data in packages
+ //TODO handle properly;
+ return new Packages();
+ }
+ }
+ return packages;
+ }
+
+
+ public static class Packages implements ReflectMapWriter {
+ @JsonProperty
+ public int znodeVersion = -1;
+
+ @JsonProperty
+ public Map<String, List<PkgVersion>> packages = new LinkedHashMap<>();
+
+
+ public Packages copy() {
+ Packages p = new Packages();
+ p.znodeVersion = this.znodeVersion;
+ p.packages = new LinkedHashMap<>();
+ packages.forEach((s, versions) ->
+ p.packages.put(s, new ArrayList<>(versions)));
+ return p;
+ }
+ }
+
+ public static class PkgVersion implements ReflectMapWriter {
+
+ @JsonProperty
+ public String version;
+
+ @JsonProperty
+ public List<String> files;
+
+ public PkgVersion() {
+ }
+
+ public PkgVersion(Package.AddVersion addVersion) {
+ this.version = addVersion.version;
+ this.files = addVersion.files;
+ }
+
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof PkgVersion) {
+ PkgVersion that = (PkgVersion) obj;
+ return Objects.equals(this.version, that.version)
+ && Objects.equals(this.files, that.files);
+
+ }
+ return false;
+ }
+ }
+
+
+ @EndPoint(method = SolrRequest.METHOD.POST,
+ path = "/cluster/package",
+ permission = PACKAGE_EDIT_PERM)
+ public class Edit {
+
+ @Command(name = "refresh")
+ public void refresh(SolrQueryRequest req, SolrQueryResponse rsp, PayloadObj<String> payload) {
+ String p = payload.get();
+ if (p == null) {
+ payload.addError("Package null");
+ return;
+ }
+ PackageLoader.Package pkg = coreContainer.getPackageLoader().getPackage(p);
+ if (pkg == null) {
+ payload.addError("No such package: " + p);
+ return;
+ }
+
+ for (String s : coreContainer.getPackageStoreAPI().shuffledNodes()) {
+ Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(),
+ coreContainer.getZkController().zkStateReader.getBaseUrlForNodeName(s).replace("/solr", "/api") + "/cluster/package?wt=javabin&omitHeader=true&refreshPackage=" + p,
+ Utils.JAVABINCONSUMER);
+ }
+
+
+ }
+
+
+ @Command(name = "add")
+ public void add(SolrQueryRequest req, SolrQueryResponse rsp, PayloadObj<Package.AddVersion> payload) {
+ if (!checkEnabled(payload)) return;
+ Package.AddVersion add = payload.get();
+ if (add.files.isEmpty()) {
+ payload.addError("No files specified");
+ return;
+ }
+ PackageStoreAPI packageStoreAPI = coreContainer.getPackageStoreAPI();
+ packageStoreAPI.validateFiles(add.files, true, s -> payload.addError(s));
+ if (payload.hasError()) return;
+ Packages[] finalState = new Packages[1];
+ try {
+ coreContainer.getZkController().getZkClient().atomicUpdate(SOLR_PKGS_PATH, (stat, bytes) -> {
+ Packages packages = null;
+ try {
+ packages = bytes == null ? new Packages() : mapper.readValue(bytes, Packages.class);
+ packages = packages.copy();
+ } catch (IOException e) {
+ log.error("Error deserializing packages.json", e);
+ packages = new Packages();
+ }
+ packages.packages.computeIfAbsent(add.pkg, Utils.NEW_ARRAYLIST_FUN).add(new PkgVersion(add));
+ packages.znodeVersion = stat.getVersion() ;
+ finalState[0] = packages;
+ return Utils.toJSON(packages);
+ });
+ } catch (KeeperException | InterruptedException e) {
+ finalState[0] = null;
+ handleZkErr(e);
+ }
+ if (finalState[0] != null) {
+// succeeded in updating
+ pkgs = finalState[0];
+ notifyAllNodesToSync(pkgs.znodeVersion);
+ packageLoader.refreshPackageConf();
+ }
+
+ }
+
+ @Command(name = "delete")
+ public void del(SolrQueryRequest req, SolrQueryResponse rsp, PayloadObj<Package.DelVersion> payload) {
+ if (!checkEnabled(payload)) return;
+ Package.DelVersion delVersion = payload.get();
+ try {
+ coreContainer.getZkController().getZkClient().atomicUpdate(SOLR_PKGS_PATH, (stat, bytes) -> {
+ Packages packages = null;
+ try {
+ packages = mapper.readValue(bytes, Packages.class);
+ packages = packages.copy();
+ } catch (IOException e) {
+ packages = new Packages();
+ }
+
+ List<PkgVersion> versions = packages.packages.get(delVersion.pkg);
+ if (versions == null || versions.isEmpty()) {
+ payload.addError("No such package: " + delVersion.pkg);
+ return null;// no change
+ }
+ int idxToremove = -1;
+ for (int i = 0; i < versions.size(); i++) {
+ if (Objects.equals(versions.get(i).version, delVersion.version)) {
+ idxToremove = i;
+ break;
+ }
+ }
+ if (idxToremove == -1) {
+ payload.addError("No such version: " + delVersion.version);
+ return null;
+ }
+ versions.remove(idxToremove);
+ packages.znodeVersion = stat.getVersion() + 1;
+ return Utils.toJSON(packages);
+ });
+ } catch (KeeperException | InterruptedException e) {
+ handleZkErr(e);
+
+ }
+
+
+ }
+
+ }
+
+ private boolean checkEnabled(CommandOperation payload) {
+ if (!enablePackages) {
+ payload.addError("Package loading is not enabled , Start your nodes with -Denable.packages=true");
+ return false;
+ }
+ return true;
+ }
+
+ @EndPoint(
+ method = SolrRequest.METHOD.GET,
+ path = {"/cluster/package/",
+ "/cluster/package/{name}"},
+ permission = PACKAGE_READ_PERM
+ )
+ public class Read {
+ @Command()
+ public void get(SolrQueryRequest req, SolrQueryResponse rsp) {
+ String refresh = req.getParams().get("refreshPackage");
+ if (refresh != null) {
+ packageLoader.notifyListeners(refresh);
+ return;
+ }
+
+ int expectedVersion = req.getParams().getInt("expectedVersion", -1);
+ if (expectedVersion != -1) {
+ syncToVersion(expectedVersion);
+ }
+ String name = req.getPathTemplateValues().get("name");
+ if (name == null) {
+ rsp.add("result", pkgs);
+ } else {
+ rsp.add("result", Collections.singletonMap(name, pkgs.packages.get(name)));
+ }
+ }
+
+ private void syncToVersion(int expectedVersion) {
+ int origVersion = pkgs.znodeVersion;
+ for (int i = 0; i < 10; i++) {
+ log.debug("my version is {} , and expected version {}", pkgs.znodeVersion, expectedVersion);
+ if (pkgs.znodeVersion >= expectedVersion) {
+ if (origVersion < pkgs.znodeVersion) {
+ packageLoader.refreshPackageConf();
+ }
+ return;
+ }
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ }
+ try {
+ pkgs = readPkgsFromZk(null, null);
+ } catch (KeeperException | InterruptedException e) {
+ handleZkErr(e);
+
+ }
+
+ }
+
+ }
+
+
+ }
+
+ void notifyAllNodesToSync(int expected) {
+ for (String s : coreContainer.getPackageStoreAPI().shuffledNodes()) {
+ Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(),
+ coreContainer.getZkController().zkStateReader.getBaseUrlForNodeName(s).replace("/solr", "/api") + "/cluster/package?wt=javabin&omitHeader=true&expectedVersion" + expected,
+ Utils.JAVABINCONSUMER);
+ }
+ }
+
+ public void handleZkErr(Exception e) {
+ log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e));
+ }
+
+
+}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java
new file mode 100644
index 0000000..0287f5e
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.pkg;
+
+import java.lang.invoke.MethodHandles;
+import java.lang.ref.Reference;
+import java.lang.ref.SoftReference;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.solr.core.PluginInfo;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.logging.MDCLoggingContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PackageListeners {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static final String PACKAGE_VERSIONS = "PKG_VERSIONS";
+ private SolrCore core;
+
+ public PackageListeners(SolrCore core) {
+ this.core = core;
+ }
+
+ // this registry only keeps a weak reference because it does not want to
+ // cause a memory leak if the listener forgets to unregister itself
+ private List<Reference<Listener>> listeners = new ArrayList<>();
+
+ public synchronized void addListener(Listener listener) {
+ listeners.add(new SoftReference<>(listener));
+
+ }
+
+ public synchronized void removeListener(Listener listener) {
+ Iterator<Reference<Listener>> it = listeners.iterator();
+ while (it.hasNext()) {
+ Reference<Listener> ref = it.next();
+ Listener pkgListener = ref.get();
+ if (pkgListener == null || pkgListener == listener) {
+ it.remove();
+ }
+
+ }
+
+ }
+
+ synchronized void packagesUpdated(List<PackageLoader.Package> pkgs) {
+ if(core != null) MDCLoggingContext.setCore(core);
+ try {
+ for (PackageLoader.Package pkgInfo : pkgs) {
+ invokeListeners(pkgInfo);
+ }
+ } finally {
+ if(core != null) MDCLoggingContext.clear();
+
+ }
+ }
+
+ private synchronized void invokeListeners(PackageLoader.Package pkg) {
+ for (Reference<Listener> ref : listeners) {
+ Listener listener = ref.get();
+ if(listener == null) continue;
+ if (listener.packageName() == null || listener.packageName().equals(pkg.name())) {
+ listener.changed(pkg);
+ }
+ }
+ }
+
+ public List<Listener> getListeners() {
+ List<Listener> result = new ArrayList<>();
+ for (Reference<Listener> ref : listeners) {
+ Listener l = ref.get();
+ if (l != null) {
+ result.add(l);
+ }
+ }
+ return result;
+ }
+
+
+ public interface Listener {
+ /**Name of the package or null to loisten to all package changes
+ */
+ String packageName();
+
+ PluginInfo pluginInfo();
+
+ void changed(PackageLoader.Package pkg);
+
+ PackageLoader.Package.Version getPackageVersion();
+
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java
new file mode 100644
index 0000000..88e4e0f
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.pkg;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.MalformedURLException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrResourceLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.lucene.util.IOUtils.closeWhileHandlingException;
+
+/**
+ * The class that holds a mapping of various packages and classloaders
+ */
+public class PackageLoader implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final CoreContainer coreContainer;
+ private final Map<String, Package> packageClassLoaders = new ConcurrentHashMap<>();
+
+ private PackageAPI.Packages myCopy;
+
+ private PackageAPI packageAPI;
+
+
+ public PackageLoader(CoreContainer coreContainer) {
+ this.coreContainer = coreContainer;
+ packageAPI = new PackageAPI(coreContainer, this);
+ myCopy = packageAPI.pkgs;
+
+ }
+
+ public PackageAPI getPackageAPI() {
+ return packageAPI;
+ }
+
+ public Package getPackage(String key) {
+ return packageClassLoaders.get(key);
+ }
+
+ public Map<String, Package> getPackages() {
+ return Collections.EMPTY_MAP;
+ }
+
+ public void refreshPackageConf() {
+ log.info("{} updated to version {}", ZkStateReader.SOLR_PKGS_PATH, packageAPI.pkgs.znodeVersion);
+
+ List<Package> updated = new ArrayList<>();
+ Map<String, List<PackageAPI.PkgVersion>> modified = getModified(myCopy, packageAPI.pkgs);
+
+ for (Map.Entry<String, List<PackageAPI.PkgVersion>> e : modified.entrySet()) {
+ if (e.getValue() != null) {
+ Package p = packageClassLoaders.get(e.getKey());
+ if (e.getValue() != null && p == null) {
+ packageClassLoaders.put(e.getKey(), p = new Package(e.getKey()));
+ }
+ p.updateVersions(e.getValue());
+ updated.add(p);
+ } else {
+ Package p = packageClassLoaders.remove(e.getKey());
+ if (p != null) {
+ //other classes are holding to a reference to this objecec
+ // they should know that this is removed
+ p.markDeleted();
+ closeWhileHandlingException(p);
+ }
+ }
+ }
+ for (SolrCore core : coreContainer.getCores()) {
+ core.getPackageListeners().packagesUpdated(updated);
+ }
+ }
+
+ public Map<String, List<PackageAPI.PkgVersion>> getModified(PackageAPI.Packages old, PackageAPI.Packages newPkgs) {
+ Map<String, List<PackageAPI.PkgVersion>> changed = new HashMap<>();
+ for (Map.Entry<String, List<PackageAPI.PkgVersion>> e : newPkgs.packages.entrySet()) {
+ List<PackageAPI.PkgVersion> versions = old.packages.get(e.getKey());
+ if (versions != null) {
+ if (!Objects.equals(e.getValue(), versions)) {
+ log.info("Package {} is modified ", e.getKey());
+ changed.put(e.getKey(), e.getValue());
+ }
+ } else {
+ log.info("A new package: {} introduced", e.getKey());
+ changed.put(e.getKey(), e.getValue());
+ }
+ }
+ //some packages are deleted altogether
+ for (String s : old.packages.keySet()) {
+ if (!newPkgs.packages.keySet().contains(s)) {
+ log.info("Package: {} is removed althogether", s);
+ changed.put(s, null);
+ }
+ }
+
+ return changed;
+
+ }
+
+ public void notifyListeners(String pkg) {
+ Package p = packageClassLoaders.get(pkg);
+ if (p != null) {
+ List<Package> l = Collections.singletonList(p);
+ for (SolrCore core : coreContainer.getCores()) {
+ core.getPackageListeners().packagesUpdated(l);
+ }
+ }
+ }
+
+ /**
+ * represents a package definition in the packages.json
+ */
+ public class Package implements Closeable {
+ final String name;
+ final Map<String, Version> myVersions = new ConcurrentHashMap<>();
+ private List<String> sortedVersions = new CopyOnWriteArrayList<>();
+ String latest;
+ private boolean deleted;
+
+
+ Package(String name) {
+ this.name = name;
+ }
+
+ public boolean isDeleted() {
+ return deleted;
+ }
+
+
+ private synchronized void updateVersions(List<PackageAPI.PkgVersion> modified) {
+ for (PackageAPI.PkgVersion v : modified) {
+ Version version = myVersions.get(v.version);
+ if (version == null) {
+ log.info("A new version: {} added for package: {} with artifacts {}", v.version, this.name, v.files);
+ myVersions.put(v.version, new Version(this, v));
+ sortedVersions.add(v.version);
+ }
+ }
+
+ Set<String> newVersions = new HashSet<>();
+ for (PackageAPI.PkgVersion v : modified) {
+ newVersions.add(v.version);
+ }
+ for (String s : new HashSet<>(myVersions.keySet())) {
+ if (!newVersions.contains(s)) {
+ log.info("version: {} is removed from package: {}", s, this.name);
+ sortedVersions.remove(s);
+ Version removed = myVersions.remove(s);
+ if (removed != null) {
+ closeWhileHandlingException(removed);
+ }
+ }
+ }
+
+ sortedVersions.sort(String::compareTo);
+ if (sortedVersions.size() > 0) {
+ String latest = sortedVersions.get(sortedVersions.size() - 1);
+ if (!latest.equals(this.latest)) {
+ log.info("version: {} is the new latest in package: {}", latest, this.name);
+ }
+ this.latest = latest;
+ } else {
+ log.error("latest version: null");
+ latest = null;
+ }
+
+ }
+
+
+ public Version getLatest() {
+ return latest == null ? null : myVersions.get(latest);
+ }
+
+ public Version getLatest(String lessThan) {
+ if (lessThan == null) {
+ return getLatest();
+ }
+ String latest = findBiggest(lessThan, new ArrayList(sortedVersions));
+ return latest == null ? null : myVersions.get(latest);
+ }
+
+ public String name() {
+ return name;
+ }
+
+ private void markDeleted() {
+ deleted = true;
+ myVersions.clear();
+ sortedVersions.clear();
+ latest = null;
+
+ }
+
+ @Override
+ public void close() throws IOException {
+ for (Version v : myVersions.values()) v.close();
+ }
+
+ public class Version implements MapWriter, Closeable {
+ private final Package parent;
+ private SolrResourceLoader loader;
+
+ private final PackageAPI.PkgVersion version;
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put("package", parent.name());
+ version.writeMap(ew);
+ }
+
+ Version(Package parent, PackageAPI.PkgVersion v) {
+ this.parent = parent;
+ this.version = v;
+ List<Path> paths = new ArrayList<>();
+ for (String file : version.files) {
+ paths.add(coreContainer.getPackageStoreAPI().getPackageStore().getRealpath(file));
+ }
+
+ try {
+ loader = new SolrResourceLoader(
+ "PACKAGE_LOADER: " + parent.name() + ":" + version,
+ paths,
+ coreContainer.getResourceLoader().getInstancePath(),
+ coreContainer.getResourceLoader().getClassLoader());
+ } catch (MalformedURLException e) {
+ log.error("Could not load classloader ", e);
+ }
+ }
+
+ public String getVersion() {
+ return version.version;
+ }
+
+ public Collection getFiles() {
+ return Collections.unmodifiableList(version.files);
+ }
+
+ public SolrResourceLoader getLoader() {
+ return loader;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (loader != null) {
+ closeWhileHandlingException(loader);
+ }
+ }
+ }
+ }
+
+ private static String findBiggest(String lessThan, List<String> sortedList) {
+ String latest = null;
+ for (String v : sortedList) {
+ if (v.compareTo(lessThan) < 1) {
+ latest = v;
+ } else break;
+ }
+ return latest;
+ }
+
+ @Override
+ public void close() {
+ for (Package p : packageClassLoaders.values()) closeWhileHandlingException(p);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
new file mode 100644
index 0000000..63facde
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.pkg;
+
+import java.lang.invoke.MethodHandles;
+
+import org.apache.solr.core.PluginBag;
+import org.apache.solr.core.PluginInfo;
+import org.apache.solr.core.RequestParams;
+import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrCore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final String LATEST = "$LATEST";
+
+ private final SolrCore core;
+ private final SolrConfig.SolrPluginInfo pluginMeta;
+ private PackageLoader.Package.Version pkgVersion;
+ private PluginInfo info;
+
+
+ public PackagePluginHolder(PluginInfo info, SolrCore core, SolrConfig.SolrPluginInfo pluginMeta) {
+ super(info);
+ this.core = core;
+ this.pluginMeta = pluginMeta;
+ this.info = info;
+
+ reload(core.getCoreContainer().getPackageLoader().getPackage(info.pkgName));
+ core.getPackageListeners().addListener(new PackageListeners.Listener() {
+ @Override
+ public String packageName() {
+ return info.pkgName;
+ }
+
+ @Override
+ public PluginInfo pluginInfo() {
+ return info;
+ }
+
+ @Override
+ public void changed(PackageLoader.Package pkg) {
+ reload(pkg);
+
+ }
+
+ @Override
+ public PackageLoader.Package.Version getPackageVersion() {
+ return pkgVersion;
+ }
+
+ });
+ }
+
+ private String maxVersion() {
+ RequestParams.ParamSet p = core.getSolrConfig().getRequestParams().getParams(PackageListeners.PACKAGE_VERSIONS);
+ if (p == null) {
+ return null;
+ }
+ Object o = p.get().get(info.pkgName);
+ if (o == null || LATEST.equals(o)) return null;
+ return o.toString();
+ }
+
+
+ private synchronized void reload(PackageLoader.Package pkg) {
+ String lessThan = maxVersion();
+ PackageLoader.Package.Version newest = pkg.getLatest(lessThan);
+ if (newest == null) {
+ log.error("No latest version available for package : {}", pkg.name());
+ return;
+ }
+ if (lessThan != null) {
+ PackageLoader.Package.Version pkgLatest = pkg.getLatest();
+ if (pkgLatest != newest) {
+ log.info("Using version :{}. latest is {}, params.json has config {} : {}", newest.getVersion(), pkgLatest.getVersion(), pkg.name(), lessThan);
+ }
+ }
+
+ if (pkgVersion != null) {
+ if (newest == pkgVersion) {
+ //I'm already using the latest classloder in the package. nothing to do
+ return;
+ }
+ }
+
+ log.info("loading plugin: {} -> {} using package {}:{}",
+ pluginInfo.type, pluginInfo.name, pkg.name(), newest.getVersion());
+
+ Object instance = SolrCore.createInstance(pluginInfo.className,
+ pluginMeta.clazz, pluginMeta.getCleanTag(), core, newest.getLoader());
+ PluginBag.initInstance(instance, pluginInfo);
+ T old = inst;
+ inst = (T) instance;
+ pkgVersion = newest;
+ if (old instanceof AutoCloseable) {
+ AutoCloseable closeable = (AutoCloseable) old;
+ try {
+ closeable.close();
+ } catch (Exception e) {
+ log.error("error closing plugin", e);
+ }
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
index a4c7c0d..b5d4099 100644
--- a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
+++ b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
@@ -53,6 +53,9 @@ public interface PermissionNameProvider {
METRICS_HISTORY_READ_PERM("metrics-history-read", null),
FILESTORE_READ_PERM("filestore-read", null),
FILESTORE_WRITE_PERM("filestore-write", null),
+ PACKAGE_EDIT_PERM("package-edit", null),
+ PACKAGE_READ_PERM("package-read", null),
+
ALL("all", unmodifiableSet(new HashSet<>(asList("*", null))))
;
final String name;
diff --git a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java
index eb3c08b..5ddaf8b 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java
@@ -23,6 +23,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.stream.Collectors;
import com.google.common.collect.ImmutableMap;
import org.apache.solr.common.SolrException;
@@ -33,9 +34,12 @@ import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.PluginBag;
import org.apache.solr.core.PluginInfo;
+import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.pkg.PackagePluginHolder;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.processor.UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder.LazyUpdateRequestProcessorFactory;
import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
@@ -126,8 +130,7 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized
// wrap in an ArrayList so we know we know we can do fast index lookups
// and that add(int,Object) is supported
- List<UpdateRequestProcessorFactory> list = new ArrayList<>
- (solrCore.initPlugins(info.getChildren("processor"),UpdateRequestProcessorFactory.class,null));
+ List<UpdateRequestProcessorFactory> list = createProcessors(info);
if(list.isEmpty()){
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
@@ -170,6 +173,23 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized
}
+ private List<UpdateRequestProcessorFactory> createProcessors(PluginInfo info) {
+ List<PluginInfo> processors = info.getChildren("processor");
+ return processors.stream().map(it -> {
+ if(it.pkgName == null){
+ return solrCore.createInitInstance(it, UpdateRequestProcessorFactory.class,
+ UpdateRequestProcessorFactory.class.getSimpleName(), null);
+
+ } else {
+ return new LazyUpdateRequestProcessorFactory(new PackagePluginHolder(
+ it,
+ solrCore,
+ SolrConfig.classVsSolrPluginInfo.get(UpdateRequestProcessorFactory.class.getName())));
+ }
+ }).collect(Collectors.toList());
+ }
+
+
/**
* Creates a chain backed directly by the specified list. Modifications to
* the array will affect future calls to <code>createProcessor</code>
@@ -328,7 +348,7 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized
public static class LazyUpdateProcessorFactoryHolder extends PluginBag.PluginHolder<UpdateRequestProcessorFactory> {
private volatile UpdateRequestProcessorFactory lazyFactory;
- public LazyUpdateProcessorFactoryHolder(final PluginBag.LazyPluginHolder holder) {
+ public LazyUpdateProcessorFactoryHolder(final PluginBag.PluginHolder holder) {
super(holder.getPluginInfo());
lazyFactory = new LazyUpdateRequestProcessorFactory(holder);
}
@@ -339,27 +359,20 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized
return lazyFactory;
}
- public class LazyUpdateRequestProcessorFactory extends UpdateRequestProcessorFactory {
- private final PluginBag.LazyPluginHolder holder;
- UpdateRequestProcessorFactory delegate;
+ public static class LazyUpdateRequestProcessorFactory extends UpdateRequestProcessorFactory {
+ private final PluginBag.PluginHolder<UpdateRequestProcessorFactory> holder;
- public LazyUpdateRequestProcessorFactory(PluginBag.LazyPluginHolder holder) {
+ public LazyUpdateRequestProcessorFactory(PluginBag.PluginHolder holder) {
this.holder = holder;
}
public UpdateRequestProcessorFactory getDelegate() {
- return delegate;
+ return holder.get();
}
@Override
public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
- if (delegate != null) return delegate.getInstance(req, rsp, next);
-
- synchronized (this) {
- if (delegate == null)
- delegate = (UpdateRequestProcessorFactory) holder.get();
- }
- return delegate.getInstance(req, rsp, next);
+ return holder.get().getInstance(req, rsp, next);
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/util/CryptoKeys.java b/solr/core/src/java/org/apache/solr/util/CryptoKeys.java
index cb368e7..e1977a3 100644
--- a/solr/core/src/java/org/apache/solr/util/CryptoKeys.java
+++ b/solr/core/src/java/org/apache/solr/util/CryptoKeys.java
@@ -87,6 +87,26 @@ public final class CryptoKeys {
return null;
}
+ public String verify(String sig, InputStream is) {
+ exception = null;
+ for (Map.Entry<String, PublicKey> entry : keys.entrySet()) {
+ boolean verified;
+ try {
+ verified = CryptoKeys.verify(entry.getValue(), Base64.base64ToByteArray(sig), is);
+ log.debug("verified {} ", verified);
+ if (verified) return entry.getKey();
+ } catch (Exception e) {
+ exception = e;
+ log.debug("NOT verified ");
+ }
+
+ }
+
+ return null;
+ }
+
+
+
/**
* Create PublicKey from a .DER file
@@ -363,12 +383,12 @@ public final class CryptoKeys {
public static void main(String[] args) throws Exception {
RSAKeyPair keyPair = new RSAKeyPair();
- System.out.println(keyPair.getPublicKeyStr());
+// CLIO.out(keyPair.getPublicKeyStr());
PublicKey pk = deserializeX509PublicKey(keyPair.getPublicKeyStr());
byte[] payload = "Hello World!".getBytes(StandardCharsets.UTF_8);
byte[] encrypted = keyPair.encrypt(ByteBuffer.wrap(payload));
String cipherBase64 = Base64.byteArrayToBase64(encrypted);
- System.out.println("encrypted: "+ cipherBase64);
+// CLIO.out("encrypted: "+ cipherBase64);
System.out.println("signed: "+ Base64.byteArrayToBase64(keyPair.signSha256(payload)));
System.out.println("decrypted "+ new String(decryptRSA(encrypted , pk), StandardCharsets.UTF_8));
}
diff --git a/solr/core/src/java/org/apache/solr/util/SolrJacksonAnnotationInspector.java b/solr/core/src/java/org/apache/solr/util/SolrJacksonAnnotationInspector.java
new file mode 100644
index 0000000..727ffcb
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/SolrJacksonAnnotationInspector.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util;
+
+import com.fasterxml.jackson.core.Version;
+import com.fasterxml.jackson.databind.AnnotationIntrospector;
+import com.fasterxml.jackson.databind.PropertyName;
+import com.fasterxml.jackson.databind.introspect.Annotated;
+import com.fasterxml.jackson.databind.introspect.AnnotatedField;
+import com.fasterxml.jackson.databind.introspect.AnnotatedMember;
+import com.fasterxml.jackson.databind.introspect.AnnotatedMethod;
+import com.fasterxml.jackson.databind.util.BeanUtil;
+import org.apache.solr.common.annotation.JsonProperty;
+
+/**this class provides a mapping between Solr's {@link JsonProperty} annotation to a corresponding annotation
+ * in jackson
+ see SOLR-13841 for more details
+ * <blockquote><pre>
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.setAnnotationIntrospector(new SolrJacksonAnnotationInspector());
+ * </pre></blockquote>
+ **/
+public class SolrJacksonAnnotationInspector extends AnnotationIntrospector {
+ public static final SolrJacksonAnnotationInspector INSTANCE = new SolrJacksonAnnotationInspector();
+
+ @Override
+ public Version version() {
+ return Version.unknownVersion();
+ }
+
+ @Override
+ public PropertyName findNameForSerialization(Annotated a) {
+ if (a instanceof AnnotatedMethod) {
+ AnnotatedMethod am = (AnnotatedMethod) a;
+ JsonProperty prop = am.getAnnotation(JsonProperty.class);
+ if (prop == null) return null;
+ if (prop.value().isEmpty()) {
+ return new PropertyName(BeanUtil.okNameForGetter(am, true));
+ } else {
+ return new PropertyName(prop.value());
+ }
+
+ }
+ if (a instanceof AnnotatedField) {
+ AnnotatedField af = (AnnotatedField) a;
+ JsonProperty prop = af.getAnnotation(JsonProperty.class);
+ if (prop == null) return null;
+ return prop.value().isEmpty() ?
+ new PropertyName(af.getName()) :
+ new PropertyName(prop.value());
+ }
+ return null;
+ }
+
+ @Override
+ public Boolean hasRequiredMarker(AnnotatedMember m) {
+ JsonProperty prop = m.getAnnotation(JsonProperty.class);
+ if (prop == null) return Boolean.FALSE;
+ return prop.required();
+ }
+
+ @Override
+ public String findPropertyDefaultValue(Annotated m) {
+ JsonProperty prop = m.getAnnotation(JsonProperty.class);
+ if (prop == null) return "";
+ return prop.defaultValue();
+ }
+
+ @Override
+ public PropertyName findNameForDeserialization(Annotated a) {
+ return findNameForSerialization(a);
+ }
+
+}
diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java
index 853582f..9ca5e1b 100644
--- a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java
+++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java
@@ -17,9 +17,6 @@
package org.apache.solr.filestore;
-import static org.apache.solr.common.util.Utils.JAVABINCONSUMER;
-import static org.apache.solr.core.TestDynamicLoading.getFileContent;
-
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -30,6 +27,7 @@ import java.util.Set;
import java.util.concurrent.Callable;
import java.util.function.Predicate;
+import com.google.common.collect.ImmutableSet;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
@@ -50,9 +48,10 @@ import org.apache.solr.util.LogLevel;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.server.ByteBufferInputStream;
-import com.google.common.collect.ImmutableSet;
+import static org.apache.solr.common.util.Utils.JAVABINCONSUMER;
+import static org.apache.solr.core.TestDynamicLoading.getFileContent;
-@LogLevel("org.apache.solr.core.PackageStoreAPI=DEBUG;org.apache.solr.core.DistribPackageStore=DEBUG")
+@LogLevel("org.apache.solr.filestore.PackageStoreAPI=DEBUG;org.apache.solr.filestore.DistribPackageStore=DEBUG")
public class TestDistribPackageStore extends SolrCloudTestCase {
public void testPackageStoreManagement() throws Exception {
@@ -105,54 +104,26 @@ public class TestDistribPackageStore extends SolrCloudTestCase {
)
);
- class Fetcher implements Callable {
- String url;
- JettySolrRunner jetty;
- Fetcher(String s, JettySolrRunner jettySolrRunner){
- this.url = s;
- this.jetty = jettySolrRunner;
- }
- @Override
- public NavigableObject call() throws Exception {
- try (HttpSolrClient solrClient = (HttpSolrClient) jetty.newClient()) {
- return (NavigableObject) Utils.executeGET(solrClient.getHttpClient(), this.url, JAVABINCONSUMER);
- }
- }
-
- @Override
- public String toString() {
- return url;
- }
-
- }
-
Map expected = Utils.makeMap(
":files:/package/mypkg/v1.0/runtimelibs.jar:name", "runtimelibs.jar",
":files:/package/mypkg/v1.0[0]:sha512", "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420"
);
- for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {
- String baseUrl = jettySolrRunner.getBaseUrl().toString().replace("/solr", "/api");
- String url = baseUrl + "/node/files/package/mypkg/v1.0/runtimelibs.jar?wt=javabin&meta=true";
-
- assertResponseValues(10, new Fetcher(url, jettySolrRunner), expected);
-
- try (HttpSolrClient solrClient = (HttpSolrClient) jettySolrRunner.newClient()) {
- ByteBuffer buf = Utils.executeGET(solrClient.getHttpClient(), baseUrl + "/node/files/package/mypkg/v1.0/runtimelibs.jar",
- Utils.newBytesConsumer(Integer.MAX_VALUE));
- assertEquals(
- "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420",
- DigestUtils.sha512Hex(new ByteBufferInputStream(buf))
- );
+ waitForAllNodesHaveFile(cluster,"/package/mypkg/v1.0/runtimelibs.jar", expected, true);
- }
-
- }
postFile(cluster.getSolrClient(), getFileContent("runtimecode/runtimelibs_v2.jar.bin"),
"/package/mypkg/v1.0/runtimelibs_v2.jar",
null
);
+ expected = Utils.makeMap(
+ ":files:/package/mypkg/v1.0/runtimelibs_v2.jar:name", "runtimelibs_v2.jar",
+ ":files:/package/mypkg/v1.0[0]:sha512",
+ "bc5ce45ad281b6a08fb7e529b1eb475040076834816570902acb6ebdd809410e31006efdeaa7f78a6c35574f3504963f5f7e4d92247d0eb4db3fc9abdda5d417"
+
+ );
+ waitForAllNodesHaveFile(cluster,"/package/mypkg/v1.0/runtimelibs_v2.jar", expected, false);
+
expected = Utils.makeMap(
":files:/package/mypkg/v1.0", (Predicate<Object>) o -> {
@@ -181,6 +152,48 @@ public class TestDistribPackageStore extends SolrCloudTestCase {
}
}
+ public static void waitForAllNodesHaveFile(MiniSolrCloudCluster cluster, String path, Map expected , boolean verifyContent) throws Exception {
+ for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {
+ String baseUrl = jettySolrRunner.getBaseUrl().toString().replace("/solr", "/api");
+ String url = baseUrl + "/node/files" + path + "?wt=javabin&meta=true";
+ assertResponseValues(10, new Fetcher(url, jettySolrRunner), expected);
+
+ if(verifyContent) {
+ try (HttpSolrClient solrClient = (HttpSolrClient) jettySolrRunner.newClient()) {
+ ByteBuffer buf = Utils.executeGET(solrClient.getHttpClient(), baseUrl + "/node/files" + path,
+ Utils.newBytesConsumer(Integer.MAX_VALUE));
+ assertEquals(
+ "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420",
+ DigestUtils.sha512Hex(new ByteBufferInputStream(buf))
+ );
+
+ }
+ }
+
+ }
+ }
+
+ static class Fetcher implements Callable {
+ String url;
+ JettySolrRunner jetty;
+ Fetcher(String s, JettySolrRunner jettySolrRunner){
+ this.url = s;
+ this.jetty = jettySolrRunner;
+ }
+ @Override
+ public NavigableObject call() throws Exception {
+ try (HttpSolrClient solrClient = (HttpSolrClient) jetty.newClient()) {
+ return (NavigableObject) Utils.executeGET(solrClient.getHttpClient(), this.url, JAVABINCONSUMER);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return url;
+ }
+
+ }
+
public static NavigableObject assertResponseValues(int repeats, SolrClient client, SolrRequest req, Map vals) throws Exception {
Callable<NavigableObject> callable = () -> req.process(client);
@@ -224,7 +237,7 @@ public class TestDistribPackageStore extends SolrCloudTestCase {
- private void postFile(SolrClient client, ByteBuffer buffer, String name, String sig)
+ public static void postFile(SolrClient client, ByteBuffer buffer, String name, String sig)
throws SolrServerException, IOException {
String resource = "/cluster/files" + name;
ModifiableSolrParams params = new ModifiableSolrParams();
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java b/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java
index fde0335..f979831 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java
@@ -27,7 +27,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.api.AnnotatedApi;
import org.apache.solr.api.Api;
@@ -37,6 +36,7 @@ import org.apache.solr.api.EndPoint;
import org.apache.solr.api.V2HttpCall;
import org.apache.solr.api.V2HttpCall.CompositeApi;
import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.common.annotation.JsonProperty;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
new file mode 100644
index 0000000..4ff0f7a
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.pkg;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.V2Request;
+import org.apache.solr.client.solrj.request.beans.Package;
+import org.apache.solr.client.solrj.util.ClientUtils;
+import org.apache.solr.cloud.ConfigRequest;
+import org.apache.solr.cloud.MiniSolrCloudCluster;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.MapWriterMap;
+import org.apache.solr.common.NavigableObject;
+import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.filestore.TestDistribPackageStore;
+import org.apache.solr.util.LogLevel;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.data.Stat;
+import org.junit.Test;
+
+import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH;
+import static org.apache.solr.common.params.CommonParams.JAVABIN;
+import static org.apache.solr.common.params.CommonParams.WT;
+import static org.apache.solr.core.TestDynamicLoading.getFileContent;
+import static org.apache.solr.filestore.TestDistribPackageStore.readFile;
+
+@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG")
+//@org.apache.lucene.util.LuceneTestCase.AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13822") // leaks files
+public class TestPackages extends SolrCloudTestCase {
+
+ @Test
+ public void testPluginLoading() throws Exception {
+ System.setProperty("enable.packages", "true");
+ MiniSolrCloudCluster cluster =
+ configureCluster(4)
+ .withJettyConfig(jetty -> jetty.enableV2(true))
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ try {
+ String FILE1 = "/mypkg/runtimelibs.jar";
+ String FILE2 = "/mypkg/runtimelibs_v2.jar";
+ String FILE3 = "/mypkg/runtimelibs_v3.jar";
+ String COLLECTION_NAME = "testPluginLoadingColl";
+ byte[] derFile = readFile("cryptokeys/pub_key512.der");
+ cluster.getZkClient().makePath("/keys/exe", true);
+ cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
+ postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1,
+ "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ==");
+
+ Package.AddVersion add = new Package.AddVersion();
+ add.version = "1.0";
+ add.pkg = "mypkg";
+ add.files = Arrays.asList(new String[]{FILE1});
+ V2Request req = new V2Request.Builder("/cluster/package")
+ .forceV2(true)
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload(Collections.singletonMap("add", add))
+ .build();
+
+ req.process(cluster.getSolrClient());
+
+
+ CollectionAdminRequest
+ .createCollection(COLLECTION_NAME, "conf", 2, 2)
+ .setMaxShardsPerNode(100)
+ .process(cluster.getSolrClient());
+ cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4);
+
+ TestDistribPackageStore.assertResponseValues(10,
+ () -> new V2Request.Builder("/cluster/package").
+ withMethod(SolrRequest.METHOD.GET)
+ .build().process(cluster.getSolrClient()),
+ Utils.makeMap(
+ ":result:packages:mypkg[0]:version", "1.0",
+ ":result:packages:mypkg[0]:files[0]", FILE1
+ ));
+
+ String payload = "{\n" +
+ "'create-requesthandler' : { 'name' : '/runtime', 'class': 'mypkg:org.apache.solr.core.RuntimeLibReqHandler' }," +
+ "'create-searchcomponent' : { 'name' : 'get', 'class': 'mypkg:org.apache.solr.core.RuntimeLibSearchComponent' }," +
+ "'create-queryResponseWriter' : { 'name' : 'json1', 'class': 'mypkg:org.apache.solr.core.RuntimeLibResponseWriter' }" +
+ "}";
+ cluster.getSolrClient().request(new ConfigRequest(payload) {
+ @Override
+ public String getCollection() {
+ return COLLECTION_NAME;
+ }
+ });
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "queryResponseWriter", "json1",
+ "mypkg", "1.0" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "searchComponent", "get",
+ "mypkg", "1.0" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "requestHandler", "/runtime",
+ "mypkg", "1.0" );
+
+
+
+ executeReq( "/" + COLLECTION_NAME + "/runtime?wt=javabin", cluster.getRandomJetty(random()),
+ Utils.JAVABINCONSUMER,
+ Utils.makeMap("class", "org.apache.solr.core.RuntimeLibReqHandler"));
+
+ executeReq( "/" + COLLECTION_NAME + "/get?wt=json", cluster.getRandomJetty(random()),
+ Utils.JSONCONSUMER,
+ Utils.makeMap("class", "org.apache.solr.core.RuntimeLibSearchComponent",
+ "Version","1"));
+
+
+ executeReq( "/" + COLLECTION_NAME + "/runtime?wt=json1", cluster.getRandomJetty(random()),
+ Utils.JSONCONSUMER,
+ Utils.makeMap("wt", "org.apache.solr.core.RuntimeLibResponseWriter"));
+
+ //now upload the second jar
+ postFileAndWait(cluster, "runtimecode/runtimelibs_v2.jar.bin", FILE2,
+ "j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA==");
+
+ //add the version using package API
+ add.version = "1.1";
+ add.files = Arrays.asList(new String[]{FILE2});
+ req.process(cluster.getSolrClient());
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "queryResponseWriter", "json1",
+ "mypkg", "1.1" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "searchComponent", "get",
+ "mypkg", "1.1" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "requestHandler", "/runtime",
+ "mypkg", "1.1" );
+
+ executeReq( "/" + COLLECTION_NAME + "/get?wt=json", cluster.getRandomJetty(random()),
+ Utils.JSONCONSUMER,
+ Utils.makeMap( "Version","2"));
+
+
+ //now upload the third jar
+ postFileAndWait(cluster, "runtimecode/runtimelibs_v3.jar.bin", FILE3,
+ "a400n4T7FT+2gM0SC6+MfSOExjud8MkhTSFylhvwNjtWwUgKdPFn434Wv7Qc4QEqDVLhQoL3WqYtQmLPti0G4Q==");
+
+ add.version = "2.1";
+ add.files = Arrays.asList(new String[]{FILE3});
+ req.process(cluster.getSolrClient());
+
+ //now let's verify that the classes are updated
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "queryResponseWriter", "json1",
+ "mypkg", "2.1" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "searchComponent", "get",
+ "mypkg", "2.1" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "requestHandler", "/runtime",
+ "mypkg", "2.1" );
+
+ executeReq( "/" + COLLECTION_NAME + "/runtime?wt=json", cluster.getRandomJetty(random()),
+ Utils.JSONCONSUMER,
+ Utils.makeMap("Version","2"));
+
+
+ Package.DelVersion delVersion = new Package.DelVersion();
+ delVersion.pkg = "mypkg";
+ delVersion.version = "1.0";
+ V2Request delete = new V2Request.Builder("/cluster/package")
+ .withMethod(SolrRequest.METHOD.POST)
+ .forceV2(true)
+ .withPayload(Collections.singletonMap("delete", delVersion))
+ .build();
+ delete.process(cluster.getSolrClient());
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "queryResponseWriter", "json1",
+ "mypkg", "2.1" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "searchComponent", "get",
+ "mypkg", "2.1" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "requestHandler", "/runtime",
+ "mypkg", "2.1" );
+
+ // now remove the hughest version. So, it will roll back to the next highest one
+ delVersion.version = "2.1";
+ delete.process(cluster.getSolrClient());
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "queryResponseWriter", "json1",
+ "mypkg", "1.1" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "searchComponent", "get",
+ "mypkg", "1.1" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "requestHandler", "/runtime",
+ "mypkg", "1.1" );
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add("collection", COLLECTION_NAME);
+ new GenericSolrRequest(SolrRequest.METHOD.POST, "/config/params", params ){
+ @Override
+ public RequestWriter.ContentWriter getContentWriter(String expectedType) {
+ return new RequestWriter.StringPayloadContentWriter("{set:{PKG_VERSIONS:{mypkg : '1.1'}}}",
+ ClientUtils.TEXT_JSON);
+ }
+ }.process(cluster.getSolrClient()) ;
+
+ add.version = "2.1";
+ add.files = Arrays.asList(new String[]{FILE3});
+ req.process(cluster.getSolrClient());
+
+ //the collections mypkg is set to use version 1.1
+ //so no upgrade
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "queryResponseWriter", "json1",
+ "mypkg", "1.1" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "searchComponent", "get",
+ "mypkg", "1.1" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "requestHandler", "/runtime",
+ "mypkg", "1.1" );
+
+ new GenericSolrRequest(SolrRequest.METHOD.POST, "/config/params", params ){
+ @Override
+ public RequestWriter.ContentWriter getContentWriter(String expectedType) {
+ return new RequestWriter.StringPayloadContentWriter("{set:{PKG_VERSIONS:{mypkg : '2.1'}}}",
+ ClientUtils.TEXT_JSON);
+ }
+ }.process(cluster.getSolrClient()) ;
+
+ //now, let's force every collection using 'mypkg' to refresh
+ //so that it uses version 2.1
+ new V2Request.Builder("/cluster/package")
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload("{refresh : mypkg}")
+ .forceV2(true)
+ .build()
+ .process(cluster.getSolrClient());
+
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "queryResponseWriter", "json1",
+ "mypkg", "2.1" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "searchComponent", "get",
+ "mypkg", "2.1" );
+
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "requestHandler", "/runtime",
+ "mypkg", "2.1" );
+
+ } finally {
+ cluster.shutdown();
+ }
+
+ }
+
+ private void executeReq(String uri, JettySolrRunner jetty, Utils.InputStreamConsumer parser, Map expected) throws Exception {
+ try(HttpSolrClient client = (HttpSolrClient) jetty.newClient()){
+ TestDistribPackageStore.assertResponseValues(10,
+ () -> {
+ Object o = Utils.executeGET(client.getHttpClient(),
+ jetty.getBaseUrl() + uri, parser);
+ if(o instanceof NavigableObject) return (NavigableObject) o;
+ if(o instanceof Map) return new MapWriterMap((Map) o);
+ throw new RuntimeException("Unknown response");
+ }, expected);
+
+ }
+ }
+
+ private void verifyCmponent(SolrClient client, String COLLECTION_NAME,
+ String componentType, String componentName, String pkg, String version) throws Exception {
+ SolrParams params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME,
+ WT, JAVABIN,
+ "componentName", componentName,
+ "meta", "true"));
+
+ String s = "queryResponseWriter";
+ GenericSolrRequest req1 = new GenericSolrRequest(SolrRequest.METHOD.GET,
+ "/config/" + componentType, params);
+ TestDistribPackageStore.assertResponseValues(10,
+ client,
+ req1, Utils.makeMap(
+ ":config:" + componentType + ":" + componentName + ":_packageinfo_:package", pkg,
+ ":config:" + componentType + ":" + componentName + ":_packageinfo_:version", version
+ ));
+ }
+
+ @Test
+ public void testAPI() throws Exception {
+ System.setProperty("enable.packages", "true");
+ MiniSolrCloudCluster cluster =
+ configureCluster(4)
+ .withJettyConfig(jetty -> jetty.enableV2(true))
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ try {
+ String errPath = "/error/details[0]/errorMessages[0]";
+ String FILE1 = "/mypkg/v.0.12/jar_a.jar";
+ String FILE2 = "/mypkg/v.0.12/jar_b.jar";
+ String FILE3 = "/mypkg/v.0.13/jar_a.jar";
+
+ Package.AddVersion add = new Package.AddVersion();
+ add.version = "0.12";
+ add.pkg = "test_pkg";
+ add.files = Arrays.asList(new String[]{FILE1, FILE2});
+ V2Request req = new V2Request.Builder("/cluster/package")
+ .forceV2(true)
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload(Collections.singletonMap("add", add))
+ .build();
+
+ //the files is not yet there. The command should fail with error saying "No such file"
+ expectError(req, cluster.getSolrClient(), errPath, "No such file :");
+
+
+ //post the jar file. No signature is sent
+ postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1, null);
+
+
+ add.files = Arrays.asList(new String[]{FILE1});
+ expectError(req, cluster.getSolrClient(), errPath,
+ FILE1 + " has no signature");
+ //now we upload the keys
+ byte[] derFile = readFile("cryptokeys/pub_key512.der");
+ cluster.getZkClient().makePath("/keys/exe", true);
+ cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
+ //and upload the same file with a different name but it has proper signature
+ postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE2,
+ "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ==");
+ // with correct signature
+ //after uploading the file, let's delete the keys to see if we get proper error message
+ cluster.getZkClient().delete("/keys/exe/pub_key512.der", -1, true);
+ add.files = Arrays.asList(new String[]{FILE2});
+ expectError(req, cluster.getSolrClient(), errPath,
+ "ZooKeeper does not have any public keys");
+
+ //Now lets' put the keys back
+ cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
+
+ //this time we have a file with proper signature, public keys are in ZK
+ // so the add {} command should succeed
+ req.process(cluster.getSolrClient());
+
+ //Now verify the data in ZK
+ TestDistribPackageStore.assertResponseValues(1,
+ () -> new MapWriterMap((Map) Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH,
+ null, new Stat(), true))),
+ Utils.makeMap(
+ ":packages:test_pkg[0]:version", "0.12",
+ ":packages:test_pkg[0]:files[0]", FILE1
+ ));
+
+ //post a new jar with a proper signature
+ postFileAndWait(cluster, "runtimecode/runtimelibs_v2.jar.bin", FILE3,
+ "j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA==");
+
+
+ //this time we are adding the second version of the package (0.13)
+ add.version = "0.13";
+ add.pkg = "test_pkg";
+ add.files = Arrays.asList(new String[]{FILE3});
+
+ //this request should succeed
+ req.process(cluster.getSolrClient());
+ //no verify the data (/packages.json) in ZK
+ TestDistribPackageStore.assertResponseValues(1,
+ () -> new MapWriterMap((Map) Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH,
+ null, new Stat(), true))),
+ Utils.makeMap(
+ ":packages:test_pkg[1]:version", "0.13",
+ ":packages:test_pkg[1]:files[0]", FILE3
+ ));
+
+ //Now we will just delete one version
+ Package.DelVersion delVersion = new Package.DelVersion();
+ delVersion.version = "0.1";//this version does not exist
+ delVersion.pkg = "test_pkg";
+ req = new V2Request.Builder("/cluster/package")
+ .forceV2(true)
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload(Collections.singletonMap("delete", delVersion))
+ .build();
+
+ //we are expecting an error
+ expectError(req, cluster.getSolrClient(), errPath, "No such version:");
+
+ delVersion.version = "0.12";//correct version. Should succeed
+ req.process(cluster.getSolrClient());
+ //Verify with ZK that the data is correcy
+ TestDistribPackageStore.assertResponseValues(1,
+ () -> new MapWriterMap((Map) Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH,
+ null, new Stat(), true))),
+ Utils.makeMap(
+ ":packages:test_pkg[0]:version", "0.13",
+ ":packages:test_pkg[0]:files[0]", FILE2
+ ));
+
+
+ //So far we have been verifying the details with ZK directly
+ //use the package read API to verify with each node that it has the correct data
+ for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
+ String path = jetty.getBaseUrl().toString().replace("/solr", "/api") + "/cluster/package?wt=javabin";
+ TestDistribPackageStore.assertResponseValues(10, new Callable<NavigableObject>() {
+ @Override
+ public NavigableObject call() throws Exception {
+ try (HttpSolrClient solrClient = (HttpSolrClient) jetty.newClient()) {
+ return (NavigableObject) Utils.executeGET(solrClient.getHttpClient(), path, Utils.JAVABINCONSUMER);
+ }
+ }
+ }, Utils.makeMap(
+ ":result:packages:test_pkg[0]:version", "0.13",
+ ":result:packages:test_pkg[0]:files[0]", FILE3
+ ));
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ static void postFileAndWait(MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception {
+ ByteBuffer fileContent = getFileContent(fname);
+ String sha512 = DigestUtils.sha512Hex(fileContent.array());
+
+ TestDistribPackageStore.postFile(cluster.getSolrClient(),
+ fileContent,
+ path, sig);// has file, but no signature
+
+ TestDistribPackageStore.waitForAllNodesHaveFile(cluster, path, Utils.makeMap(
+ ":files:" + path + ":sha512",
+ sha512
+ ), false);
+ }
+
+ private void expectError(V2Request req, SolrClient client, String errPath, String expectErrorMsg) throws IOException, SolrServerException {
+ try {
+ req.process(client);
+ fail("should have failed with message : " + expectErrorMsg);
+ } catch (BaseHttpSolrClient.RemoteExecutionException e) {
+ String msg = e.getMetaData()._getStr(errPath, "");
+ assertTrue("should have failed with message: " + expectErrorMsg + "actual message : " + msg,
+ msg.contains(expectErrorMsg)
+ );
+ }
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/update/processor/RuntimeUrp.java b/solr/core/src/test/org/apache/solr/update/processor/RuntimeUrp.java
index 889b0bf..6cee3d9 100644
--- a/solr/core/src/test/org/apache/solr/update/processor/RuntimeUrp.java
+++ b/solr/core/src/test/org/apache/solr/update/processor/RuntimeUrp.java
@@ -31,7 +31,7 @@ public class RuntimeUrp extends SimpleUpdateProcessorFactory {
List<String> names = new ArrayList<>();
for (UpdateRequestProcessorFactory p : processorChain.getProcessors()) {
if (p instanceof UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder.LazyUpdateRequestProcessorFactory) {
- p = ((UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder.LazyUpdateRequestProcessorFactory) p).delegate;
+ p = ((UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder.LazyUpdateRequestProcessorFactory) p).getDelegate();
}
names.add(p.getClass().getSimpleName());
}
diff --git a/solr/core/src/test/org/apache/solr/util/TestSolrJacksonAnnotation.java b/solr/core/src/test/org/apache/solr/util/TestSolrJacksonAnnotation.java
new file mode 100644
index 0000000..f0b9036
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/util/TestSolrJacksonAnnotation.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util;
+
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.solr.SolrTestCase;
+import org.apache.solr.common.annotation.JsonProperty;
+import org.apache.solr.common.util.JsonSchemaCreator;
+import org.apache.solr.common.util.JsonSchemaValidator;
+import org.apache.solr.common.util.Utils;
+
+public class TestSolrJacksonAnnotation extends SolrTestCase {
+
+ public void testSerDe() throws Exception {
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.setAnnotationIntrospector(new SolrJacksonAnnotationInspector());
+
+ TestObj o = new TestObj();
+ o.field = "v1";
+ o.f2 = "v2";
+ o.ifld = 1234;
+ String json = mapper.writeValueAsString(o);
+
+ Map m = (Map) Utils.fromJSONString(json);
+ assertEquals("v1", m.get("field"));
+ assertEquals("v2", m.get("friendlyName"));
+ assertEquals("1234", String.valueOf(m.get("friendlyIntFld")));
+ TestObj o1 = mapper.readValue(json, TestObj.class);
+
+ assertEquals("v1", o1.field);
+ assertEquals("v2", o1.f2);
+ assertEquals(1234, o1.ifld);
+
+ Map<String, Object> schema = JsonSchemaCreator.getSchema(TestObj.class);
+ assertEquals("string", Utils.getObjectByPath(schema,true,"/properties/friendlyName/type"));
+ assertEquals("integer", Utils.getObjectByPath(schema,true,"/properties/friendlyIntFld/type"));
+ assertEquals("friendlyName", Utils.getObjectByPath(schema,true,"/required[0]"));
+
+
+ JsonSchemaValidator validator = new JsonSchemaValidator(schema);
+ List<String> errs = validator.validateJson(m);
+ assertNull(errs);
+ m.remove("friendlyName");
+ errs = validator.validateJson(m);
+ assertFalse(errs.isEmpty());
+ assertTrue(errs.get(0).contains("Missing required attribute"));
+ m.put("friendlyIntFld", Boolean.TRUE);
+ errs = validator.validateJson(m);
+ m.put("friendlyIntFld", "another String");
+ assertTrue(errs.get(0).contains("Value is not valid"));
+ }
+
+
+
+
+ public static class TestObj {
+ @JsonProperty()
+ public String field;
+ @JsonProperty(value = "friendlyName" ,required = true)
+ public String f2;
+ @JsonProperty("friendlyIntFld")
+ public int ifld;
+ }
+}
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ZplotStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ZplotStream.java
index 0a92fa7..7ad6586 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ZplotStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ZplotStream.java
@@ -37,8 +37,8 @@ import org.apache.solr.client.solrj.io.Tuple;
import org.apache.solr.client.solrj.io.comp.StreamComparator;
import org.apache.solr.client.solrj.io.eval.DbscanEvaluator;
import org.apache.solr.client.solrj.io.eval.KmeansEvaluator;
-import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
import org.apache.solr.client.solrj.io.eval.Matrix;
+import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
import org.apache.solr.client.solrj.io.stream.expr.Explanation;
import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
import org.apache.solr.client.solrj.io.stream.expr.Expressible;
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java
new file mode 100644
index 0000000..bda1078
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.request.beans;
+
+import java.util.List;
+
+import org.apache.solr.common.annotation.JsonProperty;
+import org.apache.solr.common.util.ReflectMapWriter;
+
+/**Just a container class for POJOs used in Package APIs
+ *
+ */
+public class Package {
+ public static class AddVersion implements ReflectMapWriter {
+ @JsonProperty(value = "package", required = true)
+ public String pkg;
+ @JsonProperty(required = true)
+ public String version;
+ @JsonProperty(required = true)
+ public List<String> files;
+
+ }
+
+ public static class DelVersion implements ReflectMapWriter {
+ @JsonProperty(value = "package", required = true)
+ public String pkg;
+ @JsonProperty(required = true)
+ public String version;
+
+ }
+}
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/pakage-info.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/pakage-info.java
new file mode 100644
index 0000000..76c2606
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/pakage-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Annotated Data objects used in V2 Requests
+ */
+package org.apache.solr.client.solrj.request.beans;
+
+
diff --git a/solr/solrj/src/java/org/apache/solr/common/annotation/JsonProperty.java b/solr/solrj/src/java/org/apache/solr/common/annotation/JsonProperty.java
new file mode 100644
index 0000000..d6b5da5
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/common/annotation/JsonProperty.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**Provides a 1:1 mapping from jackson's annotation. This was created to avoid direct dependency on jackson in SolrJ
+ *
+ */
+@Target({ElementType.FIELD, ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface JsonProperty {
+
+ String value() default "";
+
+ boolean required() default false;
+// not implemented
+// int index() default -1;
+
+ String defaultValue() default "";
+}
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
index 3d22d13..fc21b57 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
@@ -34,6 +34,7 @@ import java.nio.file.Path;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.RejectedExecutionException;
+import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.regex.Pattern;
@@ -353,6 +354,10 @@ public class SolrZkClient implements Closeable {
}
public void atomicUpdate(String path, Function<byte[], byte[]> editor) throws KeeperException, InterruptedException {
+ atomicUpdate(path, (stat, bytes) -> editor.apply(bytes));
+ }
+
+ public void atomicUpdate(String path, BiFunction<Stat , byte[], byte[]> editor) throws KeeperException, InterruptedException {
for (; ; ) {
byte[] modified = null;
byte[] zkData = null;
@@ -360,7 +365,7 @@ public class SolrZkClient implements Closeable {
try {
if (exists(path, true)) {
zkData = getData(path, null, s, true);
- modified = editor.apply(zkData);
+ modified = editor.apply(s, zkData);
if (modified == null) {
//no change , no need to persist
return;
@@ -368,7 +373,7 @@ public class SolrZkClient implements Closeable {
setData(path, modified, s.getVersion(), true);
break;
} else {
- modified = editor.apply(null);
+ modified = editor.apply(s,null);
if (modified == null) {
//no change , no need to persist
return;
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index 816ddb7..aad4406 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -120,6 +120,7 @@ public class ZkStateReader implements SolrCloseable {
public static final String SOLR_AUTOSCALING_TRIGGER_STATE_PATH = "/autoscaling/triggerState";
public static final String SOLR_AUTOSCALING_NODE_ADDED_PATH = "/autoscaling/nodeAdded";
public static final String SOLR_AUTOSCALING_NODE_LOST_PATH = "/autoscaling/nodeLost";
+ public static final String SOLR_PKGS_PATH = "/packages.json";
public static final String DEFAULT_SHARD_PREFERENCES = "defaultShardPreferences";
public static final String REPLICATION_FACTOR = "replicationFactor";
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaCreator.java b/solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaCreator.java
new file mode 100644
index 0000000..028c09b
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaCreator.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.util;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.solr.common.annotation.JsonProperty;
+
+/**Creates a json schema from an annotated Java Object. This, by no means, is an exhaustive impl
+ * of schema generation. We will expand the scope as we use more types
+ *
+ */
+
+public class JsonSchemaCreator {
+ public static final Map<Class, String> natives = new HashMap<>();
+
+ static {
+ natives.put(String.class, "string");
+ natives.put(Integer.class, "integer");
+ natives.put(int.class, "integer");
+ natives.put(Float.class, "number");
+ natives.put(float.class, "number");
+ natives.put(Double.class, "number");
+ natives.put(double.class, "number");
+ natives.put(Boolean.class, "boolean");
+ natives.put(List.class, "array");
+ }
+
+ public static Map<String, Object> getSchema(java.lang.reflect.Type t) {
+ return createSchemaFromType(t, new LinkedHashMap<>());
+ }
+
+ private static Map<String, Object> createSchemaFromType(java.lang.reflect.Type t, Map<String, Object> map) {
+ if (natives.containsKey(t)) {
+ map.put("type", natives.get(t));
+ } else if (t instanceof ParameterizedType && ((ParameterizedType) t).getRawType() == List.class) {
+ Type typ = ((ParameterizedType) t).getActualTypeArguments()[0];
+ map.put("type", "array");
+ map.put("items", getSchema(typ));
+ } else {
+ createObjectSchema((Class) t, map);
+ }
+ return map;
+ }
+
+ private static void createObjectSchema(Class klas, Map<String, Object> map) {
+ map.put("type", "object");
+ Map<String, Object> props = new HashMap<>();
+ map.put("properties", props);
+ Set<String> required = new HashSet<>();
+ for (Field fld : klas.getDeclaredFields()) {
+ JsonProperty p = fld.getAnnotation(JsonProperty.class);
+ if (p == null) continue;
+ String name = p.value().isEmpty() ? fld.getName() : p.value();
+ props.put(name, getSchema(fld.getGenericType()));
+ if(p.required()) required.add(name);
+ }
+ if(!required.isEmpty()) map.put("required", new ArrayList<>(required));
+
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/util/ReflectMapWriter.java b/solr/solrj/src/java/org/apache/solr/common/util/ReflectMapWriter.java
similarity index 92%
rename from solr/core/src/java/org/apache/solr/util/ReflectMapWriter.java
rename to solr/solrj/src/java/org/apache/solr/common/util/ReflectMapWriter.java
index 9555740..1142e62 100644
--- a/solr/core/src/java/org/apache/solr/util/ReflectMapWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/ReflectMapWriter.java
@@ -15,15 +15,16 @@
* limitations under the License.
*/
-package org.apache.solr.util;
+package org.apache.solr.common.util;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
-import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.annotation.JsonProperty;
+// An implementation of MapWriter which is annotated with Jackson annotations
public interface ReflectMapWriter extends MapWriter {
@Override
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
index 2afbfc1..7e510f4 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
@@ -21,15 +21,22 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.SolrTestCase;
-import org.apache.solr.client.solrj.io.eval.*;
+import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorDay;
+import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorDayOfQuarter;
+import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorDayOfYear;
+import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorEpoch;
+import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorHour;
+import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorMinute;
+import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorMonth;
+import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorQuarter;
+import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorSecond;
+import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorWeek;
+import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorYear;
import org.apache.solr.client.solrj.io.stream.expr.Expressible;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-
import org.junit.Test;
@Slow