You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/03/10 09:45:47 UTC
[lucene] 06/21: more tests. improved schema API to emit package
details
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch jira/solr14155
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 58197f20e3ee2c686433326c4b442341815cbd65
Author: noble <no...@apache.org>
AuthorDate: Thu Dec 26 19:40:43 2019 +1100
more tests. improved schema API to emit package details
---
.../src/java/org/apache/solr/core/PluginInfo.java | 26 +++++++-----
.../src/java/org/apache/solr/core/SolrCore.java | 1 +
.../org/apache/solr/handler/SchemaHandler.java | 39 ++++++++++++++++++
.../solr/handler/component/SearchHandler.java | 3 +-
.../java/org/apache/solr/pkg/PackageListeners.java | 10 ++++-
.../java/org/apache/solr/pkg/PackageLoader.java | 4 ++
.../org/apache/solr/pkg/PackagePluginHolder.java | 43 +++++++++++---------
.../java/org/apache/solr/schema/IndexSchema.java | 33 +++++++++++-----
.../src/test/org/apache/solr/pkg/TestPackages.java | 46 ++++++++++++++++++++++
9 files changed, 166 insertions(+), 39 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/core/PluginInfo.java b/solr/core/src/java/org/apache/solr/core/PluginInfo.java
index afe4073..cfef933 100644
--- a/solr/core/src/java/org/apache/solr/core/PluginInfo.java
+++ b/solr/core/src/java/org/apache/solr/core/PluginInfo.java
@@ -41,6 +41,7 @@ import static org.apache.solr.schema.FieldType.CLASS_NAME;
*/
public class PluginInfo implements MapSerializable {
public final String name, className, type, pkgName;
+ public final ClassName cName;
public final NamedList initArgs;
public final Map<String, String> attributes;
public final List<PluginInfo> children;
@@ -51,9 +52,9 @@ public class PluginInfo implements MapSerializable {
public PluginInfo(String type, Map<String, String> attrs, NamedList initArgs, List<PluginInfo> children) {
this.type = type;
this.name = attrs.get(NAME);
- ClassName parsed = new ClassName(attrs.get(CLASS_NAME));
- this.className = parsed.klas;
- this.pkgName = parsed.pkg;
+ cName = new ClassName(attrs.get(CLASS_NAME));
+ this.className = cName.klas;
+ this.pkgName = cName.pkg;
this.initArgs = initArgs;
attributes = unmodifiableMap(attrs);
this.children = children == null ? Collections.<PluginInfo>emptyList(): unmodifiableList(children);
@@ -67,8 +68,10 @@ public class PluginInfo implements MapSerializable {
public static class ClassName {
public final String pkg;
public final String klas;
+ private final String actual;
public ClassName(String name) {
+ this.actual = name;
String pkgName = null;
String className = name;
if (name != null) {
@@ -82,15 +85,20 @@ public class PluginInfo implements MapSerializable {
this.klas = className;
this.pkg = pkgName;
}
+
+ @Override
+ public String toString() {
+ return actual;
+ }
}
public PluginInfo(Node node, String err, boolean requireName, boolean requireClass) {
type = node.getNodeName();
name = DOMUtil.getAttr(node, NAME, requireName ? err : null);
- ClassName parsed = new ClassName(DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null));
- className = parsed.klas;
- pkgName = parsed.pkg;
+ cName = new ClassName(DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null));
+ className = cName.klas;
+ pkgName = cName.pkg;
initArgs = DOMUtil.childNodesToNamedList(node);
attributes = unmodifiableMap(DOMUtil.toMap(node.getAttributes()));
children = loadSubPlugins(node);
@@ -120,9 +128,9 @@ public class PluginInfo implements MapSerializable {
}
this.type = type;
this.name = (String) m.get(NAME);
- ClassName parsed = new ClassName((String) m.get(CLASS_NAME));
- this.className = parsed.klas;
- this.pkgName = parsed.pkg;
+ cName = new ClassName((String) m.get(CLASS_NAME));
+ this.className = cName.klas;
+ this.pkgName = cName.pkg;
attributes = unmodifiableMap(m);
this.children = Collections.<PluginInfo>emptyList();
isFromSolrConfig = true;
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index e7c1be5..284bf66 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -909,6 +909,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
}
public void refreshSchema() {
+ log.info("Reloading schema...");
IndexSchema old = schema;
schema = schemaSupplier.get();
closeWhileHandlingException(old);
diff --git a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
index 5cbe9aa..c625534 100644
--- a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
@@ -24,7 +24,10 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
import org.apache.solr.api.Api;
import org.apache.solr.api.ApiBag;
@@ -32,10 +35,13 @@ import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.pkg.PackageListeners;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.SolrQueryResponse;
@@ -179,6 +185,7 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware,
SimpleOrderedMap simpleOrderedMap = (SimpleOrderedMap) obj;
if(name.equals(simpleOrderedMap.get("name"))) {
rsp.add(fieldName.substring(0, realName.length() - 1), simpleOrderedMap);
+ insertPackageInfo(rsp.getValues(), req);
return;
}
}
@@ -188,6 +195,7 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware,
} else {
rsp.add(fieldName, o);
}
+ insertPackageInfo(rsp.getValues(), req);
return;
}
@@ -200,6 +208,37 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware,
}
}
+ private void insertPackageInfo(Object o, SolrQueryRequest req) {
+ if(!req.getParams().getBool("meta")) return;
+ if (o instanceof List) {
+ List l = (List) o;
+ for (Object o1 : l) {
+ if (o1 instanceof NamedList || o1 instanceof List) insertPackageInfo(o1, req);
+ }
+
+ } else if (o instanceof NamedList) {
+ NamedList nl = (NamedList) o;
+ nl.forEach((BiConsumer) (n, v) -> {
+ if (v instanceof NamedList || v instanceof List) insertPackageInfo(v, req);
+ });
+ Object v = nl.get("class");
+ if (v instanceof String) {
+ String klas = (String) v;
+ PluginInfo.ClassName className = new PluginInfo.ClassName(klas);
+ if (className.pkg != null) {
+ req.getCore().getPackageListeners().forEachListener(listener -> {
+ if (listener.pluginInfo() == null) return;
+ if (Objects.equals(listener.pluginInfo().cName.toString(), className.toString())) {
+ nl.add("_packageinfo_", listener.getPackageVersion());
+ }
+ });
+ }
+ }
+
+ }
+
+ }
+
private static Set<String> subPaths = new HashSet<>(Arrays.asList(
"version",
"uniquekey",
diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
index 5e48262..7bcb165 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
@@ -40,6 +40,7 @@ import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.pkg.PackageAPI;
import org.apache.solr.pkg.PackageListeners;
import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.request.SolrQueryRequest;
@@ -161,7 +162,7 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware,
}
@Override
- public PackageLoader.Package.Version getPackageVersion() {
+ public PackageAPI.PkgVersion getPackageVersion() {
return null;
}
});
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java
index 5802ee0..3d13ad8 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.function.Consumer;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
@@ -96,6 +97,13 @@ public class PackageListeners {
}
}
+ public void forEachListener(Consumer<Listener> listenerConsumer){
+ listeners.forEach(ref -> {
+ Listener l = ref.get();
+ if (l != null) listenerConsumer.accept(l);
+ });
+ }
+
public List<Listener> getListeners() {
List<Listener> result = new ArrayList<>();
for (Reference<Listener> ref : listeners) {
@@ -131,7 +139,7 @@ public class PackageListeners {
void changed(PackageLoader.Package pkg, Ctx ctx);
- PackageLoader.Package.Version getPackageVersion();
+ PackageAPI.PkgVersion getPackageVersion();
}
}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java
index 8783d9b..b7a88d1 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java
@@ -244,6 +244,10 @@ public class PackageLoader implements Closeable {
version.writeMap(ew);
}
+ public PackageAPI.PkgVersion getVersionInfo(){
+ return version;
+ }
+
Version(Package parent, PackageAPI.PkgVersion v) {
this.parent = parent;
this.version = v;
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
index 50bf567..b15c524 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
@@ -62,30 +62,48 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
}
@Override
- public PackageLoader.Package.Version getPackageVersion() {
- return pkgVersion;
+ public PackageAPI.PkgVersion getPackageVersion() {
+ return pkgVersion.getVersionInfo();
}
});
}
- private String maxVersion() {
+ private static String maxAllowedVersion(SolrCore core , String pkgName) {
RequestParams.ParamSet p = core.getSolrConfig().getRequestParams().getParams(PackageListeners.PACKAGE_VERSIONS);
if (p == null) {
return null;
}
- Object o = p.get().get(info.pkgName);
+ Object o = p.get().get(pkgName);
if (o == null || LATEST.equals(o)) return null;
return o.toString();
}
private synchronized void reload(PackageLoader.Package pkg) {
- String lessThan = maxVersion();
+ PackageLoader.Package.Version rightVersion = getRightVersion(pkg, core);
+ if (pkgVersion != null) {
+ if (rightVersion == pkgVersion) {
+ //I'm already using the latest classloader in the package. nothing to do
+ return ;
+ }
+ }
+ if (rightVersion == null) return;
+
+ log.info("loading plugin: {} -> {} using package {}:{}",
+ pluginInfo.type, pluginInfo.name, pkg.name(), rightVersion.getVersion());
+
+ initNewInstance(rightVersion);
+ pkgVersion = rightVersion;
+
+ }
+
+ public static PackageLoader.Package.Version getRightVersion(PackageLoader.Package pkg, SolrCore core ) {
+ String lessThan = maxAllowedVersion(core, pkg.name);
PackageLoader.Package.Version newest = pkg.getLatest(lessThan);
if (newest == null) {
log.error("No latest version available for package : {}", pkg.name());
- return;
+ return null;
}
if (lessThan != null) {
PackageLoader.Package.Version pkgLatest = pkg.getLatest();
@@ -94,19 +112,8 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
}
}
- if (pkgVersion != null) {
- if (newest == pkgVersion) {
- //I'm already using the latest classloder in the package. nothing to do
- return;
- }
- }
-
- log.info("loading plugin: {} -> {} using package {}:{}",
- pluginInfo.type, pluginInfo.name, pkg.name(), newest.getVersion());
-
- initNewInstance(newest);
- pkgVersion = newest;
+ return newest;
}
protected void initNewInstance(PackageLoader.Package.Version newest) {
diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
index d4273a8..35ecdfd 100644
--- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
+++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
@@ -62,13 +62,16 @@ import org.apache.solr.common.util.Cache;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.core.PluginBag;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.PluginLoader;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.core.XmlConfigFile;
+import org.apache.solr.pkg.PackageAPI;
import org.apache.solr.pkg.PackageListeners;
import org.apache.solr.pkg.PackageLoader;
+import org.apache.solr.pkg.PackagePluginHolder;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.response.SchemaXmlWriter;
import org.apache.solr.response.SolrQueryResponse;
@@ -78,6 +81,7 @@ import org.apache.solr.util.ConcurrentLRUCache;
import org.apache.solr.util.DOMUtil;
import org.apache.solr.util.PayloadUtils;
import org.apache.solr.util.plugin.SolrCoreAware;
+import org.codehaus.janino.Java;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
@@ -164,6 +168,8 @@ public class IndexSchema implements Closeable {
private Map<FieldType, PayloadDecoder> decoders = new HashMap<>(); // cache to avoid scanning token filters repeatedly, unnecessarily
+ private Map<String, PackageAPI.PkgVersion> classNameVsPkg = new HashMap<>();
+
/**
* keys are all fields copied to, count is num of copyField
* directives that target them.
@@ -203,7 +209,7 @@ public class IndexSchema implements Closeable {
}
}
- class PackageAwarePluginLoader implements PluginLoader, Closeable {
+ public class PackageAwarePluginLoader implements PluginLoader, Closeable {
final SolrCore core;
private final List<PackageListeners.Listener> listeners = new ArrayList<>();
Runnable reloadSchemaRunnable = () -> getCore().refreshSchema();
@@ -219,19 +225,22 @@ public class IndexSchema implements Closeable {
@Override
public <T> T newInstance(String cname, Class<T> expectedType, String... subpackages) {
- return getIt(cname, expectedType,
+ return loadWithRightPackageLoader(cname, expectedType,
(pkgloader, name) -> pkgloader.newInstance(name, expectedType, subpackages));
}
- private <T> T getIt(String cname, Class expectedType, BiFunction<SolrResourceLoader, String, T> fun) {
+ private <T> T loadWithRightPackageLoader(String cname, Class expectedType, BiFunction<SolrResourceLoader, String, T> fun) {
PluginInfo.ClassName className = new PluginInfo.ClassName(cname);
if (className.pkg == null) {
return fun.apply(loader, className.klas);
} else {
- SolrResourceLoader pkgloader = core.getResourceLoader(className.pkg);
- T inst = fun.apply(pkgloader,className.klas);
+ PackageLoader.Package pkg = core.getCoreContainer().getPackageLoader().getPackage(className.pkg);
+ PackageLoader.Package.Version ver = PackagePluginHolder.getRightVersion(pkg, core);
+ T inst = fun.apply(ver.getLoader(),className.klas);
+ classNameVsPkg.put(cname, ver.getVersionInfo());
PackageListeners.Listener listener = new PackageListeners.Listener() {
- PluginInfo info = new PluginInfo(expectedType.getSimpleName(), singletonMap("class", className.klas));
+
+ PluginInfo info = new PluginInfo(expectedType.getSimpleName(), singletonMap("class", cname));
@Override
public String packageName() {
@@ -245,13 +254,17 @@ public class IndexSchema implements Closeable {
@Override
public void changed(PackageLoader.Package pkg, PackageListeners.Ctx ctx) {
+ PackageLoader.Package.Version rightVersion = PackagePluginHolder.getRightVersion(pkg, core);
+ if (rightVersion == null ) return;
+ PackageAPI.PkgVersion v = classNameVsPkg.get(className.toString());
+ if(Objects.equals(v.version ,rightVersion.getVersionInfo().version)) return; //nothing has changed no need to reload
Runnable old = ctx.getPostProcessor(PackageAwarePluginLoader.class.getName());// just want to do one refresh for every package laod
if (old == null) ctx.addPostProcessor(PackageAwarePluginLoader.class.getName(), reloadSchemaRunnable);
}
@Override
- public PackageLoader.Package.Version getPackageVersion() {
- return null;
+ public PackageAPI.PkgVersion getPackageVersion() {
+ return classNameVsPkg.get(cname);
}
};
listeners.add(listener);
@@ -262,12 +275,12 @@ public class IndexSchema implements Closeable {
@Override
public <T> Class<? extends T> findClass(String cname, Class<T> expectedType) {
- return getIt(cname, expectedType, (BiFunction<SolrResourceLoader, String ,Class<? extends T>>) (loader, name) -> loader.findClass(name, expectedType));
+ return loadWithRightPackageLoader(cname, expectedType, (BiFunction<SolrResourceLoader, String ,Class<? extends T>>) (loader, name) -> loader.findClass(name, expectedType));
}
@Override
public <T> T newInstance(String cName, Class<T> expectedType, String[] subPackages, Class[] params, Object[] args) {
- return getIt(cName, expectedType, (pkgloader, name) -> pkgloader.newInstance(name, expectedType, subPackages, params, args));
+ return loadWithRightPackageLoader(cName, expectedType, (pkgloader, name) -> pkgloader.newInstance(name, expectedType, subPackages, params, args));
}
diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
index fe46ef1..4a3ff7b 100644
--- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
@@ -18,6 +18,7 @@
package org.apache.solr.pkg;
import java.io.IOException;
+import java.io.StringWriter;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Collections;
@@ -52,6 +53,7 @@ import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Utils;
import org.apache.solr.filestore.TestDistribPackageStore;
import org.apache.solr.util.LogLevel;
@@ -169,7 +171,35 @@ public class TestPackages extends SolrCloudTestCase {
return new SolrResponseBase();
}
});
+ verifySchemaComponent(cluster.getSolrClient(), COLLECTION_NAME, "/schema/fieldtypes/myNewTextFieldWithAnalyzerClass",
+ Utils.makeMap(":fieldType:analyzer:charFilters[0]:_packageinfo_:version" ,"1.0",
+ ":fieldType:analyzer:tokenizer:_packageinfo_:version","1.0",
+ ":fieldType:_packageinfo_:version","1.0"));
+ add = new Package.AddVersion();
+ add.version = "2.0";
+ add.pkg = "schemapkg";
+ add.files = Arrays.asList(new String[]{FILE1});
+ req = new V2Request.Builder("/cluster/package")
+ .forceV2(true)
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload(Collections.singletonMap("add", add))
+ .build();
+ req.process(cluster.getSolrClient());
+
+ TestDistribPackageStore.assertResponseValues(10,
+ () -> new V2Request.Builder("/cluster/package").
+ withMethod(SolrRequest.METHOD.GET)
+ .build().process(cluster.getSolrClient()),
+ Utils.makeMap(
+ ":result:packages:schemapkg[0]:version", "2.0",
+ ":result:packages:schemapkg[0]:files[0]", FILE1
+ ));
+
+ verifySchemaComponent(cluster.getSolrClient(), COLLECTION_NAME, "/schema/fieldtypes/myNewTextFieldWithAnalyzerClass",
+ Utils.makeMap(":fieldType:analyzer:charFilters[0]:_packageinfo_:version" ,"2.0",
+ ":fieldType:analyzer:tokenizer:_packageinfo_:version","2.0",
+ ":fieldType:_packageinfo_:version","2.0"));
} finally {
cluster.shutdown();
@@ -499,6 +529,22 @@ public class TestPackages extends SolrCloudTestCase {
}
}
+ private void verifySchemaComponent(SolrClient client, String COLLECTION_NAME, String path,
+ Map expected) throws Exception {
+ SolrParams params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME,
+ WT, JAVABIN,
+ "meta", "true"));
+
+ GenericSolrRequest req = new GenericSolrRequest(SolrRequest.METHOD.GET,path
+ , params);
+ NamedList<Object> rsp = client.request(req);
+ System.out.println( Utils.writeJson(rsp, new StringWriter(), true).toString());
+ TestDistribPackageStore.assertResponseValues(10,
+ client,
+ req, expected);
+ }
+
+
private void verifyCmponent(SolrClient client, String COLLECTION_NAME,
String componentType, String componentName, String pkg, String version) throws Exception {
SolrParams params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME,