You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2020/09/18 07:03:00 UTC
[lucene-solr] branch branch_8x updated: SOLR-14151: support loading
encoder
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new c0fc134 SOLR-14151: support loading encoder
c0fc134 is described below
commit c0fc134e35ecf01a5392e9a51e17c16c2676f536
Author: noblepaul <no...@gmail.com>
AuthorDate: Fri Sep 18 17:02:39 2020 +1000
SOLR-14151: support loading encoder
---
.../org/apache/solr/core/SolrResourceLoader.java | 91 +++++++++++++++-------
.../org/apache/solr/schema/ManagedIndexSchema.java | 3 +
.../java/org/apache/solr/schema/SchemaManager.java | 8 ++
.../src/test/org/apache/solr/pkg/TestPackages.java | 19 ++++-
4 files changed, 91 insertions(+), 30 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
index ed97024..ccc1bd9 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
@@ -34,6 +34,8 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableSet;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.util.*;
import org.apache.lucene.codecs.Codec;
@@ -83,27 +85,6 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
private CoreContainer coreContainer;
private PackageListeningClassLoader schemaLoader ;
- private PackageListeningClassLoader createSchemaLoader() {
- CoreContainer cc = getCoreContainer();
- if (cc == null) {
- //corecontainer not available . can't load from packages
- return null;
- }
- return new PackageListeningClassLoader(cc, this, pkg -> {
- if (getSolrConfig() == null) return null;
- return getSolrConfig().maxPackageVersion(pkg);
- }, () -> {
- if(getCoreContainer() == null || config == null || coreName == null || coreId==null) return;
- try (SolrCore c = getCoreContainer().getCore(coreName, coreId)) {
- if (c != null) {
- c.fetchLatestSchema();
- }
- }
- });
- }
-
-
-
private final List<SolrCoreAware> waitingForCore = Collections.synchronizedList(new ArrayList<SolrCoreAware>());
private final List<SolrInfoBean> infoMBeans = Collections.synchronizedList(new ArrayList<SolrInfoBean>());
private final List<ResourceLoaderAware> waitingForResources = Collections.synchronizedList(new ArrayList<ResourceLoaderAware>());
@@ -115,7 +96,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
// Provide a registry so that managed resources can register themselves while the XML configuration
// documents are being parsed ... after all are registered, they are asked by the RestManager to
// initialize themselves. This two-step process is required because not all resources are available
- // (such as the SolrZkClient) when XML docs are being parsed.
+ // (such as the SolrZkClient) when XML docs are being parsed.
private RestManager.Registry managedResourceRegistry;
/** @see #reloadLuceneSPI() */
private boolean needToReloadLuceneSPI = false; // requires synchronization
@@ -514,6 +495,8 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
}
Class<? extends T> clazz = null;
+ clazz = getPackageClass(cname, expectedType);
+ if(clazz != null) return clazz;
try {
// first try legacy analysis patterns, now replaced by Lucene's Analysis package:
final Matcher m = legacyAnalysisPattern.matcher(cname);
@@ -576,6 +559,24 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
}
}
+ private <T> Class<? extends T> getPackageClass(String cname, Class<T> expectedType) {
+ PluginInfo.ClassName cName = PluginInfo.parseClassName(cname);
+ if (cName.pkg == null) return null;
+ ResourceLoaderAware aware = CURRENT_AWARE.get();
+ if (aware != null) {
+ //this is invoked from a component
+ //let's check if it's a schema component
+ @SuppressWarnings("rawtypes")
+ Class type = assertAwareCompatibility(ResourceLoaderAware.class, aware);
+ if (schemaResourceLoaderComponents.contains(type)) {
+ //this is a schema component
+ //lets use schema classloader
+ return getSchemaLoader().findClass(cname, expectedType);
+ }
+ }
+ return null;
+ }
+
static final String[] empty = new String[0];
@Override
@@ -716,8 +717,14 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
waitingForResources.clear();
}
- for( ResourceLoaderAware aware : arr) {
- aware.inform(loader);
+ for (ResourceLoaderAware aware : arr) {
+ CURRENT_AWARE.set(aware);
+ try{
+ aware.inform(loader);
+ } finally {
+ CURRENT_AWARE.remove();
+ }
+
}
}
}
@@ -802,11 +809,21 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
);
}
+ /**If these components are trying to load classes, use schema classloader
+ *
+ */
+ @SuppressWarnings("rawtypes")
+ private static final ImmutableSet<Class> schemaResourceLoaderComponents = ImmutableSet.of(
+ CharFilterFactory.class,
+ TokenFilterFactory.class,
+ TokenizerFactory.class,
+ FieldType.class);
+
/**
* Utility function to throw an exception if the class is invalid
*/
@SuppressWarnings({"rawtypes"})
- public static void assertAwareCompatibility(Class aware, Object obj) {
+ public static Class assertAwareCompatibility(Class aware, Object obj) {
Class[] valid = awareCompatibility.get( aware );
if( valid == null ) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
@@ -814,7 +831,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
}
for( Class v : valid ) {
if( v.isInstance( obj ) ) {
- return;
+ return v;
}
}
StringBuilder builder = new StringBuilder();
@@ -843,6 +860,25 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
return Collections.unmodifiableList(infoMBeans);
}
+ private PackageListeningClassLoader createSchemaLoader() {
+ CoreContainer cc = getCoreContainer();
+ if (cc == null) {
+ //corecontainer not available . can't load from packages
+ return null;
+ }
+ return new PackageListeningClassLoader(cc, this, pkg -> {
+ if (getSolrConfig() == null) return null;
+ return getSolrConfig().maxPackageVersion(pkg);
+ }, () -> {
+ if(getCoreContainer() == null || config == null || coreName == null || coreId==null) return;
+ try (SolrCore c = getCoreContainer().getCore(coreName, coreId)) {
+ if (c != null) {
+ c.fetchLatestSchema();
+ }
+ }
+ });
+ }
+
public static void persistConfLocally(SolrResourceLoader loader, String resourceName, byte[] content) {
// Persist locally
@@ -874,4 +910,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
}
}
+ //This is to verify if this requires to use the schema classloader for classes loaded from packages
+ public static final ThreadLocal<ResourceLoaderAware> CURRENT_AWARE = new ThreadLocal<>();
+
}
diff --git a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
index d44e961..f4ae860 100644
--- a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
+++ b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
@@ -1338,10 +1338,13 @@ public final class ManagedIndexSchema extends IndexSchema {
TokenFilterFactory[] filters = chain.getTokenFilterFactories();
for (TokenFilterFactory next : filters) {
if (next instanceof ResourceLoaderAware) {
+ SolrResourceLoader.CURRENT_AWARE.set((ResourceLoaderAware) next);
try {
((ResourceLoaderAware) next).inform(loader);
} catch (IOException e) {
throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ } finally {
+ SolrResourceLoader.CURRENT_AWARE.remove();
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/schema/SchemaManager.java b/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
index 3731a17..c30f662 100644
--- a/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
+++ b/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
@@ -189,6 +189,7 @@ public class SchemaManager {
mgr.managedIndexSchema = mgr.managedIndexSchema.addFieldTypes(singletonList(fieldType), false);
return true;
} catch (Exception e) {
+ log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@@ -223,6 +224,7 @@ public class SchemaManager {
mgr.managedIndexSchema = mgr.managedIndexSchema.addCopyFields(src, dests, maxChars);
return true;
} catch (Exception e) {
+ log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@@ -240,6 +242,7 @@ public class SchemaManager {
= mgr.managedIndexSchema.addFields(singletonList(field), Collections.emptyMap(), false);
return true;
} catch (Exception e) {
+ log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@@ -257,6 +260,7 @@ public class SchemaManager {
= mgr.managedIndexSchema.addDynamicFields(singletonList(field), Collections.emptyMap(), false);
return true;
} catch (Exception e) {
+ log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@@ -275,6 +279,7 @@ public class SchemaManager {
mgr.managedIndexSchema = mgr.managedIndexSchema.deleteFieldTypes(singleton(name));
return true;
} catch (Exception e) {
+ log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@@ -331,6 +336,7 @@ public class SchemaManager {
mgr.managedIndexSchema = mgr.managedIndexSchema.deleteDynamicFields(singleton(name));
return true;
} catch (Exception e) {
+ log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@@ -346,6 +352,7 @@ public class SchemaManager {
mgr.managedIndexSchema = mgr.managedIndexSchema.replaceFieldType(name, className, op.getDataMap());
return true;
} catch (Exception e) {
+ log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@@ -366,6 +373,7 @@ public class SchemaManager {
mgr.managedIndexSchema = mgr.managedIndexSchema.replaceField(name, ft, op.getValuesExcluding(NAME, TYPE));
return true;
} catch (Exception e) {
+ log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
index f967cbd..e7d9ced 100644
--- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
@@ -19,13 +19,20 @@ package org.apache.solr.pkg;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
import java.util.concurrent.Callable;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
-import org.apache.solr.client.solrj.*;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
@@ -652,10 +659,14 @@ public class TestPackages extends SolrCloudTestCase {
postFileAndWait(cluster, "runtimecode/schema-plugins.jar.bin", FILE1,
"iSRhrogDyt9P1htmSf/krh1kx9oty3TYyWm4GKHQGlb8a+X4tKCe9kKk+3tGs+bU9zq5JBZ5txNXsn96aZem5A==");
+ String FILE2 = "/schemapkg/payload-component.jar";
+ postFileAndWait(cluster, "runtimecode/payload-component.jar.bin", FILE2,
+ "gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q==");
+
Package.AddVersion add = new Package.AddVersion();
add.version = "1.0";
add.pkg = "schemapkg";
- add.files = Arrays.asList(new String[]{FILE1});
+ add.files = Arrays.asList(FILE1,FILE2);
V2Request req = new V2Request.Builder("/cluster/package")
.forceV2(true)
.withMethod(SolrRequest.METHOD.POST)
@@ -694,7 +705,7 @@ public class TestPackages extends SolrCloudTestCase {
String tokenizer =
" 'tokenizer' : { 'class':'schemapkg:my.pkg.MyWhitespaceTokenizerFactory' },\n";
String filters =
- " 'filters' : [{ 'class':'solr.ASCIIFoldingFilterFactory' }]\n";
+ " 'filters' : [{ 'class':'solr.DelimitedPayloadTokenFilterFactory', 'encoder' : 'schemapkg:com.o19s.payloads.Base64Encoder'}]\n";
String suffix = " }\n" +
"}}";
cluster.getSolrClient().request(new SolrRequest(SolrRequest.METHOD.POST, "/schema") {