You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2020/09/17 09:11:11 UTC
[lucene-solr] branch master updated: SOLR-14151: fixed the
classloading issue
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new 515608a SOLR-14151: fixed the classloading issue
515608a is described below
commit 515608a087f81c13ad22efba72e48fc64d9d5ae7
Author: noblepaul <no...@gmail.com>
AuthorDate: Thu Sep 17 19:10:04 2020 +1000
SOLR-14151: fixed the classloading issue
---
.../org/apache/solr/core/SolrResourceLoader.java | 111 +++++++++++++++------
.../org/apache/solr/schema/ManagedIndexSchema.java | 5 +-
.../java/org/apache/solr/schema/SchemaManager.java | 10 +-
.../runtimecode/payload-component.jar.bin | Bin 0 -> 17815 bytes
.../src/test/org/apache/solr/pkg/TestPackages.java | 21 +++-
5 files changed, 111 insertions(+), 36 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
index 7a10f89..532f1dd 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
@@ -16,8 +16,12 @@
*/
package org.apache.solr.core;
-import com.google.common.annotations.VisibleForTesting;
-import java.io.*;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.Constructor;
import java.net.MalformedURLException;
@@ -30,14 +34,26 @@ import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
import org.apache.lucene.analysis.WordlistLoader;
-import org.apache.lucene.analysis.util.*;
+import org.apache.lucene.analysis.util.CharFilterFactory;
+import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.lucene.analysis.util.ResourceLoaderAware;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
@@ -83,27 +99,6 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
private CoreContainer coreContainer;
private PackageListeningClassLoader schemaLoader ;
- private PackageListeningClassLoader createSchemaLoader() {
- CoreContainer cc = getCoreContainer();
- if (cc == null) {
- //corecontainer not available . can't load from packages
- return null;
- }
- return new PackageListeningClassLoader(cc, this, pkg -> {
- if (getSolrConfig() == null) return null;
- return getSolrConfig().maxPackageVersion(pkg);
- }, () -> {
- if(getCoreContainer() == null || config == null || coreName == null || coreId==null) return;
- try (SolrCore c = getCoreContainer().getCore(coreName, coreId)) {
- if (c != null) {
- c.fetchLatestSchema();
- }
- }
- });
- }
-
-
-
private final List<SolrCoreAware> waitingForCore = Collections.synchronizedList(new ArrayList<SolrCoreAware>());
private final List<SolrInfoBean> infoMBeans = Collections.synchronizedList(new ArrayList<SolrInfoBean>());
private final List<ResourceLoaderAware> waitingForResources = Collections.synchronizedList(new ArrayList<ResourceLoaderAware>());
@@ -476,8 +471,9 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
}
}
}
-
Class<? extends T> clazz = null;
+ clazz = getPackageClass(cname, expectedType);
+ if(clazz != null) return clazz;
try {
// first try legacy analysis patterns, now replaced by Lucene's Analysis package:
final Matcher m = legacyAnalysisPattern.matcher(cname);
@@ -540,6 +536,24 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
}
}
+ private <T> Class<? extends T> getPackageClass(String cname, Class<T> expectedType) {
+ PluginInfo.ClassName cName = PluginInfo.parseClassName(cname);
+ if (cName.pkg == null) return null;
+ ResourceLoaderAware aware = CURRENT_AWARE.get();
+ if (aware != null) {
+ //this is invoked from a component
+ //let's check if it's a schema component
+ @SuppressWarnings("rawtypes")
+ Class type = assertAwareCompatibility(ResourceLoaderAware.class, aware);
+ if (schemaResourceLoaderComponents.contains(type)) {
+ //this is a schema component
+ //lets use schema classloader
+ return getSchemaLoader().findClass(cname, expectedType);
+ }
+ }
+ return null;
+ }
+
static final String[] empty = new String[0];
@Override
@@ -679,7 +693,13 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
}
for (ResourceLoaderAware aware : arr) {
- aware.inform(loader);
+ CURRENT_AWARE.set(aware);
+ try{
+ aware.inform(loader);
+ } finally {
+ CURRENT_AWARE.remove();
+ }
+
}
}
}
@@ -749,6 +769,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
ResourceLoaderAware.class, new Class<?>[]{
// DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY
// VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
+ // evaluate if this must go into schemaResourceLoaderComponents
CharFilterFactory.class,
TokenFilterFactory.class,
TokenizerFactory.class,
@@ -758,11 +779,21 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
);
}
+ /**If these components are trying to load classes, use schema classloader
+ *
+ */
+ @SuppressWarnings("rawtypes")
+ private static final ImmutableSet<Class> schemaResourceLoaderComponents = ImmutableSet.of(
+ CharFilterFactory.class,
+ TokenFilterFactory.class,
+ TokenizerFactory.class,
+ FieldType.class);
+
/**
* Utility function to throw an exception if the class is invalid
*/
@SuppressWarnings({"rawtypes"})
- public static void assertAwareCompatibility(Class aware, Object obj) {
+ public static Class assertAwareCompatibility(Class aware, Object obj) {
Class[] valid = awareCompatibility.get(aware);
if (valid == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
@@ -770,7 +801,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
}
for (Class v : valid) {
if (v.isInstance(obj)) {
- return;
+ return v;
}
}
StringBuilder builder = new StringBuilder();
@@ -800,6 +831,25 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
return Collections.unmodifiableList(infoMBeans);
}
+ private PackageListeningClassLoader createSchemaLoader() {
+ CoreContainer cc = getCoreContainer();
+ if (cc == null) {
+ //corecontainer not available . can't load from packages
+ return null;
+ }
+ return new PackageListeningClassLoader(cc, this, pkg -> {
+ if (getSolrConfig() == null) return null;
+ return getSolrConfig().maxPackageVersion(pkg);
+ }, () -> {
+ if(getCoreContainer() == null || config == null || coreName == null || coreId==null) return;
+ try (SolrCore c = getCoreContainer().getCore(coreName, coreId)) {
+ if (c != null) {
+ c.fetchLatestSchema();
+ }
+ }
+ });
+ }
+
public static void persistConfLocally(SolrResourceLoader loader, String resourceName, byte[] content) {
// Persist locally
@@ -831,4 +881,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
}
}
+ //This is to verify if this requires to use the schema classloader for classes loaded from packages
+ public static final ThreadLocal<ResourceLoaderAware> CURRENT_AWARE = new ThreadLocal<>();
+
}
diff --git a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
index 9fd5243..f8c0c4f 100644
--- a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
+++ b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
@@ -66,10 +66,10 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SolrNamedThreadFactory;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.rest.schema.FieldTypeXmlAdapter;
-import org.apache.solr.common.util.SolrNamedThreadFactory;
import org.apache.solr.util.FileUtils;
import org.apache.solr.util.RTimer;
import org.apache.zookeeper.CreateMode;
@@ -1343,10 +1343,13 @@ public final class ManagedIndexSchema extends IndexSchema {
TokenFilterFactory[] filters = chain.getTokenFilterFactories();
for (TokenFilterFactory next : filters) {
if (next instanceof ResourceLoaderAware) {
+ SolrResourceLoader.CURRENT_AWARE.set((ResourceLoaderAware) next);
try {
((ResourceLoaderAware) next).inform(loader);
} catch (IOException e) {
throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ } finally {
+ SolrResourceLoader.CURRENT_AWARE.remove();
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/schema/SchemaManager.java b/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
index 3731a17..c04c9c3 100644
--- a/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
+++ b/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
@@ -31,13 +31,13 @@ import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.rest.BaseSolrResource;
-import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
@@ -189,6 +189,7 @@ public class SchemaManager {
mgr.managedIndexSchema = mgr.managedIndexSchema.addFieldTypes(singletonList(fieldType), false);
return true;
} catch (Exception e) {
+ log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@@ -223,6 +224,7 @@ public class SchemaManager {
mgr.managedIndexSchema = mgr.managedIndexSchema.addCopyFields(src, dests, maxChars);
return true;
} catch (Exception e) {
+ log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@@ -240,6 +242,7 @@ public class SchemaManager {
= mgr.managedIndexSchema.addFields(singletonList(field), Collections.emptyMap(), false);
return true;
} catch (Exception e) {
+ log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@@ -257,6 +260,7 @@ public class SchemaManager {
= mgr.managedIndexSchema.addDynamicFields(singletonList(field), Collections.emptyMap(), false);
return true;
} catch (Exception e) {
+ log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@@ -275,6 +279,7 @@ public class SchemaManager {
mgr.managedIndexSchema = mgr.managedIndexSchema.deleteFieldTypes(singleton(name));
return true;
} catch (Exception e) {
+ log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@@ -331,6 +336,7 @@ public class SchemaManager {
mgr.managedIndexSchema = mgr.managedIndexSchema.deleteDynamicFields(singleton(name));
return true;
} catch (Exception e) {
+ log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@@ -346,6 +352,7 @@ public class SchemaManager {
mgr.managedIndexSchema = mgr.managedIndexSchema.replaceFieldType(name, className, op.getDataMap());
return true;
} catch (Exception e) {
+ log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@@ -366,6 +373,7 @@ public class SchemaManager {
mgr.managedIndexSchema = mgr.managedIndexSchema.replaceField(name, ft, op.getValuesExcluding(NAME, TYPE));
return true;
} catch (Exception e) {
+ log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
diff --git a/solr/core/src/test-files/runtimecode/payload-component.jar.bin b/solr/core/src/test-files/runtimecode/payload-component.jar.bin
new file mode 100644
index 0000000..47a4305
Binary files /dev/null and b/solr/core/src/test-files/runtimecode/payload-component.jar.bin differ
diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
index 8d8583c..9473d41 100644
--- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
@@ -19,13 +19,20 @@ package org.apache.solr.pkg;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
import java.util.concurrent.Callable;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
-import org.apache.solr.client.solrj.*;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
@@ -69,9 +76,9 @@ import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH;
import static org.apache.solr.common.params.CommonParams.JAVABIN;
import static org.apache.solr.common.params.CommonParams.WT;
import static org.apache.solr.core.TestSolrConfigHandler.getFileContent;
+import static org.apache.solr.filestore.TestDistribPackageStore.checkAllNodesForFile;
import static org.apache.solr.filestore.TestDistribPackageStore.readFile;
import static org.apache.solr.filestore.TestDistribPackageStore.uploadKey;
-import static org.apache.solr.filestore.TestDistribPackageStore.checkAllNodesForFile;
import static org.hamcrest.CoreMatchers.containsString;
@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG")
@@ -655,10 +662,14 @@ public class TestPackages extends SolrCloudTestCase {
postFileAndWait(cluster, "runtimecode/schema-plugins.jar.bin", FILE1,
"iSRhrogDyt9P1htmSf/krh1kx9oty3TYyWm4GKHQGlb8a+X4tKCe9kKk+3tGs+bU9zq5JBZ5txNXsn96aZem5A==");
+ String FILE2 = "/schemapkg/payload-component.jar";
+ postFileAndWait(cluster, "runtimecode/payload-component.jar.bin", FILE2,
+ "gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q==");
+
Package.AddVersion add = new Package.AddVersion();
add.version = "1.0";
add.pkg = "schemapkg";
- add.files = Arrays.asList(new String[]{FILE1});
+ add.files = Arrays.asList(FILE1,FILE2);
V2Request req = new V2Request.Builder("/cluster/package")
.forceV2(true)
.withMethod(SolrRequest.METHOD.POST)
@@ -697,7 +708,7 @@ public class TestPackages extends SolrCloudTestCase {
String tokenizer =
" 'tokenizer' : { 'class':'schemapkg:my.pkg.MyWhitespaceTokenizerFactory' },\n";
String filters =
- " 'filters' : [{ 'class':'solr.ASCIIFoldingFilterFactory' }]\n";
+ " 'filters' : [{ 'class':'solr.DelimitedPayloadTokenFilterFactory', 'encoder' : 'schemapkg:com.o19s.payloads.Base64Encoder'}]\n";
String suffix = " }\n" +
"}}";
cluster.getSolrClient().request(new SolrRequest(SolrRequest.METHOD.POST, "/schema") {