You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2020/09/19 05:22:36 UTC
[lucene-solr] 08/10: Revert "SOLR-14151: Make schema components
load from packages bug fixes"
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a commit to branch jira/solr-14151-revert-8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 34e72fa54fc40adc00feba1e8259018ada676294
Author: Ishan Chattopadhyaya <is...@apache.org>
AuthorDate: Sat Sep 19 10:51:33 2020 +0530
Revert "SOLR-14151: Make schema components load from packages bug fixes"
This reverts commit bd22f9a5d07ab3e99047add4684dc44b60707297.
---
.../apache/solr/cloud/ZkSolrResourceLoader.java | 6 ----
.../org/apache/solr/core/ConfigSetService.java | 4 +--
.../java/org/apache/solr/core/CoreContainer.java | 21 +++++------
.../src/java/org/apache/solr/core/SolrCore.java | 24 ++++++-------
.../src/java/org/apache/solr/core/SolrCores.java | 7 +---
.../org/apache/solr/core/SolrResourceLoader.java | 8 -----
.../org/apache/solr/handler/SchemaHandler.java | 7 +---
.../org/apache/solr/handler/SolrConfigHandler.java | 2 +-
.../java/org/apache/solr/schema/IndexSchema.java | 42 ++--------------------
.../solr/schema/ManagedIndexSchemaFactory.java | 15 ++------
.../java/org/apache/solr/schema/SchemaManager.java | 4 +--
.../src/test/org/apache/solr/pkg/TestPackages.java | 1 +
12 files changed, 33 insertions(+), 108 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java b/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java
index cde48f1..4eec58f 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java
@@ -27,7 +27,6 @@ import java.util.Properties;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ZkConfigManager;
import org.apache.solr.common.cloud.ZooKeeperException;
-import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.core.SolrResourceNotFoundException;
import org.apache.solr.schema.ZkIndexSchemaReader;
@@ -159,9 +158,4 @@ public class ZkSolrResourceLoader extends SolrResourceLoader {
}
public ZkIndexSchemaReader getZkIndexSchemaReader() { return zkIndexSchemaReader; }
-
- @Override
- public CoreContainer getCoreContainer() {
- return zkController.getCoreContainer();
- }
}
diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
index 2d09bd6..76bc0c1 100644
--- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
+++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
@@ -81,8 +81,8 @@ public abstract class ConfigSetService {
) ? false: true;
SolrConfig solrConfig = createSolrConfig(dcore, coreLoader, trusted);
- coreLoader.solrConfig = solrConfig;
- return new ConfigSet(configSetName(dcore), solrConfig, force -> ConfigSetService.this.createIndexSchema(dcore, solrConfig, false), properties, trusted);
+ IndexSchema indexSchema = createIndexSchema(dcore, solrConfig, false);
+ return new ConfigSet(configSetName(dcore), solrConfig, force -> indexSchema, properties, trusted);
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Could not load conf for core " + dcore.getName() +
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index df9eef9..3e9b424 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -1611,13 +1611,6 @@ public class CoreContainer {
public void reload(String name) {
reload(name, null);
}
- public void reload(String name, UUID coreId, boolean async) {
- if(async) {
- runAsync(() -> reload(name, coreId));
- } else {
- reload(name, coreId);
- }
- }
/**
* Recreates a SolrCore.
* While the new core is loading, requests will continue to be dispatched to
@@ -1632,8 +1625,13 @@ public class CoreContainer {
throw new AlreadyClosedException();
}
SolrCore newCore = null;
- SolrCore core = solrCores.getCoreFromAnyList(name, false, coreId);
+ SolrCore core = solrCores.getCoreFromAnyList(name, false);
if (core != null) {
+ if(coreId != null && core.uniqueId != coreId) {
+ //trying to reload an already unloaded core
+ return;
+ }
+
// The underlying core properties files may have changed, we don't really know. So we have a (perhaps) stale
// CoreDescriptor and we need to reload it from the disk files
CoreDescriptor cd = reloadCoreDescriptor(core.getCoreDescriptor());
@@ -1863,9 +1861,6 @@ public class CoreContainer {
return cfg.getCoreRootDirectory();
}
- public SolrCore getCore(String name) {
- return getCore(name, null);
- }
/**
* Gets a core by name and increase its refcount.
*
@@ -1874,10 +1869,10 @@ public class CoreContainer {
* @throws SolrCoreInitializationException if a SolrCore with this name failed to be initialized
* @see SolrCore#close()
*/
- public SolrCore getCore(String name, UUID id) {
+ public SolrCore getCore(String name) {
// Do this in two phases since we don't want to lock access to the cores over a load.
- SolrCore core = solrCores.getCoreFromAnyList(name, true, id);
+ SolrCore core = solrCores.getCoreFromAnyList(name, true);
// If a core is loaded, we're done just return it.
if (core != null) {
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index ceff28b..f7533cb 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -201,7 +201,6 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
private final CoreDescriptor coreDescriptor;
private final CoreContainer coreContainer;
- private final ConfigSet configSet;
private final SolrConfig solrConfig;
private final SolrResourceLoader resourceLoader;
private volatile IndexSchema schema;
@@ -227,6 +226,8 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
private IndexReaderFactory indexReaderFactory;
private final Codec codec;
private final MemClassLoader memClassLoader;
+ //singleton listener for all packages used in schema
+ private final PackageListeningClassLoader schemaPluginsLoader;
private final List<Runnable> confListeners = new CopyOnWriteArrayList<>();
@@ -271,6 +272,9 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
public PackageListeners getPackageListeners() {
return packageListeners;
}
+ public PackageListeningClassLoader getSchemaPluginsLoader() {
+ return schemaPluginsLoader;
+ }
static int boolean_query_max_clause_count = Integer.MIN_VALUE;
@@ -934,14 +938,16 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
final CountDownLatch latch = new CountDownLatch(1);
try {
this.coreContainer = coreContainer;
- this.configSet = configSet;
this.coreDescriptor = Objects.requireNonNull(coreDescriptor, "coreDescriptor cannot be null");
setName(coreDescriptor.getName());
this.solrConfig = configSet.getSolrConfig();
this.resourceLoader = configSet.getSolrConfig().getResourceLoader();
+ schemaPluginsLoader = new PackageListeningClassLoader(coreContainer, resourceLoader,
+ solrConfig::maxPackageVersion,
+ () -> setLatestSchema(configSet.getIndexSchema()));
+ this.packageListeners.addListener(schemaPluginsLoader);
IndexSchema schema = configSet.getIndexSchema();
- schema.inform(this);
this.configSetProperties = configSet.getProperties();
// Initialize the metrics manager
@@ -2781,12 +2787,6 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
return memClassLoader;
}
- public void fetchLatestSchema() {
- IndexSchema schema = configSet.getIndexSchema(true);
- schema.inform(this);
- setLatestSchema(schema);
- }
-
public interface RawWriter {
default String getContentType() {
return BinaryResponseParser.BINARY_CONTENT_TYPE;
@@ -3099,7 +3099,7 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
SolrZkClient zkClient = cc.getZkController().getZkClient();
int solrConfigversion, overlayVersion, managedSchemaVersion = 0;
SolrConfig cfg = null;
- try (SolrCore solrCore = cc.solrCores.getCoreFromAnyList(coreName, true, coreId)) {
+ try (SolrCore solrCore = cc.solrCores.getCoreFromAnyList(coreName, true)) {
if (solrCore == null || solrCore.isClosed() || solrCore.getCoreContainer().isShutDown()) return;
cfg = solrCore.getSolrConfig();
solrConfigversion = solrCore.getSolrConfig().getOverlay().getZnodeVersion();
@@ -3120,7 +3120,7 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
if (configHandler.getReloadLock().tryLock()) {
try {
- cc.reload(coreName, coreId,true);
+ cc.reload(coreName, coreId);
} catch (SolrCoreState.CoreIsClosedException e) {
/*no problem this core is already closed*/
} finally {
@@ -3133,7 +3133,7 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
return;
}
//some files in conf directory may have other than managedschema, overlay, params
- try (SolrCore solrCore = cc.solrCores.getCoreFromAnyList(coreName, true, coreId)) {
+ try (SolrCore solrCore = cc.solrCores.getCoreFromAnyList(coreName, true)) {
if (solrCore == null || solrCore.isClosed() || cc.isShutDown()) return;
for (Runnable listener : solrCore.confListeners) {
try {
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCores.java b/solr/core/src/java/org/apache/solr/core/SolrCores.java
index d9c1539..d18e3a1 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCores.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCores.java
@@ -34,7 +34,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
-import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
@@ -283,19 +282,15 @@ class SolrCores {
return ret;
}
}
- SolrCore getCoreFromAnyList(String name, boolean incRefCount) {
- return getCoreFromAnyList(name, incRefCount, null);
- }
/* If you don't increment the reference count, someone could close the core before you use it. */
- SolrCore getCoreFromAnyList(String name, boolean incRefCount, UUID coreId) {
+ SolrCore getCoreFromAnyList(String name, boolean incRefCount) {
synchronized (modifyLock) {
SolrCore core = cores.get(name);
if (core == null && getTransientCacheHandler() != null) {
core = getTransientCacheHandler().getCore(name);
}
- if(core != null && coreId != null && coreId != core.uniqueId) return null;
if (core != null && incRefCount) {
core.open();
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
index 810c48d..a073c9a 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
@@ -75,7 +75,6 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
protected URLClassLoader classLoader;
private final Path instanceDir;
private String dataDir; // gone in 9.0
- SolrConfig solrConfig;
@@ -789,13 +788,6 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
}
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, builder.toString() );
}
- public CoreContainer getCoreContainer() {
- return null;
- }
-
- public SolrConfig getSolrConfig() {
- return solrConfig;
- }
@Override
public void close() throws IOException {
diff --git a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
index 70d4fee..fef5773 100644
--- a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
@@ -33,9 +33,7 @@ import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.PluginInfo;
-import org.apache.solr.core.SolrClassLoader;
import org.apache.solr.core.SolrCore;
-import org.apache.solr.pkg.PackageListeningClassLoader;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.SolrQueryResponse;
@@ -231,10 +229,7 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware,
String klas = (String) v;
PluginInfo.ClassName parsedClassName = new PluginInfo.ClassName(klas);
if (parsedClassName.pkg != null) {
- SolrClassLoader cl = req.getCore().getLatestSchema().getSolrClassLoader();
- MapWriter mw = cl instanceof PackageListeningClassLoader?
- ((PackageListeningClassLoader) cl).getPackageVersion(parsedClassName):
- null;
+ MapWriter mw = req.getCore().getSchemaPluginsLoader().getPackageVersion(parsedClassName);
if (mw != null) nl.add("_packageinfo_", mw);
}
}
diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
index 20e8702..7f48b63 100644
--- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
@@ -548,7 +548,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
latestVersion, 30);
} else {
SolrResourceLoader.persistConfLocally(loader, ConfigOverlay.RESOURCE_NAME, overlay.toByteArray());
- req.getCore().getCoreContainer().reload(req.getCore().getName(), req.getCore().uniqueId, true);
+ req.getCore().getCoreContainer().reload(req.getCore().getName(), req.getCore().uniqueId);
log.info("Executed config commands successfully and persisted to File System {}", ops);
}
diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
index 60727d0..3bd565f 100644
--- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
+++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
@@ -39,8 +39,6 @@ import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.TreeSet;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -65,13 +63,10 @@ import org.apache.solr.common.util.Cache;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.SimpleOrderedMap;
-import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrClassLoader;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.core.XmlConfigFile;
-import org.apache.solr.pkg.PackageListeners;
-import org.apache.solr.pkg.PackageListeningClassLoader;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.response.SchemaXmlWriter;
import org.apache.solr.response.SolrQueryResponse;
@@ -100,7 +95,7 @@ import static java.util.Collections.singletonMap;
*
*
*/
-public class IndexSchema implements SolrCoreAware {
+public class IndexSchema {
public static final String COPY_FIELD = "copyField";
public static final String COPY_FIELDS = COPY_FIELD + "s";
public static final String DEFAULT_SCHEMA_FILE = "schema.xml";
@@ -144,8 +139,6 @@ public class IndexSchema implements SolrCoreAware {
protected final SolrResourceLoader loader;
protected final SolrClassLoader solrClassLoader;
protected final Properties substitutableProperties;
- private final Map<String, UUID> cores = new ConcurrentHashMap<>();
- private CoreContainer coreContainer;
protected Map<String,SchemaField> fields = new HashMap<>();
protected Map<String,FieldType> fieldTypes = new HashMap<>();
@@ -198,41 +191,10 @@ public class IndexSchema implements SolrCoreAware {
protected IndexSchema(Version luceneVersion, SolrResourceLoader loader, Properties substitutableProperties) {
this.luceneVersion = Objects.requireNonNull(luceneVersion);
this.loader = loader;
- this.solrClassLoader = createClassLoader(loader);
+ this.solrClassLoader = loader;//loader.getCore() == null? loader: loader.getCore().getSchemaPluginsLoader();
this.substitutableProperties = substitutableProperties;
}
- private SolrClassLoader createClassLoader(SolrResourceLoader loader) {
- if(loader.getCoreContainer() != null && loader.getSolrConfig() != null){
- return new PackageListeningClassLoader(loader.getCoreContainer(), loader,
- pkg -> loader.getSolrConfig().maxPackageVersion(pkg),
- () -> reloadCoreSchema());
- }
- return loader;
- }
-
- private void reloadCoreSchema() {
- if(coreContainer != null) {
- cores.forEach((coreName, id) -> {
- try(SolrCore core = coreContainer.getCore(coreName, id)){
- if(core != null) {
- core.fetchLatestSchema();
- }
- }
- });
- }
- }
-
-
- @Override
- public void inform(SolrCore core) {
- this.cores.put(core.getName(), core.uniqueId);
- this.coreContainer = core.getCoreContainer();
- if (this.loader instanceof PackageListeners.Listener) {
- core.getPackageListeners().addListener((PackageListeners.Listener) this.loader);
- }
- }
-
/**
* The resource loader to be used to load components related to the schema when the schema is loading
* / initialising.
diff --git a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java
index d433692..ef5df42 100644
--- a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java
+++ b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java
@@ -20,7 +20,6 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
-import java.util.UUID;
import org.apache.commons.io.IOUtils;
import org.apache.solr.cloud.ZkController;
@@ -31,7 +30,6 @@ import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkCmdExecutor;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
@@ -61,9 +59,7 @@ public class ManagedIndexSchemaFactory extends IndexSchemaFactory implements Sol
public SolrResourceLoader getResourceLoader() { return loader; }
private String resourceName;
private ManagedIndexSchema schema;
- private CoreContainer coreContainer;
- private String coreName;
- private UUID coreId;
+ private SolrCore core;
private ZkIndexSchemaReader zkIndexSchemaReader;
@@ -403,9 +399,7 @@ public class ManagedIndexSchemaFactory extends IndexSchemaFactory implements Sol
@Override
public void inform(SolrCore core) {
- this.coreName = core.getName();
- this.coreId = core.uniqueId;
- this.coreContainer = core.getCoreContainer();
+ this.core = core;
if (loader instanceof ZkSolrResourceLoader) {
this.zkIndexSchemaReader = new ZkIndexSchemaReader(this, core);
ZkSolrResourceLoader zkLoader = (ZkSolrResourceLoader)loader;
@@ -433,10 +427,7 @@ public class ManagedIndexSchemaFactory extends IndexSchemaFactory implements Sol
public void setSchema(ManagedIndexSchema schema) {
this.schema = schema;
- try ( SolrCore core = coreContainer.getCore(coreName, coreId)) {
- if(core == null) return;
- core.setLatestSchema(schema);
- }
+ core.setLatestSchema(schema);
}
public boolean isMutable() {
diff --git a/solr/core/src/java/org/apache/solr/schema/SchemaManager.java b/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
index a0def5d..d2f5b67 100644
--- a/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
+++ b/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
@@ -132,7 +132,7 @@ public class SchemaManager {
latestVersion = ZkController.persistConfigResourceToZooKeeper
(zkLoader, managedIndexSchema.getSchemaZkVersion(), managedIndexSchema.getResourceName(),
sw.toString().getBytes(StandardCharsets.UTF_8), true);
- req.getCore().getCoreContainer().reload(req.getCore().getName(), req.getCore().uniqueId, true);
+ req.getCore().getCoreContainer().reload(req.getCore().getName(), req.getCore().uniqueId);
break;
} catch (ZkController.ResourceModifiedInZkException e) {
log.info("Schema was modified by another node. Retrying..");
@@ -142,7 +142,7 @@ public class SchemaManager {
//only for non cloud stuff
managedIndexSchema.persistManagedSchema(false);
core.setLatestSchema(managedIndexSchema);
- core.getCoreContainer().reload(core.getName(), core.uniqueId, true);
+ core.getCoreContainer().reload(core.getName(), core.uniqueId);
} catch (SolrException e) {
log.warn(errorMsg);
errors = singletonList(errorMsg + e.getMessage());
diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
index 9542620..46b4071 100644
--- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
@@ -637,6 +637,7 @@ public class TestPackages extends SolrCloudTestCase {
}
@SuppressWarnings("rawtypes")
+ @Ignore("SOLR-14750")
public void testSchemaPlugins() throws Exception {
String COLLECTION_NAME = "testSchemaLoadingColl";
System.setProperty("managed.schema.mutable", "true");