You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2020/09/17 01:21:00 UTC
[lucene-solr] branch master updated: Revert "Revert "SOLR-14151:
Bug fixes (#1815)""
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new cbb1659 Revert "Revert "SOLR-14151: Bug fixes (#1815)""
cbb1659 is described below
commit cbb1659640cd51be8b403eda8399c527af1c848e
Author: noblepaul <no...@gmail.com>
AuthorDate: Thu Sep 17 11:19:21 2020 +1000
Revert "Revert "SOLR-14151: Bug fixes (#1815)""
This reverts commit 27a14fe48139019a4c09ba072751e093fc5cb5f1.
Undoing accidental commit
---
.../apache/solr/cloud/ZkSolrResourceLoader.java | 6 +++
.../org/apache/solr/core/ConfigSetService.java | 3 +-
.../java/org/apache/solr/core/CoreContainer.java | 21 ++++++----
.../src/java/org/apache/solr/core/SolrCore.java | 21 ++++------
.../src/java/org/apache/solr/core/SolrCores.java | 7 +++-
.../org/apache/solr/core/SolrResourceLoader.java | 48 +++++++++++++++++++++-
.../java/org/apache/solr/handler/IndexFetcher.java | 2 +-
.../org/apache/solr/handler/SchemaHandler.java | 6 ++-
.../org/apache/solr/handler/SolrConfigHandler.java | 2 +-
.../java/org/apache/solr/schema/IndexSchema.java | 2 +-
.../java/org/apache/solr/schema/SchemaManager.java | 4 +-
.../src/test/org/apache/solr/pkg/TestPackages.java | 2 -
12 files changed, 92 insertions(+), 32 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java b/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java
index 4d9d910..0e17b5c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java
@@ -26,6 +26,7 @@ import java.nio.file.Path;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ZkConfigManager;
import org.apache.solr.common.cloud.ZooKeeperException;
+import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.core.SolrResourceNotFoundException;
import org.apache.solr.schema.ZkIndexSchemaReader;
@@ -157,4 +158,9 @@ public class ZkSolrResourceLoader extends SolrResourceLoader {
}
public ZkIndexSchemaReader getZkIndexSchemaReader() { return zkIndexSchemaReader; }
+
+ @Override
+ public CoreContainer getCoreContainer() {
+ return zkController.getCoreContainer();
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
index c77d0b4..1d45bc5 100644
--- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
+++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
@@ -81,8 +81,7 @@ public abstract class ConfigSetService {
) ? false: true;
SolrConfig solrConfig = createSolrConfig(dcore, coreLoader, trusted);
- IndexSchema indexSchema = createIndexSchema(dcore, solrConfig, false);
- return new ConfigSet(configSetName(dcore), solrConfig, force -> indexSchema, properties, trusted);
+ return new ConfigSet(configSetName(dcore), solrConfig, force -> createIndexSchema(dcore, solrConfig, force), properties, trusted);
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Could not load conf for core " + dcore.getName() +
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index e8fed0c..9e0a890 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -1586,6 +1586,13 @@ public class CoreContainer {
public void reload(String name) {
reload(name, null);
}
+ public void reload(String name, UUID coreId, boolean async) {
+ if(async) {
+ runAsync(() -> reload(name, coreId));
+ } else {
+ reload(name, coreId);
+ }
+ }
/**
* Recreates a SolrCore.
* While the new core is loading, requests will continue to be dispatched to
@@ -1600,13 +1607,8 @@ public class CoreContainer {
throw new AlreadyClosedException();
}
SolrCore newCore = null;
- SolrCore core = solrCores.getCoreFromAnyList(name, false);
+ SolrCore core = solrCores.getCoreFromAnyList(name, false, coreId);
if (core != null) {
- if(coreId != null && core.uniqueId != coreId) {
- //trying to reload an already unloaded core
- return;
- }
-
// The underlying core properties files may have changed, we don't really know. So we have a (perhaps) stale
// CoreDescriptor and we need to reload it from the disk files
CoreDescriptor cd = reloadCoreDescriptor(core.getCoreDescriptor());
@@ -1836,6 +1838,9 @@ public class CoreContainer {
return cfg.getCoreRootDirectory();
}
+ public SolrCore getCore(String name) {
+ return getCore(name, null);
+ }
/**
* Gets a core by name and increase its refcount.
*
@@ -1844,10 +1849,10 @@ public class CoreContainer {
* @throws SolrCoreInitializationException if a SolrCore with this name failed to be initialized
* @see SolrCore#close()
*/
- public SolrCore getCore(String name) {
+ public SolrCore getCore(String name, UUID id) {
// Do this in two phases since we don't want to lock access to the cores over a load.
- SolrCore core = solrCores.getCoreFromAnyList(name, true);
+ SolrCore core = solrCores.getCoreFromAnyList(name, true, id);
// If a core is loaded, we're done just return it.
if (core != null) {
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index b44866e..f3e0408 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -225,8 +225,8 @@ public final class SolrCore implements SolrInfoBean, Closeable {
private final RecoveryStrategy.Builder recoveryStrategyBuilder;
private IndexReaderFactory indexReaderFactory;
private final Codec codec;
+ private final ConfigSet configSet;
//singleton listener for all packages used in schema
- private final PackageListeningClassLoader schemaPluginsLoader;
private final CircuitBreakerManager circuitBreakerManager;
@@ -273,9 +273,6 @@ public final class SolrCore implements SolrInfoBean, Closeable {
public PackageListeners getPackageListeners() {
return packageListeners;
}
- public PackageListeningClassLoader getSchemaPluginsLoader() {
- return schemaPluginsLoader;
- }
static int boolean_query_max_clause_count = Integer.MIN_VALUE;
@@ -939,15 +936,12 @@ public final class SolrCore implements SolrInfoBean, Closeable {
final CountDownLatch latch = new CountDownLatch(1);
try {
this.coreContainer = coreContainer;
+ this.configSet = configSet;
this.coreDescriptor = Objects.requireNonNull(coreDescriptor, "coreDescriptor cannot be null");
setName(coreDescriptor.getName());
this.solrConfig = configSet.getSolrConfig();
this.resourceLoader = configSet.getSolrConfig().getResourceLoader();
- schemaPluginsLoader = new PackageListeningClassLoader(coreContainer, resourceLoader,
- solrConfig::maxPackageVersion,
- () -> setLatestSchema(configSet.getIndexSchema()));
- this.packageListeners.addListener(schemaPluginsLoader);
IndexSchema schema = configSet.getIndexSchema();
this.configSetProperties = configSet.getProperties();
@@ -2768,7 +2762,10 @@ public final class SolrCore implements SolrInfoBean, Closeable {
};
}
-
+ public void fetchLatestSchema() {
+ IndexSchema schema = configSet.getIndexSchema(true);
+ setLatestSchema(schema);
+ }
public interface RawWriter {
default String getContentType() {
return BinaryResponseParser.BINARY_CONTENT_TYPE;
@@ -3076,7 +3073,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
SolrZkClient zkClient = cc.getZkController().getZkClient();
int solrConfigversion, overlayVersion, managedSchemaVersion = 0;
SolrConfig cfg = null;
- try (SolrCore solrCore = cc.solrCores.getCoreFromAnyList(coreName, true)) {
+ try (SolrCore solrCore = cc.solrCores.getCoreFromAnyList(coreName, true, coreId)) {
if (solrCore == null || solrCore.isClosed() || solrCore.getCoreContainer().isShutDown()) return;
cfg = solrCore.getSolrConfig();
solrConfigversion = solrCore.getSolrConfig().getOverlay().getZnodeVersion();
@@ -3097,7 +3094,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
if (configHandler.getReloadLock().tryLock()) {
try {
- cc.reload(coreName, coreId);
+ cc.reload(coreName, coreId, false);
} catch (SolrCoreState.CoreIsClosedException e) {
/*no problem this core is already closed*/
} finally {
@@ -3110,7 +3107,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
return;
}
//some files in conf directory may have other than managedschema, overlay, params
- try (SolrCore solrCore = cc.solrCores.getCoreFromAnyList(coreName, true)) {
+ try (SolrCore solrCore = cc.solrCores.getCoreFromAnyList(coreName, true, coreId)) {
if (solrCore == null || solrCore.isClosed() || cc.isShutDown()) return;
for (Runnable listener : solrCore.confListeners) {
try {
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCores.java b/solr/core/src/java/org/apache/solr/core/SolrCores.java
index 205b70f..574bba0 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCores.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCores.java
@@ -34,6 +34,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
+import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
@@ -282,15 +283,19 @@ class SolrCores {
return ret;
}
}
+ SolrCore getCoreFromAnyList(String name, boolean incRefCount) {
+ return getCoreFromAnyList(name, incRefCount, null);
+ }
/* If you don't increment the reference count, someone could close the core before you use it. */
- SolrCore getCoreFromAnyList(String name, boolean incRefCount) {
+ SolrCore getCoreFromAnyList(String name, boolean incRefCount, UUID coreId) {
synchronized (modifyLock) {
SolrCore core = cores.get(name);
if (core == null && getTransientCacheHandler() != null) {
core = getTransientCacheHandler().getCore(name);
}
+ if(core != null && coreId != null && coreId != core.uniqueId) return null;
if (core != null && incRefCount) {
core.open();
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
index c5ad5fc..7a10f89 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
@@ -46,6 +46,7 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.SolrClassLoader;
import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.handler.component.ShardHandlerFactory;
+import org.apache.solr.pkg.PackageListeningClassLoader;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.rest.RestManager;
@@ -61,7 +62,7 @@ import org.slf4j.LoggerFactory;
/**
* @since solr 1.3
*/
-public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassLoader {
+public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassLoader, SolrCoreAware {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final String base = "org.apache.solr";
@@ -76,6 +77,30 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
private String name = "";
protected URLClassLoader classLoader;
private final Path instanceDir;
+ private String coreName;
+ private UUID coreId;
+ private SolrConfig config;
+ private CoreContainer coreContainer;
+ private PackageListeningClassLoader schemaLoader ;
+
+ private PackageListeningClassLoader createSchemaLoader() {
+ CoreContainer cc = getCoreContainer();
+ if (cc == null) {
+ //corecontainer not available . can't load from packages
+ return null;
+ }
+ return new PackageListeningClassLoader(cc, this, pkg -> {
+ if (getSolrConfig() == null) return null;
+ return getSolrConfig().maxPackageVersion(pkg);
+ }, () -> {
+ if(getCoreContainer() == null || config == null || coreName == null || coreId==null) return;
+ try (SolrCore c = getCoreContainer().getCore(coreName, coreId)) {
+ if (c != null) {
+ c.fetchLatestSchema();
+ }
+ }
+ });
+ }
@@ -101,6 +126,13 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
return managedResourceRegistry;
}
+ public SolrClassLoader getSchemaLoader() {
+ if (schemaLoader == null) {
+ schemaLoader = createSchemaLoader();
+ }
+ return schemaLoader;
+ }
+
public SolrResourceLoader() {
this(SolrPaths.locateSolrHome(), null);
}
@@ -607,6 +639,12 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
* Tell all {@link SolrCoreAware} instances about the SolrCore
*/
public void inform(SolrCore core) {
+ this.coreName = core.getName();
+ this.config = core.getSolrConfig();
+ this.coreId = core.uniqueId;
+ this.coreContainer = core.getCoreContainer();
+ if(getSchemaLoader() != null) core.getPackageListeners().addListener(schemaLoader);
+
// make a copy to avoid potential deadlock of a callback calling newInstance and trying to
// add something to waitingForCore.
SolrCoreAware[] arr;
@@ -745,6 +783,14 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, builder.toString());
}
+ public CoreContainer getCoreContainer(){
+ return coreContainer;
+ }
+
+ public SolrConfig getSolrConfig() {
+ return config;
+
+ }
@Override
public void close() throws IOException {
IOUtils.close(classLoader);
diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
index 2bd2045..c8a0512 100644
--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
@@ -909,7 +909,7 @@ public class IndexFetcher {
final CountDownLatch latch = new CountDownLatch(1);
new Thread(() -> {
try {
- solrCore.getCoreContainer().reload(solrCore.getName());
+ solrCore.getCoreContainer().reload(solrCore.getName(), solrCore.uniqueId);
} catch (Exception e) {
log.error("Could not reload core ", e);
} finally {
diff --git a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
index 0ab79e5..606bab5 100644
--- a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
@@ -26,6 +26,7 @@ import org.apache.solr.api.ApiBag;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.SolrClassLoader;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
@@ -34,6 +35,7 @@ import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.pkg.PackageListeningClassLoader;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.SolrQueryResponse;
@@ -225,7 +227,9 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware,
String klas = (String) v;
PluginInfo.ClassName parsedClassName = new PluginInfo.ClassName(klas);
if (parsedClassName.pkg != null) {
- MapWriter mw = req.getCore().getSchemaPluginsLoader().getPackageVersion(parsedClassName);
+ SolrClassLoader solrClassLoader = req.getCore().getLatestSchema().getSolrClassLoader();
+ MapWriter mw = solrClassLoader instanceof PackageListeningClassLoader ?
+ ((PackageListeningClassLoader) solrClassLoader).getPackageVersion(parsedClassName) : null;
if (mw != null) nl.add("_packageinfo_", mw);
}
}
diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
index fc7ca3b..4a1aadb 100644
--- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
@@ -547,7 +547,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
latestVersion, 30);
} else {
SolrResourceLoader.persistConfLocally(loader, ConfigOverlay.RESOURCE_NAME, overlay.toByteArray());
- req.getCore().getCoreContainer().reload(req.getCore().getName(), req.getCore().uniqueId);
+ req.getCore().getCoreContainer().reload(req.getCore().getName(), req.getCore().uniqueId, false);
log.info("Executed config commands successfully and persisted to File System {}", ops);
}
diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
index 3e2237f..bd4ae7e 100644
--- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
+++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
@@ -190,7 +190,7 @@ public class IndexSchema {
protected IndexSchema(Version luceneVersion, SolrResourceLoader loader, Properties substitutableProperties) {
this.luceneVersion = Objects.requireNonNull(luceneVersion);
this.loader = loader;
- this.solrClassLoader = loader;//loader.getCore() == null? loader: loader.getCore().getSchemaPluginsLoader();
+ this.solrClassLoader = loader.getSchemaLoader() == null ? loader : loader.getSchemaLoader();
this.substitutableProperties = substitutableProperties;
}
diff --git a/solr/core/src/java/org/apache/solr/schema/SchemaManager.java b/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
index d2f5b67..3731a17 100644
--- a/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
+++ b/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
@@ -132,7 +132,7 @@ public class SchemaManager {
latestVersion = ZkController.persistConfigResourceToZooKeeper
(zkLoader, managedIndexSchema.getSchemaZkVersion(), managedIndexSchema.getResourceName(),
sw.toString().getBytes(StandardCharsets.UTF_8), true);
- req.getCore().getCoreContainer().reload(req.getCore().getName(), req.getCore().uniqueId);
+ req.getCore().getCoreContainer().reload(req.getCore().getName(), req.getCore().uniqueId, false);
break;
} catch (ZkController.ResourceModifiedInZkException e) {
log.info("Schema was modified by another node. Retrying..");
@@ -142,7 +142,7 @@ public class SchemaManager {
//only for non cloud stuff
managedIndexSchema.persistManagedSchema(false);
core.setLatestSchema(managedIndexSchema);
- core.getCoreContainer().reload(core.getName(), core.uniqueId);
+ core.getCoreContainer().reload(core.getName(), core.uniqueId, false);
} catch (SolrException e) {
log.warn(errorMsg);
errors = singletonList(errorMsg + e.getMessage());
diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
index c4bf29e..8d8583c 100644
--- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
@@ -63,7 +63,6 @@ import org.apache.solr.util.plugin.SolrCoreAware;
import org.apache.zookeeper.data.Stat;
import org.junit.After;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH;
@@ -640,7 +639,6 @@ public class TestPackages extends SolrCloudTestCase {
}
@SuppressWarnings("rawtypes")
- @Ignore("SOLR-14750")
public void testSchemaPlugins() throws Exception {
String COLLECTION_NAME = "testSchemaLoadingColl";
System.setProperty("managed.schema.mutable", "true");