You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2020/09/12 07:02:31 UTC
[lucene-solr] 01/01: SOLR-14151: Another take
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch jira/solr14151-5
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit a2779d2d61169d27d5b0504bae48e90e67f81d7f
Author: noblepaul <no...@gmail.com>
AuthorDate: Sat Sep 12 16:58:02 2020 +1000
SOLR-14151: Another take
---
.../java/org/apache/solr/core/CoreContainer.java | 8 +--
.../src/java/org/apache/solr/core/SolrCore.java | 51 +++++++++++++---
.../org/apache/solr/core/SolrResourceLoader.java | 69 +++++++++++-----------
.../java/org/apache/solr/handler/IndexFetcher.java | 4 +-
.../org/apache/solr/handler/SolrConfigHandler.java | 4 +-
.../org/apache/solr/handler/StreamHandler.java | 2 +-
.../org/apache/solr/pkg/PackagePluginHolder.java | 26 ++++----
.../java/org/apache/solr/schema/SchemaManager.java | 6 +-
8 files changed, 101 insertions(+), 69 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 6480fa8..98dd7ca 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -1582,13 +1582,7 @@ public class CoreContainer {
public void reload(String name) {
reload(name, null);
}
- public void reload(String name, UUID coreId, boolean async) {
- if(async) {
- runAsync(() -> reload(name, coreId));
- } else {
- reload(name, coreId);
- }
- }
+
/**
* Recreates a SolrCore.
* While the new core is loading, requests will continue to be dispatched to
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index f3e0408..be0bf0f 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -57,6 +57,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Consumer;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Timer;
@@ -111,7 +112,8 @@ import org.apache.solr.logging.MDCLoggingContext;
import org.apache.solr.metrics.SolrCoreMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
-import org.apache.solr.pkg.*;
+import org.apache.solr.pkg.PackageListeners;
+import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.BinaryResponseWriter;
@@ -247,6 +249,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
public volatile boolean searchEnabled = true;
public volatile boolean indexEnabled = true;
public volatile boolean readOnly = false;
+ private final CoreProvider coreProvider;
private PackageListeners packageListeners = new PackageListeners(this);
@@ -299,6 +302,10 @@ public final class SolrCore implements SolrInfoBean, Closeable {
return latest.getLoader();
}
+ public CoreProvider getCoreProvider(){
+ return coreProvider;
+ }
+
/**
* Gets the configuration resource name used by this core instance.
*
@@ -939,7 +946,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
this.configSet = configSet;
this.coreDescriptor = Objects.requireNonNull(coreDescriptor, "coreDescriptor cannot be null");
setName(coreDescriptor.getName());
-
+ this.coreProvider = new CoreProvider(coreContainer, uniqueId, name );
this.solrConfig = configSet.getSolrConfig();
this.resourceLoader = configSet.getSolrConfig().getResourceLoader();
IndexSchema schema = configSet.getIndexSchema();
@@ -3058,7 +3065,6 @@ public final class SolrCore implements SolrInfoBean, Closeable {
public static Runnable getConfListener(SolrCore core, ZkSolrResourceLoader zkSolrResourceLoader) {
final String coreName = core.getName();
- final UUID coreId = core.uniqueId;
final CoreContainer cc = core.getCoreContainer();
final String overlayPath = zkSolrResourceLoader.getConfigSetZkPath() + "/" + ConfigOverlay.RESOURCE_NAME;
final String solrConfigPath = zkSolrResourceLoader.getConfigSetZkPath() + "/" + core.getSolrConfig().getName();
@@ -3073,7 +3079,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
SolrZkClient zkClient = cc.getZkController().getZkClient();
int solrConfigversion, overlayVersion, managedSchemaVersion = 0;
SolrConfig cfg = null;
- try (SolrCore solrCore = cc.solrCores.getCoreFromAnyList(coreName, true, coreId)) {
+ try (SolrCore solrCore = cc.solrCores.getCoreFromAnyList(coreName, true)) {
if (solrCore == null || solrCore.isClosed() || solrCore.getCoreContainer().isShutDown()) return;
cfg = solrCore.getSolrConfig();
solrConfigversion = solrCore.getSolrConfig().getOverlay().getZnodeVersion();
@@ -3086,15 +3092,17 @@ public final class SolrCore implements SolrInfoBean, Closeable {
if (cfg != null) {
cfg.refreshRequestParams();
}
+ if(checkStale(zkClient, managedSchmaResourcePath, managedSchemaVersion)) {
+ core.fetchLatestSchema();
+ }
if (checkStale(zkClient, overlayPath, solrConfigversion) ||
- checkStale(zkClient, solrConfigPath, overlayVersion) ||
- checkStale(zkClient, managedSchmaResourcePath, managedSchemaVersion)) {
+ checkStale(zkClient, solrConfigPath, overlayVersion) ) {
log.info("core reload {}", coreName);
SolrConfigHandler configHandler = ((SolrConfigHandler) core.getRequestHandler("/config"));
if (configHandler.getReloadLock().tryLock()) {
try {
- cc.reload(coreName, coreId, false);
+ cc.reload(coreName);
} catch (SolrCoreState.CoreIsClosedException e) {
/*no problem this core is already closed*/
} finally {
@@ -3107,7 +3115,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
return;
}
//some files in conf directory may have other than managedschema, overlay, params
- try (SolrCore solrCore = cc.solrCores.getCoreFromAnyList(coreName, true, coreId)) {
+ try (SolrCore solrCore = cc.solrCores.getCoreFromAnyList(coreName, true)) {
if (solrCore == null || solrCore.isClosed() || cc.isShutDown()) return;
for (Runnable listener : solrCore.confListeners) {
try {
@@ -3245,4 +3253,31 @@ public final class SolrCore implements SolrInfoBean, Closeable {
public void runAsync(Runnable r) {
coreAsyncTaskExecutor.submit(r);
}
+
+ /**Use this to get the core just in time instead of holding on to a core reference
+ * Cores may get reloaded and that core may become stale
+ *
+ */
+ public static class CoreProvider {
+ public final CoreContainer coreContainer;
+ public final UUID coreId;
+ public final String coreName;
+
+ public CoreProvider(CoreContainer coreContainer, UUID coreId, String coreName) {
+ this.coreContainer = coreContainer;
+ this.coreId = coreId;
+ this.coreName = coreName;
+ }
+
+ /**fetch the core just in time
+ * If the core is already closed, the consumer do not get a callback
+ */
+ public void withCore(Consumer<SolrCore> r) {
+ try(SolrCore core = coreContainer.getCore(coreName, coreId)) {
+ if (core == null) return;
+ r.accept(core);
+ }
+
+ }
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
index 7a10f89..4300bac 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
@@ -16,8 +16,12 @@
*/
package org.apache.solr.core;
-import com.google.common.annotations.VisibleForTesting;
-import java.io.*;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.Constructor;
import java.net.MalformedURLException;
@@ -30,14 +34,24 @@ import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.lucene.analysis.WordlistLoader;
-import org.apache.lucene.analysis.util.*;
+import org.apache.lucene.analysis.util.CharFilterFactory;
+import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.lucene.analysis.util.ResourceLoaderAware;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
@@ -77,33 +91,10 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
private String name = "";
protected URLClassLoader classLoader;
private final Path instanceDir;
- private String coreName;
- private UUID coreId;
+ private SolrCore.CoreProvider coreProvider;
private SolrConfig config;
- private CoreContainer coreContainer;
private PackageListeningClassLoader schemaLoader ;
- private PackageListeningClassLoader createSchemaLoader() {
- CoreContainer cc = getCoreContainer();
- if (cc == null) {
- //corecontainer not available . can't load from packages
- return null;
- }
- return new PackageListeningClassLoader(cc, this, pkg -> {
- if (getSolrConfig() == null) return null;
- return getSolrConfig().maxPackageVersion(pkg);
- }, () -> {
- if(getCoreContainer() == null || config == null || coreName == null || coreId==null) return;
- try (SolrCore c = getCoreContainer().getCore(coreName, coreId)) {
- if (c != null) {
- c.fetchLatestSchema();
- }
- }
- });
- }
-
-
-
private final List<SolrCoreAware> waitingForCore = Collections.synchronizedList(new ArrayList<SolrCoreAware>());
private final List<SolrInfoBean> infoMBeans = Collections.synchronizedList(new ArrayList<SolrInfoBean>());
private final List<ResourceLoaderAware> waitingForResources = Collections.synchronizedList(new ArrayList<ResourceLoaderAware>());
@@ -130,7 +121,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
if (schemaLoader == null) {
schemaLoader = createSchemaLoader();
}
- return schemaLoader;
+ return schemaLoader == null? this: schemaLoader;
}
public SolrResourceLoader() {
@@ -450,6 +441,18 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
public <T> Class<? extends T> findClass(String cname, Class<T> expectedType) {
return findClass(cname, expectedType, empty);
}
+ private PackageListeningClassLoader createSchemaLoader() {
+ CoreContainer cc = getCoreContainer();
+ if (coreProvider == null) {
+ //corecontainer not available . can't load from packages
+ return null;
+ }
+ return new PackageListeningClassLoader(cc, this, pkg -> {
+ if (getSolrConfig() == null) return null;
+ return getSolrConfig().maxPackageVersion(pkg);
+ }, () -> coreProvider.withCore(core -> core.fetchLatestSchema()));
+ }
+
/**
* This method loads a class either with its FQN or a short-name (solr.class-simplename or class-simplename).
@@ -639,10 +642,8 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
* Tell all {@link SolrCoreAware} instances about the SolrCore
*/
public void inform(SolrCore core) {
- this.coreName = core.getName();
+ this.coreProvider = core.getCoreProvider();
this.config = core.getSolrConfig();
- this.coreId = core.uniqueId;
- this.coreContainer = core.getCoreContainer();
if(getSchemaLoader() != null) core.getPackageListeners().addListener(schemaLoader);
// make a copy to avoid potential deadlock of a callback calling newInstance and trying to
@@ -783,8 +784,8 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, builder.toString());
}
- public CoreContainer getCoreContainer(){
- return coreContainer;
+ public CoreContainer getCoreContainer() {
+ return coreProvider == null ? null : coreProvider.coreContainer;
}
public SolrConfig getSolrConfig() {
diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
index c8a0512..cd10578 100644
--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
@@ -85,6 +85,7 @@ import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.FastInputStream;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SolrNamedThreadFactory;
import org.apache.solr.common.util.SuppressForbidden;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.DirectoryFactory.DirContext;
@@ -94,7 +95,6 @@ import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.CommitUpdateCommand;
-import org.apache.solr.common.util.SolrNamedThreadFactory;
import org.apache.solr.util.FileUtils;
import org.apache.solr.util.PropertiesOutputStream;
import org.apache.solr.util.RTimer;
@@ -909,7 +909,7 @@ public class IndexFetcher {
final CountDownLatch latch = new CountDownLatch(1);
new Thread(() -> {
try {
- solrCore.getCoreContainer().reload(solrCore.getName(), solrCore.uniqueId);
+ solrCore.getCoreContainer().reload(solrCore.getName());
} catch (Exception e) {
log.error("Could not reload core ", e);
} finally {
diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
index 0160473..564b4f1 100644
--- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
@@ -60,6 +60,7 @@ import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SolrNamedThreadFactory;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.ConfigOverlay;
@@ -76,7 +77,6 @@ import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.SchemaManager;
import org.apache.solr.security.AuthorizationContext;
import org.apache.solr.security.PermissionNameProvider;
-import org.apache.solr.common.util.SolrNamedThreadFactory;
import org.apache.solr.util.RTimer;
import org.apache.solr.util.SolrPluginUtils;
import org.apache.solr.util.plugin.SolrCoreAware;
@@ -547,7 +547,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
latestVersion, 30);
} else {
SolrResourceLoader.persistConfLocally(loader, ConfigOverlay.RESOURCE_NAME, overlay.toByteArray());
- req.getCore().getCoreContainer().reload(req.getCore().getName(), req.getCore().uniqueId, false);
+ req.getCore().getCoreContainer().reload(req.getCore().getName());
log.info("Executed config commands successfully and persisted to File System {}", ops);
}
diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
index 0877c54..5b9cb27 100644
--- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
@@ -158,7 +158,7 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
}
@Override
- protected Object initNewInstance(PackageLoader.Package.Version newest) {
+ protected Object initNewInstance(PackageLoader.Package.Version newest, SolrCore core) {
return clazz = newest.getLoader().findClass(pluginInfo.className, Expressible.class);
}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
index 0c6fd80..0278d69 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
@@ -37,20 +37,23 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String LATEST = "$LATEST";
- private final SolrCore core;
private final SolrConfig.SolrPluginInfo pluginMeta;
private PackageLoader.Package.Version pkgVersion;
private PluginInfo info;
+ private final SolrCore.CoreProvider coreProvider;
public PackagePluginHolder(PluginInfo info, SolrCore core, SolrConfig.SolrPluginInfo pluginMeta) {
super(info);
- this.core = core;
+ this.coreProvider = core.getCoreProvider();
this.pluginMeta = pluginMeta;
this.info = info;
+ reload(core.getCoreContainer().getPackageLoader().getPackage(info.pkgName), core);
+ core.getPackageListeners().addListener(getListener(info));
+ }
- reload(core.getCoreContainer().getPackageLoader().getPackage(info.pkgName));
- core.getPackageListeners().addListener(new PackageListeners.Listener() {
+ public PackageListeners.Listener getListener(PluginInfo info) {
+ return new PackageListeners.Listener() {
@Override
public String packageName() {
return info.pkgName;
@@ -63,8 +66,7 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
@Override
public void changed(PackageLoader.Package pkg, Ctx ctx) {
- reload(pkg);
-
+ coreProvider.withCore(core -> reload(pkg, core));
}
@Override
@@ -72,7 +74,7 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
return pkgVersion == null ? null : ew -> pkgVersion.writeMap(ew);
}
- });
+ };
}
public static <T> PluginBag.PluginHolder<T> createHolder(PluginInfo info, SolrCore core, Class<T> type, String msg) {
@@ -83,7 +85,7 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
}
}
- private synchronized void reload(PackageLoader.Package pkg) {
+ private synchronized void reload(PackageLoader.Package pkg, SolrCore core) {
String lessThan = core.getSolrConfig().maxPackageVersion(info.pkgName);
PackageLoader.Package.Version newest = pkg.getLatest(lessThan);
if (newest == null) {
@@ -111,17 +113,17 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
pluginInfo.type, pluginInfo.name, pkg.name(), newest.getVersion());
}
- initNewInstance(newest);
+ initNewInstance(newest, core);
pkgVersion = newest;
}
@SuppressWarnings({"unchecked"})
- protected Object initNewInstance(PackageLoader.Package.Version newest) {
+ protected Object initNewInstance(PackageLoader.Package.Version newest, SolrCore core) {
Object instance = SolrCore.createInstance(pluginInfo.className,
pluginMeta.clazz, pluginMeta.getCleanTag(), core, newest.getLoader());
PluginBag.initInstance(instance, pluginInfo);
- handleAwareCallbacks(newest.getLoader(), instance);
+ handleAwareCallbacks(newest.getLoader(), instance, core);
T old = inst;
inst = (T) instance;
if (old instanceof AutoCloseable) {
@@ -135,7 +137,7 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
return inst;
}
- private void handleAwareCallbacks(SolrResourceLoader loader, Object instance) {
+ private void handleAwareCallbacks(SolrResourceLoader loader, Object instance, SolrCore core) {
if (instance instanceof SolrCoreAware) {
SolrCoreAware coreAware = (SolrCoreAware) instance;
if (!core.getResourceLoader().addToCoreAware(coreAware)) {
diff --git a/solr/core/src/java/org/apache/solr/schema/SchemaManager.java b/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
index 3731a17..0555b3c 100644
--- a/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
+++ b/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
@@ -31,13 +31,13 @@ import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.rest.BaseSolrResource;
-import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
@@ -132,7 +132,7 @@ public class SchemaManager {
latestVersion = ZkController.persistConfigResourceToZooKeeper
(zkLoader, managedIndexSchema.getSchemaZkVersion(), managedIndexSchema.getResourceName(),
sw.toString().getBytes(StandardCharsets.UTF_8), true);
- req.getCore().getCoreContainer().reload(req.getCore().getName(), req.getCore().uniqueId, false);
+ req.getCore().fetchLatestSchema();
break;
} catch (ZkController.ResourceModifiedInZkException e) {
log.info("Schema was modified by another node. Retrying..");
@@ -142,7 +142,7 @@ public class SchemaManager {
//only for non cloud stuff
managedIndexSchema.persistManagedSchema(false);
core.setLatestSchema(managedIndexSchema);
- core.getCoreContainer().reload(core.getName(), core.uniqueId, false);
+ core.fetchLatestSchema();
} catch (SolrException e) {
log.warn(errorMsg);
errors = singletonList(errorMsg + e.getMessage());