You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by md...@apache.org on 2020/10/27 21:45:45 UTC
[lucene-solr] branch branch_8x updated: SOLR-14940: Fix
ReplicationHandler memory leak through SolrCore.closeHooks
This is an automated email from the ASF dual-hosted git repository.
mdrob pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new b8d2855 SOLR-14940: Fix ReplicationHandler memory leak through SolrCore.closeHooks
b8d2855 is described below
commit b8d28555f9f68cd00fc9b65342a175db595c11a3
Author: Anver Sotnikov <as...@tripadvisor.com>
AuthorDate: Fri Oct 16 14:15:53 2020 -0400
SOLR-14940: Fix ReplicationHandler memory leak through SolrCore.closeHooks
* Added ability to remove SolrCore.closeHooks
* Keep references to CloseHooks in ReplicationHandler and remove them on ReplicationHandler.shutdown()
closes #1997
---
solr/CHANGES.txt | 4 +-
.../org/apache/solr/cloud/ReplicateFromLeader.java | 10 ++-
.../java/org/apache/solr/cloud/ZkController.java | 2 +-
.../src/java/org/apache/solr/core/SolrCore.java | 14 ++++
.../apache/solr/handler/ReplicationHandler.java | 82 +++++++++++-----------
.../solr/cloud/TestPullReplicaErrorHandling.java | 21 +++++-
6 files changed, 88 insertions(+), 45 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 81e07da..9049e4e 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -23,10 +23,12 @@ Optimizations
Bug Fixes
---------------------
-(No changes)
+
+* SOLR-14940: ReplicationHandler memory leak through SolrCore.closeHooks with unstable ZK connection. (Anver Sotnikov, Mike Drob)
Other Changes
---------------------
+
* SOLR-14954: Heavily edit reindexing.adoc (Sameul García Martínez and Erick Erickson)
================== 8.7.0 ==================
diff --git a/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java b/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java
index 7e2b872..1fa86c0 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java
@@ -52,10 +52,18 @@ public class ReplicateFromLeader {
/**
* Start a replication handler thread that will periodically pull indices from the shard leader
+ *
+ * <p>This is separate from the ReplicationHandler that listens at /replication, used for recovery
+ * and leader actions. It is simpler to discard the entire polling ReplicationHandler rather then
+ * worrying about disabling polling and correctly setting all of the leader bits if we need to reset.
+ *
+ * <p>TODO: It may be cleaner to extract the polling logic use that directly instead of creating
+ * what might be a fairly heavyweight instance here.
+ *
* @param switchTransactionLog if true, ReplicationHandler will rotate the transaction log once
* the replication is done
*/
- public void startReplication(boolean switchTransactionLog) throws InterruptedException {
+ public void startReplication(boolean switchTransactionLog) {
try (SolrCore core = cc.getCore(coreName)) {
if (core == null) {
if (cc.isShutDown()) {
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index 189c713..0ab5b25 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -1302,7 +1302,7 @@ public class ZkController implements Closeable {
return replica;
}
- public void startReplicationFromLeader(String coreName, boolean switchTransactionLog) throws InterruptedException {
+ public void startReplicationFromLeader(String coreName, boolean switchTransactionLog) {
log.info("{} starting background replication from leader", coreName);
ReplicateFromLeader replicateFromLeader = new ReplicateFromLeader(cc, coreName);
synchronized (replicateFromLeader) { // synchronize to prevent any stop before we finish the start
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index a623cc5..d29e6ff 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -1758,6 +1758,20 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
}
/**
+ * Remove a close callback hook
+ */
+ public void removeCloseHook(CloseHook hook) {
+ if (closeHooks != null) {
+ closeHooks.remove(hook);
+ }
+ }
+
+ // Visible for testing
+ public Collection<CloseHook> getCloseHooks() {
+ return Collections.unmodifiableCollection(closeHooks);
+ }
+
+ /**
* @lucene.internal Debugging aid only. No non-test code should be released with uncommented verbose() calls.
*/
public static boolean VERBOSE = Boolean.parseBoolean(System.getProperty("tests.verbose", "false"));
diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
index 5c65c4c..709b085 100644
--- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
@@ -1425,55 +1425,55 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
return Boolean.TRUE.equals( enable );
}
- /**
- * register a closehook
- */
- private void registerCloseHook() {
- core.addCloseHook(new CloseHook() {
- @Override
- public void preClose(SolrCore core) {
- if (executorService != null) executorService.shutdown(); // we don't wait for shutdown - this can deadlock core reload
- }
+ private final CloseHook startShutdownHook = new CloseHook() {
+ @Override
+ public void preClose(SolrCore core) {
+ if (executorService != null)
+ executorService.shutdown(); // we don't wait for shutdown - this can deadlock core reload
+ }
- @Override
- public void postClose(SolrCore core) {
- if (pollingIndexFetcher != null) {
- pollingIndexFetcher.destroy();
- }
- if (currentIndexFetcher != null && currentIndexFetcher != pollingIndexFetcher) {
- currentIndexFetcher.destroy();
- }
+ @Override
+ public void postClose(SolrCore core) {
+ if (pollingIndexFetcher != null) {
+ pollingIndexFetcher.destroy();
}
- });
-
- core.addCloseHook(new CloseHook() {
- @Override
- public void preClose(SolrCore core) {
- ExecutorUtil.shutdownAndAwaitTermination(restoreExecutor);
- if (restoreFuture != null) {
- restoreFuture.cancel(false);
- }
+ if (currentIndexFetcher != null && currentIndexFetcher != pollingIndexFetcher) {
+ currentIndexFetcher.destroy();
}
+ }
+ };
+ private final CloseHook finishShutdownHook = new CloseHook() {
+ @Override
+ public void preClose(SolrCore core) {
+ ExecutorUtil.shutdownAndAwaitTermination(restoreExecutor);
+ if (restoreFuture != null) {
+ restoreFuture.cancel(false);
+ }
+ }
- @Override
- public void postClose(SolrCore core) {}
- });
+ @Override
+ public void postClose(SolrCore core) {
+ }
+ };
+
+ /**
+ * register a closehook
+ */
+ private void registerCloseHook() {
+ core.addCloseHook(startShutdownHook);
+ core.addCloseHook(finishShutdownHook);
}
public void shutdown() {
- if (executorService != null) executorService.shutdown();
- if (pollingIndexFetcher != null) {
- pollingIndexFetcher.destroy();
- }
- if (currentIndexFetcher != null && currentIndexFetcher != pollingIndexFetcher) {
- currentIndexFetcher.destroy();
- }
- ExecutorUtil.shutdownAndAwaitTermination(restoreExecutor);
- if (restoreFuture != null) {
- restoreFuture.cancel(false);
- }
-
+ startShutdownHook.preClose(core);
+ startShutdownHook.postClose(core);
+ finishShutdownHook.preClose(core);
+ finishShutdownHook.postClose(core);
+
ExecutorUtil.shutdownAndAwaitTermination(executorService);
+
+ core.removeCloseHook(startShutdownHook);
+ core.removeCloseHook(finishShutdownHook);
}
/**
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java b/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java
index 6a79c61..d66b313 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java
@@ -45,6 +45,7 @@ import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.TimeSource;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.util.TestInjection;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
@@ -237,11 +238,29 @@ public void testCantConnectToPullReplica() throws Exception {
addDocs(30);
waitForState("Expecting node to be disconnected", collectionName, activeReplicaCount(1, 0, 0));
addDocs(40);
- waitForState("Expecting node to be disconnected", collectionName, activeReplicaCount(1, 0, 1));
+ waitForState("Expecting node to be reconnected", collectionName, activeReplicaCount(1, 0, 1));
try (HttpSolrClient pullReplicaClient = getHttpSolrClient(s.getReplicas(EnumSet.of(Replica.Type.PULL)).get(0).getCoreUrl())) {
assertNumDocs(40, pullReplicaClient);
}
}
+
+ public void testCloseHooksDeletedOnReconnect() throws Exception {
+ CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1, 0, 1)
+ .process(cluster.getSolrClient());
+ addDocs(10);
+
+ DocCollection docCollection = assertNumberOfReplicas(1, 0, 1, false, true);
+ Slice s = docCollection.getSlices().iterator().next();
+ JettySolrRunner jetty = getJettyForReplica(s.getReplicas(EnumSet.of(Replica.Type.PULL)).get(0));
+ SolrCore core = jetty.getCoreContainer().getCores().iterator().next();
+
+ for (int i = 0; i < 5; i++) {
+ cluster.expireZkSession(jetty);
+ waitForState("Expecting node to reconnect", collectionName, activeReplicaCount(1, 0, 1));
+ // We have two active ReplicationHandler with two close hooks each, one for triggering recovery and one for doing interval polling
+ assertEquals(5, core.getCloseHooks().size());
+ }
+ }
private void assertNumDocs(int numDocs, SolrClient client, int timeoutSecs) throws InterruptedException, SolrServerException, IOException {
TimeOut t = new TimeOut(timeoutSecs, TimeUnit.SECONDS, TimeSource.NANO_TIME);