You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ho...@apache.org on 2025/04/28 16:32:43 UTC
(solr) branch main updated: Solr now enables Jetty's Graceful Shutdown features to prevent client connections from being abruptly terminated on orderly shutdown
This is an automated email from the ASF dual-hosted git repository.
hossman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new d0d71a7b38f Solr now enables Jetty's Graceful Shutdown features to prevent client connections from being abruptly terminated on orderly shutdown
d0d71a7b38f is described below
commit d0d71a7b38f87f3c70551678b2eab62d925483ef
Author: Chris Hostetter <ho...@apache.org>
AuthorDate: Mon Apr 28 09:32:32 2025 -0700
Solr now enables Jetty's Graceful Shutdown features to prevent client connections from being abruptly terminated on orderly shutdown
---
solr/CHANGES.txt | 2 +
solr/bin/solr | 5 +
.../test/org/apache/solr/cloud/SplitShardTest.java | 1 +
.../solr/cloud/TestGracefulJettyShutdown.java | 161 +++++++++++++++++++++
solr/server/etc/jetty-graceful.xml | 22 +++
solr/server/modules/graceful.mod | 8 +
.../apache/solr/cloud/MiniSolrCloudCluster.java | 2 +
.../org/apache/solr/embedded/JettySolrRunner.java | 7 +
8 files changed, 208 insertions(+)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 827010c59c1..5501eb95017 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -231,6 +231,8 @@ Improvements
* SOLR-17732: Score-based return fields other than "score" can now be returned in distributed queries. (Houston Putman)
+* SOLR-17744: Solr now enables Jetty's Graceful Shutdown features to prevent client connections from being abruptly terminated on orderly shutdown (hossman)
+
Optimizations
---------------------
* SOLR-17578: Remove ZkController internal core supplier, for slightly faster reconnection after Zookeeper session loss. (Pierre Salagnac)
diff --git a/solr/bin/solr b/solr/bin/solr
index 3466cdc6b23..846ff39ce32 100755
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -300,6 +300,11 @@ else
fi
export SOLR_URL_SCHEME
+# Gracefully wait for existing requests on shutdown
+if [ "${SOLR_JETTY_GRACEFUL:-true}" == "true" ]; then
+ SOLR_JETTY_CONFIG+=("--module=graceful")
+fi
+
# Requestlog options
if [ "${SOLR_REQUESTLOG_ENABLED:-true}" == "true" ]; then
SOLR_JETTY_CONFIG+=("--module=requestlog")
diff --git a/solr/core/src/test/org/apache/solr/cloud/SplitShardTest.java b/solr/core/src/test/org/apache/solr/cloud/SplitShardTest.java
index 515664f1150..ba5949b7a13 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SplitShardTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SplitShardTest.java
@@ -387,6 +387,7 @@ public class SplitShardTest extends SolrCloudTestCase {
(s, replica) -> {
if (replica.getNodeName().equals(jetty.getNodeName())
&& !replica.isLeader()
+ && replica.getState().equals(Replica.State.ACTIVE)
&& set.contains(replica.shard)) {
set.remove(replica.shard);
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestGracefulJettyShutdown.java b/solr/core/src/test/org/apache/solr/cloud/TestGracefulJettyShutdown.java
new file mode 100644
index 00000000000..44caa313c52
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/TestGracefulJettyShutdown.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.embedded.JettyConfig;
+import org.apache.solr.embedded.JettySolrRunner;
+import org.apache.solr.handler.component.SearchHandler;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestGracefulJettyShutdown extends SolrTestCaseJ4 {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public void testSingleShardInFlightRequestsDuringShutDown() throws Exception {
+ final String collection = getSaferTestName();
+ final String handler = "/foo";
+
+ final Semaphore handlerGate = new Semaphore(0);
+ final Semaphore handlerSignal = new Semaphore(0);
+
+ final ExecutorService exec = ExecutorUtil.newMDCAwareCachedThreadPool("client-requests");
+ final MiniSolrCloudCluster cluster =
+ new MiniSolrCloudCluster(1, createTempDir(), JettyConfig.builder().build());
+ try {
+ assertTrue(
+ CollectionAdminRequest.createCollection(collection, "_default", 1, 1)
+ .process(cluster.getSolrClient())
+ .isSuccess());
+
+ final JettySolrRunner nodeToStop = cluster.getJettySolrRunner(0);
+
+ // register our custom handler with "all" (one) of our SolrCores
+ for (String coreName : nodeToStop.getCoreContainer().getLoadedCoreNames()) {
+ try (SolrCore core = nodeToStop.getCoreContainer().getCore(coreName)) {
+ final BlockingSearchHandler h = new BlockingSearchHandler(handlerGate, handlerSignal);
+ h.inform(core);
+ core.registerRequestHandler(handler, h);
+ }
+ }
+
+ final CloudSolrClient cloudClient = cluster.getSolrClient();
+
+ // add a few docs...
+ cloudClient.add(collection, sdoc("id", "xxx", "foo_s", "aaa"));
+ cloudClient.add(collection, sdoc("id", "yyy", "foo_s", "bbb"));
+ cloudClient.add(collection, sdoc("id", "zzz", "foo_s", "aaa"));
+ cloudClient.commit(collection);
+
+ final List<Future<QueryResponse>> results = new ArrayList<>(13);
+
+ try (SolrClient jettyClient = nodeToStop.newClient()) {
+ final QueryRequest req = new QueryRequest(params("q", "foo_s:aaa"));
+ req.setPath(handler);
+
+ // check inflight requests using both clients...
+ for (SolrClient client : Arrays.asList(cloudClient, jettyClient)) {
+ results.add(
+ exec.submit(
+ () -> {
+ return req.process(client, collection);
+ }));
+ }
+
+ // wait for our handlers to indicate they have recieved the requests and started processing
+ log.info("Waiting for signals from both requests");
+ assertTrue(handlerSignal.tryAcquire(2, 300, TimeUnit.SECONDS)); // safety valve
+
+ // stop our node (via executor so it doesn't block) and open the gate for our handlers
+ log.info("Stopping jetty node");
+ final Future<Boolean> stopped =
+ exec.submit(
+ () -> {
+ nodeToStop.stop();
+ return true;
+ });
+ log.info("Releasing gate for requests");
+ handlerGate.release(2);
+ log.info("Released gate for requests");
+
+ // confirm success of requests
+ assertEquals(2, results.size());
+ for (Future<QueryResponse> f : results) {
+ final QueryResponse rsp = f.get(300, TimeUnit.SECONDS); // safety valve
+ assertEquals(2, rsp.getResults().getNumFound());
+ }
+ assertTrue(stopped.get(300, TimeUnit.SECONDS)); // safety valve
+ }
+
+ } finally {
+ handlerGate.release(9999999); // safety valve
+ handlerSignal.release(9999999); // safety valve
+ cluster.shutdown();
+ ExecutorUtil.shutdownAndAwaitTermination(exec);
+ }
+ }
+
+ /**
+ * On every request, prior to handling, releases a ticket to it's signal Semaphre, and then blocks
+ * until it can acquire a ticket from it's gate semaphore. Has a safety valve that gives up on the
+ * gate after 300 seconds
+ */
+ private static final class BlockingSearchHandler extends SearchHandler {
+ private final Semaphore gate;
+ private final Semaphore signal;
+
+ public BlockingSearchHandler(final Semaphore gate, final Semaphore signal) {
+ super();
+ this.gate = gate;
+ this.signal = signal;
+ super.init(new NamedList<>());
+ }
+
+ @Override
+ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ log.info("Starting request");
+ signal.release();
+ if (gate.tryAcquire(300, TimeUnit.SECONDS)) {
+ super.handleRequestBody(req, rsp);
+ } else {
+ log.error("Gate safety valve timeout");
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Gate timeout");
+ }
+ log.info("Finishing request");
+ }
+ }
+}
diff --git a/solr/server/etc/jetty-graceful.xml b/solr/server/etc/jetty-graceful.xml
new file mode 100644
index 00000000000..34ec9316445
--- /dev/null
+++ b/solr/server/etc/jetty-graceful.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" "https://www.eclipse.org/jetty/configure_10_0.dtd">
+
+<!-- =============================================================== -->
+<!-- Configure Jetty to finish existing requests on STOP. -->
+<!-- -->
+<!-- This is inspired by the "official" Jetty 12 graceful module, -->
+<!-- adapted to use StatisticsHandler for Jetty 10 and 11. -->
+<!-- -->
+<!-- IF THIS IS MODIFIED, JettySolrRunner MUST BE MODIFIED! -->
+<!-- =============================================================== -->
+
+<Configure id="Server" class="org.eclipse.jetty.server.Server">
+ <Call name="insertHandler">
+ <Arg>
+ <New id="GracefulHandler" class="org.eclipse.jetty.server.handler.StatisticsHandler">
+ <Set name="gracefulShutdownWaitsForRequests">true</Set>
+ </New>
+ </Arg>
+ </Call>
+ <Set name="stopTimeout"><Property name="solr.jetty.stop.timeout" default="15000"/></Set>
+</Configure>
diff --git a/solr/server/modules/graceful.mod b/solr/server/modules/graceful.mod
new file mode 100644
index 00000000000..581ebca898f
--- /dev/null
+++ b/solr/server/modules/graceful.mod
@@ -0,0 +1,8 @@
+[description]
+Enables Graceful processing of requests on STOP. Inspired by graceful.mod for Jetty-12, this works with Jetty 10 and 11.
+
+[depend]
+server
+
+[xml]
+etc/jetty-graceful.xml
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
index 7f9a05229e8..8049fc59d70 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
@@ -775,9 +775,11 @@ public class MiniSolrCloudCluster {
try {
future.get();
} catch (ExecutionException e) {
+ log.error(message, e);
parsed.addSuppressed(e.getCause());
ok = false;
} catch (InterruptedException e) {
+ log.error(message, e);
Thread.interrupted();
throw e;
}
diff --git a/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java b/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java
index 12baf8a0e37..93bf50e7471 100644
--- a/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java
+++ b/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java
@@ -83,6 +83,7 @@ import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.SslConnectionFactory;
import org.eclipse.jetty.server.handler.HandlerWrapper;
+import org.eclipse.jetty.server.handler.StatisticsHandler;
import org.eclipse.jetty.server.handler.gzip.GzipHandler;
import org.eclipse.jetty.server.session.DefaultSessionIdManager;
import org.eclipse.jetty.servlet.FilterHolder;
@@ -431,6 +432,12 @@ public class JettySolrRunner {
gzipHandler.setIncludedMethods("GET");
server.setHandler(gzipHandler);
+
+ // Mimic "graceful.mod"
+ final StatisticsHandler graceful = new StatisticsHandler();
+ graceful.setGracefulShutdownWaitsForRequests(true);
+ server.insertHandler(graceful);
+ server.setStopTimeout(15 * 1000);
}
/**