You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by he...@apache.org on 2024/06/26 23:33:39 UTC
(pulsar) 07/09: [fix][broker] Ensure that PulsarService is ready for serving incoming requests (#22977)
This is an automated email from the ASF dual-hosted git repository.
heesung pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 04d3fd47cfc5775d0511f343b2f0e79228337968
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Wed Jun 26 17:54:19 2024 +0300
[fix][broker] Ensure that PulsarService is ready for serving incoming requests (#22977)
(cherry picked from commit 53df683b0f78f5f7c12f87e6fbb4d73637ca5bd5)
---
.../org/apache/pulsar/broker/PulsarService.java | 16 ++-
.../extensions/ExtensibleLoadManagerImpl.java | 131 ++++++++++++---------
.../pulsar/broker/namespace/NamespaceService.java | 4 +-
.../broker/service/PulsarChannelInitializer.java | 7 +-
.../apache/pulsar/broker/service/ServerCnx.java | 4 +
.../org/apache/pulsar/broker/web/WebService.java | 50 ++++++++
6 files changed, 156 insertions(+), 56 deletions(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
index 1a45bedfce4..47842e4722e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
@@ -277,6 +277,7 @@ public class PulsarService implements AutoCloseable, ShutdownService {
private final ExecutorProvider transactionExecutorProvider;
private final DefaultMonotonicSnapshotClock monotonicSnapshotClock;
private String brokerId;
+ private final CompletableFuture<Void> readyForIncomingRequestsFuture = new CompletableFuture<>();
public enum State {
Init, Started, Closing, Closed
@@ -904,6 +905,9 @@ public class PulsarService implements AutoCloseable, ShutdownService {
this.metricsGenerator = new MetricsGenerator(this);
+ // the broker is ready to accept incoming requests by Pulsar binary protocol and http/https
+ readyForIncomingRequestsFuture.complete(null);
+
// Initialize the message protocol handlers.
// start the protocol handlers only after the broker is ready,
// so that the protocol handlers can access broker service properly.
@@ -952,12 +956,22 @@ public class PulsarService implements AutoCloseable, ShutdownService {
state = State.Started;
} catch (Exception e) {
LOG.error("Failed to start Pulsar service: {}", e.getMessage(), e);
- throw new PulsarServerException(e);
+ PulsarServerException startException = new PulsarServerException(e);
+ readyForIncomingRequestsFuture.completeExceptionally(startException);
+ throw startException;
} finally {
mutex.unlock();
}
}
+ public void runWhenReadyForIncomingRequests(Runnable runnable) {
+ readyForIncomingRequestsFuture.thenRun(runnable);
+ }
+
+ public void waitUntilReadyForIncomingRequests() throws ExecutionException, InterruptedException {
+ readyForIncomingRequestsFuture.get();
+ }
+
protected BrokerInterceptor newBrokerInterceptor() throws IOException {
return BrokerInterceptors.load(config);
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index cba9bd22c03..2a644a20a46 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -36,7 +36,6 @@ import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -165,10 +164,10 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
private TopBundleLoadDataReporter topBundleLoadDataReporter;
- private ScheduledFuture brokerLoadDataReportTask;
- private ScheduledFuture topBundlesLoadDataReportTask;
+ private volatile ScheduledFuture brokerLoadDataReportTask;
+ private volatile ScheduledFuture topBundlesLoadDataReportTask;
- private ScheduledFuture monitorTask;
+ private volatile ScheduledFuture monitorTask;
private SplitScheduler splitScheduler;
private UnloadManager unloadManager;
@@ -195,7 +194,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
private final ConcurrentHashMap<String, CompletableFuture<Optional<BrokerLookupData>>>
lookupRequests = new ConcurrentHashMap<>();
- private final CountDownLatch initWaiter = new CountDownLatch(1);
+ private final CompletableFuture<Void> initWaiter = new CompletableFuture<>();
/**
* Get all the bundles that are owned by this broker.
@@ -366,12 +365,14 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
pulsar.getCoordinationService(), pulsar.getBrokerId(),
pulsar.getSafeWebServiceAddress(), ELECTION_ROOT,
state -> {
- pulsar.getLoadManagerExecutor().execute(() -> {
- if (state == LeaderElectionState.Leading) {
- playLeader();
- } else {
- playFollower();
- }
+ pulsar.runWhenReadyForIncomingRequests(() -> {
+ pulsar.getLoadManagerExecutor().execute(() -> {
+ if (state == LeaderElectionState.Leading) {
+ playLeader();
+ } else {
+ playFollower();
+ }
+ });
});
});
this.serviceUnitStateChannel = new ServiceUnitStateChannelImpl(pulsar);
@@ -381,7 +382,13 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
this.serviceUnitStateChannel.listen(unloadManager);
this.serviceUnitStateChannel.listen(splitManager);
this.leaderElectionService.start();
- this.serviceUnitStateChannel.start();
+ pulsar.runWhenReadyForIncomingRequests(() -> {
+ try {
+ this.serviceUnitStateChannel.start();
+ } catch (Exception e) {
+ failStarting(e);
+ }
+ });
this.antiAffinityGroupPolicyHelper =
new AntiAffinityGroupPolicyHelper(pulsar, serviceUnitStateChannel);
antiAffinityGroupPolicyHelper.listenFailureDomainUpdate();
@@ -413,54 +420,72 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
new TopBundleLoadDataReporter(pulsar, brokerRegistry.getBrokerId(), topBundlesLoadDataStore);
this.serviceUnitStateChannel.listen(brokerLoadDataReporter);
this.serviceUnitStateChannel.listen(topBundleLoadDataReporter);
- var interval = conf.getLoadBalancerReportUpdateMinIntervalMillis();
- this.brokerLoadDataReportTask = this.pulsar.getLoadManagerExecutor()
- .scheduleAtFixedRate(() -> {
- try {
- brokerLoadDataReporter.reportAsync(false);
- // TODO: update broker load metrics using getLocalData
- } catch (Throwable e) {
- log.error("Failed to run the broker load manager executor job.", e);
- }
- },
- interval,
- interval, TimeUnit.MILLISECONDS);
-
- this.topBundlesLoadDataReportTask = this.pulsar.getLoadManagerExecutor()
- .scheduleAtFixedRate(() -> {
- try {
- // TODO: consider excluding the bundles that are in the process of split.
- topBundleLoadDataReporter.reportAsync(false);
- } catch (Throwable e) {
- log.error("Failed to run the top bundles load manager executor job.", e);
- }
- },
- interval,
- interval, TimeUnit.MILLISECONDS);
-
- this.monitorTask = this.pulsar.getLoadManagerExecutor()
- .scheduleAtFixedRate(() -> {
- monitor();
- },
- MONITOR_INTERVAL_IN_MILLIS,
- MONITOR_INTERVAL_IN_MILLIS, TimeUnit.MILLISECONDS);
this.unloadScheduler = new UnloadScheduler(
pulsar, pulsar.getLoadManagerExecutor(), unloadManager, context,
serviceUnitStateChannel, unloadCounter, unloadMetrics);
this.splitScheduler = new SplitScheduler(
pulsar, serviceUnitStateChannel, splitManager, splitCounter, splitMetrics, context);
- this.splitScheduler.start();
- this.initWaiter.countDown();
- this.started = true;
- log.info("Started load manager.");
+
+ pulsar.runWhenReadyForIncomingRequests(() -> {
+ try {
+ var interval = conf.getLoadBalancerReportUpdateMinIntervalMillis();
+
+ this.brokerLoadDataReportTask = this.pulsar.getLoadManagerExecutor()
+ .scheduleAtFixedRate(() -> {
+ try {
+ brokerLoadDataReporter.reportAsync(false);
+ // TODO: update broker load metrics using getLocalData
+ } catch (Throwable e) {
+ log.error("Failed to run the broker load manager executor job.", e);
+ }
+ },
+ interval,
+ interval, TimeUnit.MILLISECONDS);
+
+ this.topBundlesLoadDataReportTask = this.pulsar.getLoadManagerExecutor()
+ .scheduleAtFixedRate(() -> {
+ try {
+ // TODO: consider excluding the bundles that are in the process of split.
+ topBundleLoadDataReporter.reportAsync(false);
+ } catch (Throwable e) {
+ log.error("Failed to run the top bundles load manager executor job.", e);
+ }
+ },
+ interval,
+ interval, TimeUnit.MILLISECONDS);
+
+ this.monitorTask = this.pulsar.getLoadManagerExecutor()
+ .scheduleAtFixedRate(() -> {
+ monitor();
+ },
+ MONITOR_INTERVAL_IN_MILLIS,
+ MONITOR_INTERVAL_IN_MILLIS, TimeUnit.MILLISECONDS);
+
+ this.splitScheduler.start();
+ this.initWaiter.complete(null);
+ this.started = true;
+ log.info("Started load manager.");
+ } catch (Exception ex) {
+ failStarting(ex);
+ }
+ });
} catch (Exception ex) {
- log.error("Failed to start the extensible load balance and close broker registry {}.",
- this.brokerRegistry, ex);
- if (this.brokerRegistry != null) {
+ failStarting(ex);
+ }
+ }
+
+ private void failStarting(Exception ex) {
+ log.error("Failed to start the extensible load balance and close broker registry {}.",
+ this.brokerRegistry, ex);
+ if (this.brokerRegistry != null) {
+ try {
brokerRegistry.close();
+ } catch (PulsarServerException e) {
+ // ignore
}
}
+ initWaiter.completeExceptionally(ex);
}
@Override
@@ -817,7 +842,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
boolean becameFollower = false;
while (!Thread.currentThread().isInterrupted()) {
try {
- initWaiter.await();
+ initWaiter.get();
if (!serviceUnitStateChannel.isChannelOwner()) {
becameFollower = true;
break;
@@ -867,7 +892,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
boolean becameLeader = false;
while (!Thread.currentThread().isInterrupted()) {
try {
- initWaiter.await();
+ initWaiter.get();
if (serviceUnitStateChannel.isChannelOwner()) {
becameLeader = true;
break;
@@ -929,7 +954,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
@VisibleForTesting
protected void monitor() {
try {
- initWaiter.await();
+ initWaiter.get();
// Monitor role
// Periodically check the role in case ZK watcher fails.
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
index 528f7cd9aa7..622331e8fbe 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
@@ -1278,7 +1278,9 @@ public class NamespaceService implements AutoCloseable {
bundleOwnershipListeners.add(listener);
}
}
- getOwnedServiceUnits().forEach(bundle -> notifyNamespaceBundleOwnershipListener(bundle, listeners));
+ pulsar.runWhenReadyForIncomingRequests(() -> {
+ getOwnedServiceUnits().forEach(bundle -> notifyNamespaceBundleOwnershipListener(bundle, listeners));
+ });
}
public void addNamespaceBundleSplitListener(NamespaceBundleSplitListener... listeners) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java
index 5308b3c981e..e276ea24fed 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java
@@ -19,6 +19,7 @@
package org.apache.pulsar.broker.service;
import com.google.common.annotations.VisibleForTesting;
+import io.netty.channel.ChannelHandler;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.socket.SocketChannel;
import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
@@ -104,6 +105,9 @@ public class PulsarChannelInitializer extends ChannelInitializer<SocketChannel>
@Override
protected void initChannel(SocketChannel ch) throws Exception {
+ // disable auto read explicitly so that requests aren't served until auto read is enabled
+ // ServerCnx must enable auto read in channelActive after PulsarService is ready to accept incoming requests
+ ch.config().setAutoRead(false);
ch.pipeline().addLast("consolidation", new FlushConsolidationHandler(1024, true));
if (this.enableTls) {
if (this.tlsEnabledWithKeyStore) {
@@ -128,7 +132,8 @@ public class PulsarChannelInitializer extends ChannelInitializer<SocketChannel>
// ServerCnx ends up reading higher number of messages and broker can not throttle the messages by disabling
// auto-read.
ch.pipeline().addLast("flowController", new FlowControlHandler());
- ServerCnx cnx = newServerCnx(pulsar, listenerName);
+ // using "ChannelHandler" type to workaround an IntelliJ bug that shows a false positive error
+ ChannelHandler cnx = newServerCnx(pulsar, listenerName);
ch.pipeline().addLast("handler", cnx);
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index e50e5c3cd1b..59a0b3deeff 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -365,6 +365,10 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
this.commandSender = new PulsarCommandSenderImpl(brokerInterceptor, this);
this.service.getPulsarStats().recordConnectionCreate();
cnxsPerThread.get().add(this);
+ service.getPulsar().runWhenReadyForIncomingRequests(() -> {
+ // enable auto read after PulsarService is ready to accept incoming requests
+ ctx.channel().config().setAutoRead(true);
+ });
}
@Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java
index 9a439268a8b..89c12c6771e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java
@@ -20,12 +20,21 @@ package org.apache.pulsar.broker.web;
import io.prometheus.client.CollectorRegistry;
import io.prometheus.client.jetty.JettyStatisticsCollector;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.concurrent.ExecutionException;
import javax.servlet.DispatcherType;
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletResponse;
import lombok.Getter;
import org.apache.pulsar.broker.PulsarServerException;
import org.apache.pulsar.broker.PulsarService;
@@ -228,6 +237,7 @@ public class WebService implements AutoCloseable {
private final FilterHolder authenticationFilterHolder;
FilterInitializer(PulsarService pulsarService) {
ServiceConfiguration config = pulsarService.getConfiguration();
+
if (config.getMaxConcurrentHttpRequests() > 0) {
FilterHolder filterHolder = new FilterHolder(QoSFilter.class);
filterHolder.setInitParameter("maxRequests", String.valueOf(config.getMaxConcurrentHttpRequests()));
@@ -239,6 +249,10 @@ public class WebService implements AutoCloseable {
new RateLimitingFilter(config.getHttpRequestsMaxPerSecond())));
}
+ // wait until the PulsarService is ready to serve incoming requests
+ filterHolders.add(
+ new FilterHolder(new WaitUntilPulsarServiceIsReadyForIncomingRequestsFilter(pulsarService)));
+
boolean brokerInterceptorEnabled = pulsarService.getBrokerInterceptor() != null;
if (brokerInterceptorEnabled) {
ExceptionHandler handler = new ExceptionHandler();
@@ -280,6 +294,42 @@ public class WebService implements AutoCloseable {
}
}
+ // Filter that waits until the PulsarService is ready to serve incoming requests
+ private static class WaitUntilPulsarServiceIsReadyForIncomingRequestsFilter implements Filter {
+ private final PulsarService pulsarService;
+
+ public WaitUntilPulsarServiceIsReadyForIncomingRequestsFilter(PulsarService pulsarService) {
+ this.pulsarService = pulsarService;
+ }
+
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+
+ }
+
+ @Override
+ public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain)
+ throws IOException, ServletException {
+ try {
+ // Wait until the PulsarService is ready to serve incoming requests
+ pulsarService.waitUntilReadyForIncomingRequests();
+ } catch (ExecutionException e) {
+ ((HttpServletResponse) response).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE,
+ "PulsarService failed to start.");
+ return;
+ } catch (InterruptedException e) {
+ ((HttpServletResponse) response).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE,
+ "PulsarService is not ready.");
+ return;
+ }
+ chain.doFilter(request, response);
+ }
+
+ @Override
+ public void destroy() {
+
+ }
+ }
}
public void addServlet(String path, ServletHolder servletHolder, boolean requiresAuthentication,