[pulsar] 05/08: Introduce metrics servlet timeout setting (#10886)

[pe...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

penghui pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit a750f975799a8690535fa493a0b09a3a226c06db
Author: Sijie Guo <si...@apache.org>
AuthorDate: Thu Jun 10 15:02:35 2021 +0800

    Introduce metrics servlet timeout setting (#10886)
    
    *Motivation*
    
    Generating metrics is an expensive task and it can take more than 30 seconds
    if you have close to or more than a million topics.
    
    *Modification*
    
    This change provides a setting to adjust the async context timeout.
    
    (cherry picked from commit c75d45b3f9be7a210938f5e87c7b4301e4fd25ee)
---
 conf/broker.conf                                                  | 5 +++++
 conf/standalone.conf                                              | 5 +++++
 .../main/java/org/apache/pulsar/broker/ServiceConfiguration.java  | 8 ++++++++
 .../pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java  | 3 +++
 4 files changed, 21 insertions(+)

diff --git a/conf/broker.conf b/conf/broker.conf
index f0f0da3..dc9d732 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -1142,6 +1142,11 @@ exposeManagedCursorMetricsInPrometheus=false
 # Classname of Pluggable JVM GC metrics logger that can log GC specific metrics
 # jvmGCMetricsLoggerClassName=
 
+# Time in milliseconds that metrics endpoint would time out. Default is 30s.
+# Increase it if there are a lot of topics to expose topic-level metrics.
+# Set it to 0 to disable timeout.
+metricsServletTimeoutMs=30000
+
 ### --- Functions --- ###
 
 # Enable Functions Worker Service in Broker
diff --git a/conf/standalone.conf b/conf/standalone.conf
index ebb110a..7c818d3 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -831,6 +831,11 @@ webSocketMaxTextFrameSize=1048576
 # Enable topic level metrics
 exposeTopicLevelMetricsInPrometheus=true
 
+# Time in milliseconds that metrics endpoint would time out. Default is 30s.
+# Increase it if there are a lot of topics to expose topic-level metrics.
+# Set it to 0 to disable timeout.
+metricsServletTimeoutMs=30000
+
 # Classname of Pluggable JVM GC metrics logger that can log GC specific metrics
 # jvmGCMetricsLoggerClassName=
 
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 21cdcef..2299e42 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -1962,6 +1962,14 @@ public class ServiceConfiguration implements PulsarConfiguration {
     private boolean exposePreciseBacklogInPrometheus = false;
 
     @FieldContext(
+        category = CATEGORY_METRICS,
+        doc = "Time in milliseconds that metrics endpoint would time out. Default is 30s.\n" +
+            " Increase it if there are a lot of topics to expose topic-level metrics.\n" +
+            " Set it to 0 to disable timeout."
+    )
+    private long metricsServletTimeoutMs = 30000;
+
+    @FieldContext(
             category = CATEGORY_METRICS,
             doc = "Enable expose the backlog size for each subscription when generating stats.\n" +
                     " Locking is used for fetching the status so default to false."
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java
index 7e469a2..026c26f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java
@@ -43,6 +43,7 @@ public class PrometheusMetricsServlet extends HttpServlet {
     private final boolean shouldExportTopicMetrics;
     private final boolean shouldExportConsumerMetrics;
     private final boolean shouldExportProducerMetrics;
+    private final long metricsServletTimeoutMs;
     private List<PrometheusRawMetricsProvider> metricsProviders;
 
     private ExecutorService executor = null;
@@ -53,6 +54,7 @@ public class PrometheusMetricsServlet extends HttpServlet {
         this.shouldExportTopicMetrics = includeTopicMetrics;
         this.shouldExportConsumerMetrics = includeConsumerMetrics;
         this.shouldExportProducerMetrics = shouldExportProducerMetrics;
+        this.metricsServletTimeoutMs = pulsar.getConfiguration().getMetricsServletTimeoutMs();
     }
 
     @Override
@@ -64,6 +66,7 @@ public class PrometheusMetricsServlet extends HttpServlet {
     protected void doGet(HttpServletRequest request, HttpServletResponse response)
             throws ServletException, IOException {
         AsyncContext context = request.startAsync();
+        context.setTimeout(metricsServletTimeoutMs);
         executor.execute(safeRun(() -> {
             HttpServletResponse res = (HttpServletResponse) context.getResponse();
             try {