You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by bo...@apache.org on 2022/11/16 01:43:18 UTC

[pulsar] branch master updated: [fix] [txn] Make txn metrics name conforms to the rule (#17905)

This is an automated email from the ASF dual-hosted git repository.

bogong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new 3715934d604 [fix] [txn] Make txn metrics name conforms to the rule (#17905)
3715934d604 is described below

commit 3715934d6046560669e876a94d3cdd580a21c5fa
Author: fengyubiao <yu...@streamnative.io>
AuthorDate: Wed Nov 16 09:43:11 2022 +0800

    [fix] [txn] Make txn metrics name conforms to the rule (#17905)
    
    Fixes: #17921
    
    <strong>Note</strong>:
    
    This patch will change metrics names `s_bufferedwriter_batch_record_count` and `s_bufferedwriter_batch_oldest_record_delay_time_second`. These two names were first used in this PR #17701, and PR #17701 hasn't cherry-picked any branches yet, so this change will not cause any breaking changes.
    
    ### Motivation
    
    https://github.com/poorbarcode/pulsar/actions/runs/3156649582/jobs/5136584463
    https://github.com/apache/pulsar/actions/runs/3156649597/jobs/5136596447
    
    #### Problem-1
    
    If the `Prometheus-Colloctor` which typed `Counter` is named 'xxx_count',  then the output `metrics-api` will be named 'xxx_count_count'.
    
    `TxnLogBufferedWriterMetricsStats` hits this error.
    
    https://github.com/apache/pulsar/blob/fb7307d8f4998e42b18df3a4599fd7ec34cb04a9/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/TxnLogBufferedWriterMetricsStats.java#L105-L106
    
    
    ----
    
    #### Problem-2
    
    `PrometheusMetricsTest` defines the standard metrics name(see code below):
    
    ```
    ["_sum", "_bucket", "_count", "_total", "_created"]
    ```
    
    But the standard Prometheus name has three others( see: https://github.com/prometheus/client_java/blob/c28b901225e35e7c1df0eacae8b58fdfbb390162/simpleclient/src/main/java/io/prometheus/client/Collector.java#L152-L186 ):
    
    ```
    ["_info", "_gsum", "_gcount"]
    ```
    
    
    https://github.com/apache/pulsar/blob/fb7307d8f4998e42b18df3a4599fd7ec34cb04a9/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java#L834-L861
    
    ----
    
    ### Modifications
    
    - Make `PrometheusMetricsTest` run with transaction feature
    - Make txn metrics name conforms to the rule. see: https://prometheus.io/docs/practices/naming/
    - Make `PrometheusMetricsTest` support all suffix of prometheus metrics name
    
    ### Documentation
    
    - [x] `doc-not-needed`
    (Please explain why)
    
    ### Matching PR in forked repository
    
    PR in forked repository:
    
    - https://github.com/poorbarcode/pulsar/pull/19
---
 .../pulsar/broker/stats/PrometheusMetricsTest.java | 64 +++++++++++++---------
 .../stats/TransactionBatchWriterMetricsTest.java   |  8 ++-
 .../impl/TxnLogBufferedWriterMetricsStats.java     |  4 +-
 .../coordinator/impl/TxnLogBufferedWriterTest.java |  6 +-
 4 files changed, 50 insertions(+), 32 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
index a33c54a9ed4..b79fdecce95 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
@@ -28,32 +28,38 @@ import com.google.common.base.Splitter;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Multimap;
 import io.jsonwebtoken.SignatureAlgorithm;
+import io.prometheus.client.Collector;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.lang.reflect.Field;
 import java.math.RoundingMode;
 import java.nio.charset.StandardCharsets;
 import java.text.NumberFormat;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Properties;
 import java.util.Random;
+import java.util.Set;
 import java.util.TreeMap;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 import javax.crypto.SecretKey;
 import javax.naming.AuthenticationException;
 import lombok.Cleanup;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
 import org.apache.pulsar.broker.authentication.AuthenticationProviderToken;
@@ -88,6 +94,9 @@ public class PrometheusMetricsTest extends BrokerTestBase {
     protected void setup() throws Exception {
         conf.setTopicLevelPoliciesEnabled(false);
         conf.setSystemTopicEnabled(false);
+        conf.setTransactionCoordinatorEnabled(true);
+        conf.setTransactionLogBatchedWriteEnabled(true);
+        conf.setTransactionPendingAckBatchedWriteEnabled(true);
         super.baseSetup();
         AuthenticationProviderToken.resetMetrics();
     }
@@ -773,6 +782,7 @@ public class PrometheusMetricsTest extends BrokerTestBase {
     // Running the test twice to make sure types are present when generated multiple times
     @Test(invocationCount = 2)
     public void testDuplicateMetricTypeDefinitions() throws Exception {
+        Set<String> allPrometheusSuffixString = allPrometheusSuffixEnums();
         Producer<byte[]> p1 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
         Producer<byte[]> p2 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic2").create();
         for (int i = 0; i < 10; i++) {
@@ -831,32 +841,18 @@ public class PrometheusMetricsTest extends BrokerTestBase {
 
             if (!typeDefs.containsKey(metricName)) {
                 // This may be OK if this is a _sum or _count metric from a summary
-                if (metricName.endsWith("_sum")) {
-                    String summaryMetricName = metricName.substring(0, metricName.indexOf("_sum"));
-                    if (!typeDefs.containsKey(summaryMetricName)) {
-                        fail("Metric " + metricName + " does not have a corresponding summary type definition");
+                boolean isNorm = false;
+                for (String suffix : allPrometheusSuffixString){
+                    if (metricName.endsWith(suffix)){
+                        String summaryMetricName = metricName.substring(0, metricName.indexOf(suffix));
+                        if (!typeDefs.containsKey(summaryMetricName)) {
+                            fail("Metric " + metricName + " does not have a corresponding summary type definition");
+                        }
+                        isNorm = true;
+                        break;
                     }
-                } else if (metricName.endsWith("_count")) {
-                    String summaryMetricName = metricName.substring(0, metricName.indexOf("_count"));
-                    if (!typeDefs.containsKey(summaryMetricName)) {
-                        fail("Metric " + metricName + " does not have a corresponding summary type definition");
-                    }
-                } else if (metricName.endsWith("_bucket")) {
-                    String summaryMetricName = metricName.substring(0, metricName.indexOf("_bucket"));
-                    if (!typeDefs.containsKey(summaryMetricName)) {
-                        fail("Metric " + metricName + " does not have a corresponding summary type definition");
-                    }
-                } else if (metricName.endsWith("_created")) {
-                    String summaryMetricName = metricName.substring(0, metricName.indexOf("_created"));
-                    if (!typeDefs.containsKey(summaryMetricName)) {
-                        fail("Metric " + metricName + " does not have a corresponding summary type definition");
-                    }
-                } else if (metricName.endsWith("_total")) {
-                    String summaryMetricName = metricName.substring(0, metricName.indexOf("_total"));
-                    if (!typeDefs.containsKey(summaryMetricName)) {
-                        fail("Metric " + metricName + " does not have a corresponding counter type definition");
-                    }
-                } else {
+                }
+                if (!isNorm){
                     fail("Metric " + metricName + " does not have a type definition");
                 }
 
@@ -867,6 +863,24 @@ public class PrometheusMetricsTest extends BrokerTestBase {
         p2.close();
     }
 
+    /***
+     * this method will return ["_sum", "_info", "_bucket", "_count", "_total", "_created", "_gsum", "_gcount"]
+     */
+    public static Set<String> allPrometheusSuffixEnums(){
+        HashSet<String> result = new HashSet<>();
+        final String metricsName = "123";
+        for (Collector.Type type : Collector.Type.values()){
+            Collector.MetricFamilySamples metricFamilySamples =
+                    new Collector.MetricFamilySamples(metricsName, type, "", new ArrayList<>());
+            result.addAll(Arrays.asList(metricFamilySamples.getNames()));
+        }
+        return result.stream()
+                .map(str -> str.substring(metricsName.length()))
+                .filter(str -> StringUtils.isNotBlank(str))
+                .collect(Collectors.toSet());
+    }
+
+
     @Test
     public void testManagedLedgerCacheStats() throws Exception {
         Producer<byte[]> p1 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionBatchWriterMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionBatchWriterMetricsTest.java
index 69521a5e646..b0b2fcce351 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionBatchWriterMetricsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionBatchWriterMetricsTest.java
@@ -41,6 +41,7 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
 import org.apache.pulsar.broker.resources.ClusterResources;
 import org.apache.pulsar.broker.resources.NamespaceResources;
 import org.apache.pulsar.broker.resources.TenantResources;
+import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStoreProvider;
 import org.apache.pulsar.client.api.Consumer;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.Producer;
@@ -56,6 +57,7 @@ import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
 import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.Policies;
 import org.apache.pulsar.common.policies.data.TenantInfoImpl;
+import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStoreProvider;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
@@ -75,6 +77,8 @@ public class TransactionBatchWriterMetricsTest extends MockedPulsarServiceBaseTe
 
     @BeforeClass
     public void setup() throws Exception {
+        MLTransactionMetadataStoreProvider.initBufferedWriterMetrics("localhost");
+        MLPendingAckStoreProvider.initBufferedWriterMetrics("localhost");
         super.internalSetup();
     }
 
@@ -130,7 +134,7 @@ public class TransactionBatchWriterMetricsTest extends MockedPulsarServiceBaseTe
 
         // verify tc.
         String metrics_key_txn_tc_record_count_sum =
-                "pulsar_txn_tc_bufferedwriter_batch_record_count_sum{cluster=\"%s\",broker=\"%s\"} ";
+                "pulsar_txn_tc_bufferedwriter_batch_records_sum{cluster=\"%s\",broker=\"%s\"} ";
         Assert.assertTrue(searchMetricsValue(metricsLines,
                 String.format(metrics_key_txn_tc_record_count_sum, metricsLabelCluster, metricsLabelBroker))
                 > 0);
@@ -146,7 +150,7 @@ public class TransactionBatchWriterMetricsTest extends MockedPulsarServiceBaseTe
                 > 0);
         // verify pending ack.
         String metrics_key_txn_pending_ack_record_count_sum =
-                "pulsar_txn_pending_ack_store_bufferedwriter_batch_record_count_sum{cluster=\"%s\",broker=\"%s\"} ";
+                "pulsar_txn_pending_ack_store_bufferedwriter_batch_records_sum{cluster=\"%s\",broker=\"%s\"} ";
         Assert.assertTrue(searchMetricsValue(metricsLines,
                 String.format(metrics_key_txn_pending_ack_record_count_sum, metricsLabelCluster, metricsLabelBroker))
                 > 0);
diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/TxnLogBufferedWriterMetricsStats.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/TxnLogBufferedWriterMetricsStats.java
index a0d4596b75d..f46cda89515 100644
--- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/TxnLogBufferedWriterMetricsStats.java
+++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/TxnLogBufferedWriterMetricsStats.java
@@ -103,7 +103,7 @@ public class TxnLogBufferedWriterMetricsStats implements Closeable {
         this.labelValues = labelValues.clone();
 
         String recordsPerBatchMetricName =
-                String.format("%s_bufferedwriter_batch_record_count", metricsPrefix);
+                String.format("%s_bufferedwriter_batch_records", metricsPrefix);
         recordsPerBatchMetric = new Histogram.Builder()
                         .name(recordsPerBatchMetricName)
                         .labelNames(this.labelNames)
@@ -122,7 +122,7 @@ public class TxnLogBufferedWriterMetricsStats implements Closeable {
         batchSizeBytesHistogram = batchSizeBytesMetric.labels(this.labelValues);
 
         String oldestRecordInBatchDelayTimeSecondsMetricName =
-                String.format("%s_bufferedwriter_batch_oldest_record_delay_time_second", metricsPrefix);
+                String.format("%s_bufferedwriter_batch_oldest_record_delay_seconds", metricsPrefix);
         oldestRecordInBatchDelayTimeSecondsMetric = new Histogram.Builder()
                         .name(oldestRecordInBatchDelayTimeSecondsMetricName)
                         .labelNames(this.labelNames)
diff --git a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/impl/TxnLogBufferedWriterTest.java b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/impl/TxnLogBufferedWriterTest.java
index a690d52266e..56fc9c9f2e3 100644
--- a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/impl/TxnLogBufferedWriterTest.java
+++ b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/impl/TxnLogBufferedWriterTest.java
@@ -997,17 +997,17 @@ public class TxnLogBufferedWriterTest extends MockedBookKeeperTestCase {
     private void verifyTheHistogramMetrics(int batchFlushCount, int totalRecordsCount, int totalSize){
         // Total flush count.
         assertEquals(
-                getHistogramCount(String.format("%s_bufferedwriter_batch_record_count", metricsPrefix)),
+                getHistogramCount(String.format("%s_bufferedwriter_batch_records", metricsPrefix)),
                 batchFlushCount);
         assertEquals(
                 getHistogramCount(String.format("%s_bufferedwriter_batch_size_bytes", metricsPrefix)),
                 batchFlushCount);
         assertEquals(
-                getHistogramCount(String.format("%s_bufferedwriter_batch_oldest_record_delay_time_second", metricsPrefix)),
+                getHistogramCount(String.format("%s_bufferedwriter_batch_oldest_record_delay_seconds", metricsPrefix)),
                 batchFlushCount);
         // Total records count.
         assertEquals(
-                getHistogramSum(String.format("%s_bufferedwriter_batch_record_count", metricsPrefix)),
+                getHistogramSum(String.format("%s_bufferedwriter_batch_records", metricsPrefix)),
                 totalRecordsCount);
         // Total data size.
         assertEquals(