You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ma...@apache.org on 2021/06/22 22:24:57 UTC
[incubator-pinot] branch master updated: Adding Additional JMX
Metrics to Prometheus config (#7079)
This is an automated email from the ASF dual-hosted git repository.
mayanks pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 709abb0 Adding Additional JMX Metrics to Prometheus config (#7079)
709abb0 is described below
commit 709abb01dabe8487856c1c2a2cffb01795f2a00c
Author: Tim Santos <ti...@cortexdata.io>
AuthorDate: Tue Jun 22 15:24:45 2021 -0700
Adding Additional JMX Metrics to Prometheus config (#7079)
* Adding prometheus configs in progress
* Adding additional metrics to Prometheus config
* Adding README and removing redundant metrics in config.
---
.../pinot/etc/jmx_prometheus_javaagent/README.md | 22 +++++++
.../etc/jmx_prometheus_javaagent/configs/pinot.yml | 72 +++++++++++++++++++++-
2 files changed, 93 insertions(+), 1 deletion(-)
diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/README.md b/docker/images/pinot/etc/jmx_prometheus_javaagent/README.md
new file mode 100644
index 0000000..18708bd
--- /dev/null
+++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/README.md
@@ -0,0 +1,22 @@
+#### What is Prometheus?
+
+Prometheus is an open-source event monitoring system. It records real-time metrics in a time series database and can be queried with its own query language PromQL. Prometheus has 4 metric types (Gauge, Counter, Histogram, Summary).
+
+Prometheus is not a full-fledged dashboarding solution and needs to be hooked up with Grafana to generate dashboards.
+
+#### How do Pinot metrics end up in Prometheus?
+
+Currently, Pinot metrics are exposed as JMX mbeans through the PinotJmxReporter. These JMX mbeans are consumed by Prometheus using the [Prometheus JMX Exporter](https://github.com/prometheus/jmx_exporter). A fairly comprehensive Prometheus JMX Exporter config can be found under `docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml`.
+
+See the [Pinot docs](https://docs.pinot.apache.org/operators/operating-pinot/monitoring) for more info.
+
+#### How can I view and test metrics?
+
+First, you need to make sure the metrics are exposed though JMX. Note, that if no metric has been published (e.g. no values recorded), the metric will not show up in JMX or Prometheus server.
+With a local Pinot deployment, you can launch `jconsole`, select your local deployment and view all the metrics exposed as jmx mbeans. To see if the metrics are being consumed with the Prometheus JMX Exporter and your config file, you can set the JAVA_OPTS env variable before running Pinot locally.
+
+`export JAVA_OPTS="-javaagent:jmx_prometheus_javaagent-0.12.0.jar=8080:pinot.yml -Xms4G -Xmx4G -XX:MaxDirectMemorySize=30g -Dlog4j2.configurationFile=conf/pinot-admin-log4j2.xml -Dplugins.dir=$BASEDIR/plugins"
+bin/pinot-admin.sh ....
+`
+
+This will expose a port at 8080 to dump metrics as Prometheus format for Prometheus scraper to fetch.
diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml
index 9b38cc9..8850d67 100644
--- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml
+++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml
@@ -77,7 +77,29 @@ rules:
labels:
taskType: "$1"
status: "$2"
-# Pinot Broker
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\", name=\"pinot.controller.pinotLeadControllerResourceEnabled\"><>(\\w+)"
+ name: "pinot_controller_pinotLeadControllerResourceEnabled_$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\", name=\"pinot.controller.offlineTableEstimatedSize.(\\w+)\"><>(\\w+)"
+ name: "pinot_controller_offlineTableEstimatedSize_$2"
+ labels:
+ table: "$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\", name=\"pinot.controller.tableQuota.(\\w+)_(\\w+)\"><>(\\w+)"
+ name: "pinot_controller_tableQuota_$3"
+ labels:
+ table: "$1"
+ tableType: "$2"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\", name=\"pinot.controller.tableStorageQuotaUtilization.(\\w+)_(\\w+)\"><>(\\w+)"
+ name: "pinot_controller_tableStorageQuotaUtilization_$3"
+ labels:
+ table: "$1"
+ tableType: "$2"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\", name=\"pinot.controller.tableStorageEstMissingSegmentPercent.(\\w+)_(\\w+)\"><>(\\w+)"
+ name: "pinot_controller_tableStorageEstMissingSegmentPercent_$3"
+ labels:
+ table: "$1"
+ tableType: "$2"
+
+ # Pinot Broker
- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(\\w+).authorization\"><>(\\w+)"
name: "pinot_broker_authorization_$2"
labels:
@@ -152,6 +174,36 @@ rules:
name: "pinot_broker_exceptions_$1_$2"
- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.routingTableUpdateTime\"><>(\\w+)"
name: "pinot_broker_routingTableUpdateTime_$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(\\w+)\\.brokerResponsesWithPartialServersResponded\"><>(\\w+)"
+ name: "pinot_broker_brokerResponsesWithPartialServersResponded_$2"
+ labels:
+ table: "$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(\\w+)\\.brokerResponsesWithNumGroupsLimitReached\"><>(\\w+)"
+ name: "pinot_broker_brokerResponsesWithNumGroupsLimitReached_$2"
+ labels:
+ table: "$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(\\w+)\\.requestConnectionTimeouts\"><>(\\w+)"
+ name: "pinot_broker_requestConnectionTimeouts_$2"
+ labels:
+ table: "$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(\\w+)\\.queryQuotaExceeded\"><>(\\w+)"
+ name: "pinot_broker_queryQuotaExceeded_$2"
+ labels:
+ table: "$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(\\w+)_(\\w+).serverMissingForRouting\"><>(\\w+)"
+ name: "pinot_broker_serverMissingForRouting_$3"
+ labels:
+ table: "$1"
+ tableType: "$2"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(\\w+)\\.deserialization\"><>(\\w+)"
+ name: "pinot_broker_deserialization_$2"
+ labels:
+ table: "$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(\\w+)\\.requestConnectionWait\"><>(\\w+)"
+ name: "pinot_broker_requestConnectionWait_$2"
+ labels:
+ table: "$1"
+
# Pinot Server
- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ServerMetrics\", name=\"pinot.server.documentCount.(\\w+)_(\\w+)\"><>(\\w+)"
name: "pinot_server_documentCount_$3"
@@ -231,6 +283,24 @@ rules:
name: "pinot_server_netty_tcp_$2_$3"
labels:
id: "$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ServerMetrics\", name=\"pinot.server.nettyConnection(\\w+)\"><>(\\w+)"
+ name: "pinot_server_nettyConnection_$1_$2"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ServerMetrics\", name=\"pinot.server.realtimeSegmentNumPartitions.(\\w+)\"><>(\\w+)"
+ name: "pinot_server_realtimeSegmentNumPartitions_$2"
+ labels:
+ table: "$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ServerMetrics\", name=\"pinot.server.resizeTimeMs.(\\w+)_(\\w+)\"><>(\\w+)"
+ name: "pinot_server_resizeTimeMs_$3"
+ labels:
+ table: "$1"
+ tableType: "$2"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ServerMetrics\", name=\"pinot.server.upsertPrimaryKeysCount.(\\w+)_(\\w+).(\\w+)\"><>(\\w+)"
+ name: "pinot_server_upsertPrimaryKeysCount_$4"
+ labels:
+ table: "$1"
+ tableType: "$2"
+ partition: "$3"
+
# Pinot Minions
- pattern: "\"org.apache.pinot.minion.metrics\"<type=\"MinionMetrics\", name=\"pinot.minion.(\\w+)\"><>(\\w+)"
name: "pinot_minion_$1_$2"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org