You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2021/01/28 21:47:45 UTC
[incubator-pinot] 01/01: Adding cron scheduler metrics reporting
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch adding_cron_scheduler_metrics
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 4492e901838d8c7ada3cd0982f3f6268bda120b6
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Thu Jan 28 12:35:22 2021 -0800
Adding cron scheduler metrics reporting
---
.../etc/jmx_prometheus_javaagent/configs/pinot.yml | 15 +++++++++++++++
.../apache/pinot/common/metrics/ControllerMeter.java | 4 +++-
.../apache/pinot/common/metrics/ControllerTimer.java | 2 +-
.../helix/core/minion/CronJobScheduleJob.java | 9 +++++++++
.../helix/core/minion/PinotTaskManager.java | 19 +++++++++++--------
.../core/periodictask/ControllerPeriodicTask.java | 4 ++++
6 files changed, 43 insertions(+), 10 deletions(-)
diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml
index 312512c..787d340 100644
--- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml
+++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml
@@ -52,6 +52,21 @@ rules:
name: "pinot_controller_validateion_$2_$3"
labels:
table: "$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\", name=\"pinot.controller.(\\w+)\\.(\\w+).cronSchedulerJobScheduled\"><>(\\w+)"
+ name: "pinot_controller_cronSchedulerJobScheduled_$3"
+ labels:
+ table: "$1"
+ taskType: "$2"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\", name=\"pinot.controller.(\\w+)\\.(\\w+).cronSchedulerTriggered\"><>(\\w+)"
+ name: "pinot_controller_cronSchedulerTriggered_$3"
+ labels:
+ table: "$1"
+ taskType: "$2"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\", name=\"pinot.controller.(\\w+)\\.(\\w+).cronSchedulerJobExecutionTimeMs\"><>(\\w+)"
+ name: "pinot_controller_cronSchedulerJobExecutionTimeMs_$3"
+ labels:
+ table: "$1"
+ taskType: "$2"
# Pinot Broker
- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(\\w+).authorization\"><>(\\w+)"
name: "pinot_broker_authorization_$2"
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerMeter.java b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerMeter.java
index 1e7fc46..0caf005 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerMeter.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerMeter.java
@@ -48,7 +48,9 @@ public enum ControllerMeter implements AbstractMetrics.Meter {
CONTROLLER_PERIODIC_TASK_ERROR("periodicTaskError", false),
NUMBER_TIMES_SCHEDULE_TASKS_CALLED("tasks", true),
NUMBER_TASKS_SUBMITTED("tasks", false),
- NUMBER_SEGMENT_UPLOAD_TIMEOUT_EXCEEDED("SegmentUploadTimeouts", true);
+ NUMBER_SEGMENT_UPLOAD_TIMEOUT_EXCEEDED("SegmentUploadTimeouts", true),
+ CRON_SCHEDULER_JOB_SCHEDULED("cronSchedulerJobScheduled", false),
+ CRON_SCHEDULER_JOB_TRIGGERED("cronSchedulerJobTriggered", false);
private final String brokerMeterName;
private final String unit;
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerTimer.java b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerTimer.java
index 7617f5a..40ad9d5 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerTimer.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerTimer.java
@@ -26,7 +26,7 @@ import org.apache.pinot.common.Utils;
*
*/
public enum ControllerTimer implements AbstractMetrics.Timer {
- ;
+ CRON_SCHEDULER_JOB_EXECUTION_TIME_MS("cronSchedulerJobExecutionTimeMs", false);
private final String timerName;
private final boolean global;
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/CronJobScheduleJob.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/CronJobScheduleJob.java
index e5653f4..553d326 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/CronJobScheduleJob.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/CronJobScheduleJob.java
@@ -18,6 +18,9 @@
*/
package org.apache.pinot.controller.helix.core.minion;
+import java.util.concurrent.TimeUnit;
+import org.apache.pinot.common.metrics.ControllerMeter;
+import org.apache.pinot.common.metrics.ControllerTimer;
import org.apache.pinot.controller.LeadControllerManager;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
@@ -42,11 +45,17 @@ public class CronJobScheduleJob implements Job {
.get(PinotTaskManager.LEAD_CONTROLLER_MANAGER_KEY);
String table = jobExecutionContext.getJobDetail().getKey().getName();
String taskType = jobExecutionContext.getJobDetail().getKey().getGroup();
+ pinotTaskManager.getControllerMetrics().addMeteredTableValue(PinotTaskManager.getCronJobName(table, taskType),
+ ControllerMeter.CRON_SCHEDULER_JOB_TRIGGERED, 1);
if (leadControllerManager.isLeaderForTable(table)) {
+ long jobStartTime = System.currentTimeMillis();
LOGGER.info("Execute CronJob: table - {}, task - {} at {}", table, taskType, jobExecutionContext.getFireTime());
pinotTaskManager.scheduleTask(taskType, table);
LOGGER.info("Finished CronJob: table - {}, task - {}, next runtime is {}", table, taskType,
jobExecutionContext.getNextFireTime());
+ pinotTaskManager.getControllerMetrics().addTimedTableValue(PinotTaskManager.getCronJobName(table, taskType),
+ ControllerTimer.CRON_SCHEDULER_JOB_EXECUTION_TIME_MS, (System.currentTimeMillis() - jobStartTime),
+ TimeUnit.MILLISECONDS);
} else {
LOGGER.info("Not Lead, skip processing CronJob: table - {}, task - {}", table, taskType);
}
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotTaskManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotTaskManager.java
index 436f10f..b5298e9 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotTaskManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotTaskManager.java
@@ -124,14 +124,6 @@ public class PinotTaskManager extends ControllerPeriodicTask<Void> {
return TABLE_CONFIG_PATH_PREFIX + tableWithType;
}
- protected synchronized void cleanupCronTaskScheduler() {
- try {
- _scheduledExecutorService.clear();
- } catch (SchedulerException e) {
- LOGGER.error("Failed to clear all tasks in scheduler", e);
- }
- }
-
public synchronized void cleanUpCronTaskSchedulerForTable(String tableWithType) {
LOGGER.info("Cleaning up task in scheduler for table {}", tableWithType);
TableTaskSchedulerUpdater tableTaskSchedulerUpdater = _tableTaskSchedulerUpdaterMap.get(tableWithType);
@@ -158,6 +150,8 @@ public class PinotTaskManager extends ControllerPeriodicTask<Void> {
if (jobKey.getName().equals(tableWithType)) {
try {
_scheduledExecutorService.deleteJob(jobKey);
+ _controllerMetrics.addMeteredTableValue(getCronJobName(tableWithType, jobKey.getGroup()),
+ ControllerMeter.CRON_SCHEDULER_JOB_SCHEDULED, -1L);
} catch (SchedulerException e) {
LOGGER.error("Got exception when deleting the scheduled job - {}", jobKey, e);
}
@@ -166,6 +160,10 @@ public class PinotTaskManager extends ControllerPeriodicTask<Void> {
_tableTaskTypeToCronExpressionMap.remove(tableWithType);
}
+ public static String getCronJobName(String tableWithType, String taskType) {
+ return String.format("%s.%s", tableWithType, taskType);
+ }
+
public synchronized void subscribeTableConfigChanges(String tableWithType) {
if (_tableTaskSchedulerUpdaterMap.containsKey(tableWithType)) {
return;
@@ -220,6 +218,8 @@ public class PinotTaskManager extends ControllerPeriodicTask<Void> {
if (!taskToCronExpressionMap.containsKey(existingTaskType)) {
try {
_scheduledExecutorService.deleteJob(JobKey.jobKey(tableWithType, existingTaskType));
+ _controllerMetrics.addMeteredTableValue(getCronJobName(tableWithType, existingTaskType),
+ ControllerMeter.CRON_SCHEDULER_JOB_SCHEDULED, -11L);
} catch (SchedulerException e) {
LOGGER.error("Failed to delete scheduled job for table {}, task type {}", tableWithType,
existingScheduledTasks, e);
@@ -290,6 +290,9 @@ public class PinotTaskManager extends ControllerPeriodicTask<Void> {
.build();
try {
_scheduledExecutorService.scheduleJob(jobDetail, trigger);
+ _controllerMetrics
+ .addMeteredTableValue(getCronJobName(tableWithType, taskType), ControllerMeter.CRON_SCHEDULER_JOB_SCHEDULED,
+ 1L);
} catch (Exception e) {
LOGGER.error("Failed to parse Cron expression - " + cronExprStr, e);
throw e;
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/periodictask/ControllerPeriodicTask.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/periodictask/ControllerPeriodicTask.java
index d2c2b13..9139f1d 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/periodictask/ControllerPeriodicTask.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/periodictask/ControllerPeriodicTask.java
@@ -72,6 +72,10 @@ public abstract class ControllerPeriodicTask<C> extends BasePeriodicTask {
}
}
+ public final ControllerMetrics getControllerMetrics() {
+ return _controllerMetrics;
+ }
+
/**
* Processes the given list of tables, and returns the number of tables processed.
* <p>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org