You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by pm...@apache.org on 2018/02/16 18:34:25 UTC
samza git commit: SAMZA-1588: Add random jitter to monitor’s scheduling interval.
Repository: samza
Updated Branches:
refs/heads/master c3e6b5517 -> 718c66612
SAMZA-1588: Add random jitter to monitor’s scheduling interval.
We’ve observed in LinkedIn execution environments that, all the monitors running on the YARN node-manager machines hitting an external service at the same time based upon the configured monitor scheduling interval.
To eliminate unnecessary monitor execution spike and congestion caused to an external service at the same time, it’s essential to add a random jitter to the monitor scheduling interval.
Random jitter will be added to monitor scheduling interval based upon a boolean configuration.
Author: Shanthoosh Venkataraman <sv...@linkedin.com>
Reviewers: Prateek Maheshwari <pm...@apache.org>
Closes #422 from shanthoosh/SAMZA-1588
Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/718c6661
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/718c6661
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/718c6661
Branch: refs/heads/master
Commit: 718c666127624f959a711e08221d8b8c389720a2
Parents: c3e6b55
Author: Shanthoosh Venkataraman <sv...@linkedin.com>
Authored: Fri Feb 16 10:34:21 2018 -0800
Committer: Prateek Maheshwari <pm...@linkedin.com>
Committed: Fri Feb 16 10:34:21 2018 -0800
----------------------------------------------------------------------
docs/learn/documentation/versioned/rest/monitors.md | 8 ++++++++
.../main/java/org/apache/samza/monitor/MonitorConfig.java | 8 ++++++++
.../java/org/apache/samza/monitor/SamzaMonitorService.java | 8 +++++---
3 files changed, 21 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/samza/blob/718c6661/docs/learn/documentation/versioned/rest/monitors.md
----------------------------------------------------------------------
diff --git a/docs/learn/documentation/versioned/rest/monitors.md b/docs/learn/documentation/versioned/rest/monitors.md
index 52fd27e..7946968 100644
--- a/docs/learn/documentation/versioned/rest/monitors.md
+++ b/docs/learn/documentation/versioned/rest/monitors.md
@@ -47,6 +47,14 @@ The following configurations are required for each of the monitors.
not defined, it is defaulted to 60 seconds.</td>
</tr>
<tr>
+ <td>monitor.monitorName.scheduling.jitter.percent</td>
+ <td></td>
+ <td>
+ Defines the random jitter percentage that should be added to the monitor
+ scheduling interval for a monitor named monitorName. If undefined,
+ it is defaulted to zero.</td>
+ </tr>
+ <tr>
<td>monitor.monitorName.factory.class</td>
<td></td>
<td>
http://git-wip-us.apache.org/repos/asf/samza/blob/718c6661/samza-rest/src/main/java/org/apache/samza/monitor/MonitorConfig.java
----------------------------------------------------------------------
diff --git a/samza-rest/src/main/java/org/apache/samza/monitor/MonitorConfig.java b/samza-rest/src/main/java/org/apache/samza/monitor/MonitorConfig.java
index 6743291..ced6fc1 100644
--- a/samza-rest/src/main/java/org/apache/samza/monitor/MonitorConfig.java
+++ b/samza-rest/src/main/java/org/apache/samza/monitor/MonitorConfig.java
@@ -42,6 +42,10 @@ public class MonitorConfig extends MapConfig {
private static final String MONITOR_PREFIX = String.format("monitor%s", MONITOR_CONFIG_KEY_SEPARATOR);
+ private static final String CONFIG_SCHEDULING_JITTER_PERCENT = "scheduling.jitter.percent";
+
+ private static final int DEFAULT_SCHEDULING_JITTER_PERCENT = 0;
+
public MonitorConfig(Config config) {
super(config);
}
@@ -96,4 +100,8 @@ public class MonitorConfig extends MapConfig {
public int getSchedulingIntervalInMs() {
return getInt(CONFIG_SCHEDULING_INTERVAL, DEFAULT_SCHEDULING_INTERVAL_IN_MS);
}
+
+ public double getSchedulingJitterPercent() {
+ return getDouble(CONFIG_SCHEDULING_JITTER_PERCENT, DEFAULT_SCHEDULING_JITTER_PERCENT);
+ }
}
http://git-wip-us.apache.org/repos/asf/samza/blob/718c6661/samza-rest/src/main/java/org/apache/samza/monitor/SamzaMonitorService.java
----------------------------------------------------------------------
diff --git a/samza-rest/src/main/java/org/apache/samza/monitor/SamzaMonitorService.java b/samza-rest/src/main/java/org/apache/samza/monitor/SamzaMonitorService.java
index 85491ad..9911d93 100644
--- a/samza-rest/src/main/java/org/apache/samza/monitor/SamzaMonitorService.java
+++ b/samza-rest/src/main/java/org/apache/samza/monitor/SamzaMonitorService.java
@@ -19,6 +19,7 @@
package org.apache.samza.monitor;
import com.google.common.base.Strings;
+import java.security.SecureRandom;
import java.util.Map;
import org.apache.samza.SamzaException;
import org.apache.samza.metrics.MetricsRegistry;
@@ -40,6 +41,7 @@ import static org.apache.samza.monitor.MonitorLoader.instantiateMonitor;
public class SamzaMonitorService {
private static final Logger LOGGER = LoggerFactory.getLogger(SamzaMonitorService.class);
+ private static final SecureRandom RANDOM = new SecureRandom();
private final SchedulingProvider scheduler;
private final SamzaRestConfig config;
@@ -62,9 +64,9 @@ public class SamzaMonitorService {
if (!Strings.isNullOrEmpty(monitorConfig.getMonitorFactoryClass())) {
int schedulingIntervalInMs = monitorConfig.getSchedulingIntervalInMs();
- LOGGER.info("Scheduling monitor {} to run every {} ms", monitorName, schedulingIntervalInMs);
- // MetricsRegistry has been added in the Monitor interface, since it's required in the eventual future to record metrics.
- // We have plans to record metrics, hence adding this as a placeholder. We just aren't doing it yet.
+ int monitorSchedulingJitterInMs = (int) (RANDOM.nextInt(schedulingIntervalInMs + 1) * (monitorConfig.getSchedulingJitterPercent() / 100.0));
+ schedulingIntervalInMs += monitorSchedulingJitterInMs;
+ LOGGER.info("Scheduling the monitor: {} to run every {} ms.", monitorName, schedulingIntervalInMs);
scheduler.schedule(getRunnable(instantiateMonitor(monitorName, monitorConfig, metricsRegistry)),
schedulingIntervalInMs);
} else {