You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ne...@apache.org on 2019/01/08 23:23:27 UTC
[incubator-pinot] 04/08: Review comments
This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch split_vm_tasks_2
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 0ff4723735ed8c8cd7146d69cf10a2c716b3cd89
Author: Neha Pawar <np...@linkedin.com>
AuthorDate: Mon Jan 7 10:49:35 2019 -0800
Review comments
---
.../linkedin/pinot/controller/ControllerConf.java | 31 +++++++++++++++++++++-
.../pinot/controller/ControllerStarter.java | 3 +--
.../BrokerResourceValidationManager.java | 1 -
.../validation/OfflineSegmentIntervalChecker.java | 5 +---
4 files changed, 32 insertions(+), 8 deletions(-)
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerConf.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerConf.java
index 1b496d9..9f84691 100644
--- a/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerConf.java
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerConf.java
@@ -57,6 +57,7 @@ public class ControllerConf extends PropertiesConfiguration {
public static class ControllerPeriodicTasksConf {
private static final String RETENTION_MANAGER_FREQUENCY_IN_SECONDS = "controller.retention.frequencyInSeconds";
+ private static final String VALIDATION_MANAGER_FREQUENCY_IN_SECONDS = "controller.validation.frequencyInSeconds";
private static final String OFFLINE_SEGMENT_INTERVAL_CHECKER_FREQUENCY_IN_SECONDS =
"controller.offline.segment.interval.checker.frequencyInSeconds";
private static final String REALTIME_SEGMENT_VALIDATION_FREQUENCY_IN_SECONDS =
@@ -75,6 +76,7 @@ public class ControllerConf extends PropertiesConfiguration {
"controller.segment.level.validation.intervalInSeconds";
private static final int DEFAULT_RETENTION_CONTROLLER_FREQUENCY_IN_SECONDS = 6 * 60 * 60; // 6 Hours.
+ private static final int DEFAULT_VALIDATION_CONTROLLER_FREQUENCY_IN_SECONDS = 60 * 60; // 1 Hour.
private static final int DEFAULT_OFFLINE_SEGMENT_INTERVAL_CHECKER_FREQUENCY_IN_SECONDS = 6 * 60 * 60; // 6 Hours.
private static final int DEFAULT_REALTIME_SEGMENT_VALIDATION_FREQUENCY_IN_SECONDS = 60 * 60; // 1 Hour.
private static final int DEFAULT_BROKER_RESOURCE_VALIDATION_FREQUENCY_IN_SECONDS = 60 * 60; // 1 Hour.
@@ -351,6 +353,28 @@ public class ControllerConf extends PropertiesConfiguration {
Integer.toString(retentionFrequencyInSeconds));
}
+ /**
+ * Deprecated. The validation manager has been split into 3 separate tasks, each having their own frequency config
+ * @return
+ */
+ @Deprecated
+ public int getValidationControllerFrequencyInSeconds() {
+ if (containsKey(ControllerPeriodicTasksConf.VALIDATION_MANAGER_FREQUENCY_IN_SECONDS)) {
+ return Integer.parseInt(
+ (String) getProperty(ControllerPeriodicTasksConf.VALIDATION_MANAGER_FREQUENCY_IN_SECONDS));
+ }
+ return ControllerPeriodicTasksConf.DEFAULT_VALIDATION_CONTROLLER_FREQUENCY_IN_SECONDS;
+ }
+
+ /**
+ * Deprecated. The validation manager has been split into 3 separate tasks, each having their own frequency config
+ * @return
+ */
+ public void setValidationControllerFrequencyInSeconds(int validationFrequencyInSeconds) {
+ setProperty(ControllerPeriodicTasksConf.VALIDATION_MANAGER_FREQUENCY_IN_SECONDS,
+ Integer.toString(validationFrequencyInSeconds));
+ }
+
public int getOfflineSegmentIntervalCheckerFrequencyInSeconds() {
if (containsKey(ControllerPeriodicTasksConf.OFFLINE_SEGMENT_INTERVAL_CHECKER_FREQUENCY_IN_SECONDS)) {
return Integer.parseInt(
@@ -378,12 +402,17 @@ public class ControllerConf extends PropertiesConfiguration {
Integer.toString(validationFrequencyInSeconds));
}
+ /**
+ * Return broker resource validation frequency if present, else return the validation manager frequency
+ * This is so that we can rollout with no config changes to the frequency of this task
+ * @return
+ */
public int getBrokerResourceValidationFrequencyInSeconds() {
if (containsKey(ControllerPeriodicTasksConf.BROKER_RESOURCE_VALIDATION_FREQUENCY_IN_SECONDS)) {
return Integer.parseInt(
(String) getProperty(ControllerPeriodicTasksConf.BROKER_RESOURCE_VALIDATION_FREQUENCY_IN_SECONDS));
}
- return ControllerPeriodicTasksConf.DEFAULT_BROKER_RESOURCE_VALIDATION_FREQUENCY_IN_SECONDS;
+ return getValidationControllerFrequencyInSeconds();
}
public void setBrokerResourceValidationFrequencyInSeconds(int validationFrequencyInSeconds) {
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerStarter.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerStarter.java
index 07ca195..2bca8ae 100644
--- a/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerStarter.java
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerStarter.java
@@ -193,8 +193,7 @@ public class ControllerStarter {
periodicTasks.add(_taskManager);
periodicTasks.add(_retentionManager);
_offlineSegmentIntervalChecker =
- new OfflineSegmentIntervalChecker(_config, _helixResourceManager, PinotLLCRealtimeSegmentManager.getInstance(),
- new ValidationMetrics(_metricsRegistry));
+ new OfflineSegmentIntervalChecker(_config, _helixResourceManager, new ValidationMetrics(_metricsRegistry));
_realtimeSegmentValidationManager =
new RealtimeSegmentValidationManager(_config, _helixResourceManager, PinotLLCRealtimeSegmentManager.getInstance(),
new ValidationMetrics(_metricsRegistry));
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/validation/BrokerResourceValidationManager.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/validation/BrokerResourceValidationManager.java
index e3264c4..1f09b64 100644
--- a/pinot-controller/src/main/java/com/linkedin/pinot/controller/validation/BrokerResourceValidationManager.java
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/validation/BrokerResourceValidationManager.java
@@ -41,7 +41,6 @@ public class BrokerResourceValidationManager extends ControllerPeriodicTask {
@Override
protected void preprocess() {
- // Cache instance configs to reduce ZK access
_instanceConfigs = _pinotHelixResourceManager.getAllHelixInstanceConfigs();
}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/validation/OfflineSegmentIntervalChecker.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/validation/OfflineSegmentIntervalChecker.java
index 592bd0a..3c79205 100644
--- a/pinot-controller/src/main/java/com/linkedin/pinot/controller/validation/OfflineSegmentIntervalChecker.java
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/validation/OfflineSegmentIntervalChecker.java
@@ -29,7 +29,6 @@ import com.linkedin.pinot.common.utils.time.TimeUtils;
import com.linkedin.pinot.controller.ControllerConf;
import com.linkedin.pinot.controller.helix.core.PinotHelixResourceManager;
import com.linkedin.pinot.controller.helix.core.periodictask.ControllerPeriodicTask;
-import com.linkedin.pinot.controller.helix.core.realtime.PinotLLCRealtimeSegmentManager;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
@@ -46,14 +45,12 @@ import org.slf4j.LoggerFactory;
public class OfflineSegmentIntervalChecker extends ControllerPeriodicTask {
private static final Logger LOGGER = LoggerFactory.getLogger(OfflineSegmentIntervalChecker.class);
- private final PinotLLCRealtimeSegmentManager _llcRealtimeSegmentManager;
private final ValidationMetrics _validationMetrics;
public OfflineSegmentIntervalChecker(ControllerConf config, PinotHelixResourceManager pinotHelixResourceManager,
- PinotLLCRealtimeSegmentManager llcRealtimeSegmentManager, ValidationMetrics validationMetrics) {
+ ValidationMetrics validationMetrics) {
super("OfflineSegmentIntervalChecker", config.getOfflineSegmentIntervalCheckerFrequencyInSeconds(),
pinotHelixResourceManager);
- _llcRealtimeSegmentManager = llcRealtimeSegmentManager;
_validationMetrics = validationMetrics;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org