You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2019/02/19 23:20:35 UTC
[incubator-pinot] branch master updated: When counting running
segments for tasks, only consider tasks scheduled in one day (#3848)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 6adfa98 When counting running segments for tasks, only consider tasks scheduled in one day (#3848)
6adfa98 is described below
commit 6adfa98c8d1b4cd6c675ea3627e7f16f304ab34b
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Tue Feb 19 15:20:30 2019 -0800
When counting running segments for tasks, only consider tasks scheduled in one day (#3848)
Sometimes the task might stuck for long time, and in that case we want to re-schedule the segments
---
.../core/minion/generator/TaskGeneratorUtils.java | 30 ++++++++++++++++------
1 file changed, 22 insertions(+), 8 deletions(-)
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/generator/TaskGeneratorUtils.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/generator/TaskGeneratorUtils.java
index b3d9398..43dfe73 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/generator/TaskGeneratorUtils.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/generator/TaskGeneratorUtils.java
@@ -26,13 +26,17 @@ import org.apache.helix.task.TaskState;
import org.apache.pinot.common.config.PinotTaskConfig;
import org.apache.pinot.common.data.Segment;
import org.apache.pinot.controller.helix.core.minion.ClusterInfoProvider;
+import org.apache.pinot.controller.helix.core.minion.PinotHelixTaskResourceManager;
import org.apache.pinot.core.common.MinionConstants;
public class TaskGeneratorUtils {
+ private static final long ONE_DAY_IN_MILLIS = 24 * 60 * 60 * 1000L;
/**
- * Returns all the segments that have been scheduled but not finished
+ * Returns all the segments that have been scheduled in one day but not finished.
+ * <p>
+ * NOTE: we consider tasks not finished in one day as stuck and don't count the segments in them
*
* @param taskType Task type
* @param clusterInfoProvider Cluster info provider
@@ -43,13 +47,23 @@ public class TaskGeneratorUtils {
Set<Segment> runningSegments = new HashSet<>();
Map<String, TaskState> taskStates = clusterInfoProvider.getTaskStates(taskType);
for (Map.Entry<String, TaskState> entry : taskStates.entrySet()) {
- TaskState taskState = entry.getValue();
- if (taskState == TaskState.NOT_STARTED || taskState == TaskState.IN_PROGRESS || taskState == TaskState.STOPPED) {
- for (PinotTaskConfig pinotTaskConfig : clusterInfoProvider.getTaskConfigs(entry.getKey())) {
- Map<String, String> configs = pinotTaskConfig.getConfigs();
- runningSegments.add(
- new Segment(configs.get(MinionConstants.TABLE_NAME_KEY), configs.get(MinionConstants.SEGMENT_NAME_KEY)));
- }
+ // Skip COMPLETED tasks
+ if (entry.getValue() == TaskState.COMPLETED) {
+ continue;
+ }
+
+ // Skip tasks scheduled for more than one day
+ String taskName = entry.getKey();
+ long scheduleTimeMs = Long.parseLong(
+ taskName.substring(taskName.lastIndexOf(PinotHelixTaskResourceManager.TASK_NAME_SEPARATOR) + 1));
+ if (System.currentTimeMillis() - scheduleTimeMs > ONE_DAY_IN_MILLIS) {
+ continue;
+ }
+
+ for (PinotTaskConfig pinotTaskConfig : clusterInfoProvider.getTaskConfigs(entry.getKey())) {
+ Map<String, String> configs = pinotTaskConfig.getConfigs();
+ runningSegments.add(
+ new Segment(configs.get(MinionConstants.TABLE_NAME_KEY), configs.get(MinionConstants.SEGMENT_NAME_KEY)));
}
}
return runningSegments;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org