You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by GitBox <gi...@apache.org> on 2021/06/26 00:31:12 UTC
[GitHub] [incubator-pinot] jackjlli commented on a change in pull request #7091: Added TaskMetricsEmitted periodic controller job
jackjlli commented on a change in pull request #7091:
URL: https://github.com/apache/incubator-pinot/pull/7091#discussion_r659090404
##########
File path: pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotHelixTaskResourceManager.java
##########
@@ -258,6 +262,50 @@ public synchronized String submitTask(List<PinotTaskConfig> pinotTaskConfigs, St
return taskStates;
}
+ /**
+ * This method returns a count of sub-tasks in various states, given the top-level task name.
+ * @param parentTaskName (e.g. "Task_TestTask_1624403781879")
+ * @return TaskCount object
+ */
+ public synchronized TaskCount getTaskCount(String parentTaskName) {
+ JobContext jobContext = _taskDriver.getJobContext(getHelixJobName(parentTaskName));
+ Set<Integer> partitionSet = jobContext.getPartitionSet();
+ TaskCount taskCount = new TaskCount();
+ taskCount.addToTotal(partitionSet.size());
+ for (int partition : partitionSet) {
+ TaskPartitionState state = jobContext.getPartitionState(partition);
+ // Helix returns state as null if the task is not enqueued anywhere yet
+ if (state == null) {
+ // task is not yet assigned to a participant
+ taskCount.addToWaiting(1);
+ } else if (state.equals(TaskPartitionState.INIT) || state.equals(TaskPartitionState.RUNNING)) {
+ taskCount.addToRunning(1);
+ } else if (state.equals(TaskPartitionState.TASK_ERROR)) {
+ taskCount.addToError(1);
+ }
+ }
+ return taskCount;
+ }
+
+ /**
+ * Returns a set of Task names (in the form "Task_TestTask_1624403781879") that are in progress or not started yet.
+ *
+ * @param taskType
+ * @return Set of task names
+ */
+ public synchronized Set<String> getTasksInProgress(String taskType) {
+ Set<String> tasksInProgress = new HashSet<>();
+ Map<String, TaskState> helixJobStates =
Review comment:
Same for the returned object here.
##########
File path: pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/TaskMetricsEmitter.java
##########
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.helix.core.minion;
+
+import java.util.Set;
+import org.apache.pinot.common.metrics.ControllerGauge;
+import org.apache.pinot.common.metrics.ControllerMetrics;
+import org.apache.pinot.controller.ControllerConf;
+import org.apache.pinot.controller.LeadControllerManager;
+import org.apache.pinot.controller.helix.core.minion.PinotHelixTaskResourceManager.TaskCount;
+import org.apache.pinot.core.periodictask.BasePeriodicTask;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class TaskMetricsEmitter extends BasePeriodicTask {
+ private static final Logger LOGGER = LoggerFactory.getLogger(TaskMetricsEmitter.class);
+ private final static String TASK_NAME = "TaskMetricsEmitter";
+
+ private final PinotHelixTaskResourceManager _helixTaskResourceManager;
+ private final ControllerMetrics _controllerMetrics;
+ private final LeadControllerManager _leadControllerManager;
+
+ public TaskMetricsEmitter(PinotHelixTaskResourceManager helixTaskResourceManager,
+ LeadControllerManager leadControllerManager, ControllerConf controllerConf, ControllerMetrics controllerMetrics) {
+ super(TASK_NAME, controllerConf.getTaskMetricsEmitterFrequencyInSeconds(),
+ controllerConf.getPeriodicTaskInitialDelayInSeconds());
+ _helixTaskResourceManager = helixTaskResourceManager;
+ _controllerMetrics = controllerMetrics;
+ _leadControllerManager = leadControllerManager;
+ }
+
+ @Override
+ protected final void runTask() {
+ // Make it so that only one controller returns the metric for all the tasks.
+ if (!_leadControllerManager.isLeaderForTable(TASK_NAME)) {
+ return;
+ }
+
+ // The call to get task types can take time if there are a lot of tasks.
Review comment:
It'd be good to mention the default frequency is 5 mins here.
##########
File path: pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/TaskMetricsEmitter.java
##########
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.helix.core.minion;
+
+import java.util.Set;
+import org.apache.pinot.common.metrics.ControllerGauge;
+import org.apache.pinot.common.metrics.ControllerMetrics;
+import org.apache.pinot.controller.ControllerConf;
+import org.apache.pinot.controller.LeadControllerManager;
+import org.apache.pinot.controller.helix.core.minion.PinotHelixTaskResourceManager.TaskCount;
+import org.apache.pinot.core.periodictask.BasePeriodicTask;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class TaskMetricsEmitter extends BasePeriodicTask {
Review comment:
It'd be good to add some javadoc here?
##########
File path: pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotHelixTaskResourceManager.java
##########
@@ -258,6 +262,50 @@ public synchronized String submitTask(List<PinotTaskConfig> pinotTaskConfigs, St
return taskStates;
}
+ /**
+ * This method returns a count of sub-tasks in various states, given the top-level task name.
+ * @param parentTaskName (e.g. "Task_TestTask_1624403781879")
+ * @return TaskCount object
+ */
+ public synchronized TaskCount getTaskCount(String parentTaskName) {
+ JobContext jobContext = _taskDriver.getJobContext(getHelixJobName(parentTaskName));
Review comment:
JobContext could be null according to this code from Helix:
```
protected static JobContext getJobContext(HelixPropertyStore<ZNRecord> propertyStore, String jobResource) {
ZNRecord r = (ZNRecord)propertyStore.get(Joiner.on("/").join("/TaskRebalancer", jobResource, new Object[]{"Context"}), (Stat)null, AccessOption.PERSISTENT);
return r != null ? new JobContext(r) : null;
}
```
It'd be good to handle the null case here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org