You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by jx...@apache.org on 2018/03/21 23:29:03 UTC

[2/3] helix git commit: Support Workflow level timeout

Support Workflow level timeout

Current Helix supports job level timeout, we believe it does make sense to support workflow level timeout, when users are confident with workflow running time. It could be used for following cases:
1. Prevent long run workflows stuck there forever.
2. If someone pause the workflow forget to resume back, workflow can be terminated with timeout.


Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/debb9bb2
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/debb9bb2
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/debb9bb2

Branch: refs/heads/master
Commit: debb9bb23125f4c073a3dba75bc16df3558e0c35
Parents: d742d09
Author: Junkai Xue <jx...@linkedin.com>
Authored: Thu Feb 8 17:15:41 2018 -0800
Committer: Junkai Xue <jx...@linkedin.com>
Committed: Tue Mar 20 11:55:34 2018 -0700

----------------------------------------------------------------------
 .../org/apache/helix/task/JobRebalancer.java    |  5 +-
 .../org/apache/helix/task/TaskRebalancer.java   | 12 ++-
 .../org/apache/helix/task/WorkflowConfig.java   | 39 ++++++++-
 .../apache/helix/task/WorkflowRebalancer.java   | 13 +++
 .../integration/task/TestWorkflowTimeout.java   | 84 ++++++++++++++++++++
 5 files changed, 143 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/helix/blob/debb9bb2/helix-core/src/main/java/org/apache/helix/task/JobRebalancer.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/task/JobRebalancer.java b/helix-core/src/main/java/org/apache/helix/task/JobRebalancer.java
index a855f7d..d00ffaf 100644
--- a/helix-core/src/main/java/org/apache/helix/task/JobRebalancer.java
+++ b/helix-core/src/main/java/org/apache/helix/task/JobRebalancer.java
@@ -213,9 +213,10 @@ public class JobRebalancer extends TaskRebalancer {
       ClusterDataCache cache) {
     TargetState jobTgtState = workflowConfig.getTargetState();
     TaskState jobState = workflowCtx.getJobState(jobResource);
+    TaskState workflowState = workflowCtx.getWorkflowState();
 
-    if (jobState == TaskState.IN_PROGRESS && isTimeout(jobCtx.getStartTime(),
-        jobCfg.getTimeout())) {
+    if (jobState == TaskState.IN_PROGRESS && (isTimeout(jobCtx.getStartTime(), jobCfg.getTimeout())
+        || TaskState.TIMED_OUT.equals(workflowState))) {
       jobState = TaskState.TIMING_OUT;
       workflowCtx.setJobState(jobResource, TaskState.TIMING_OUT);
     } else if (jobState != TaskState.TIMING_OUT && jobState != TaskState.FAILING) {

http://git-wip-us.apache.org/repos/asf/helix/blob/debb9bb2/helix-core/src/main/java/org/apache/helix/task/TaskRebalancer.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/task/TaskRebalancer.java b/helix-core/src/main/java/org/apache/helix/task/TaskRebalancer.java
index a5e5271..5e0fe01 100644
--- a/helix-core/src/main/java/org/apache/helix/task/TaskRebalancer.java
+++ b/helix-core/src/main/java/org/apache/helix/task/TaskRebalancer.java
@@ -72,11 +72,13 @@ public abstract class TaskRebalancer implements Rebalancer, MappingCalculator {
       Map<String, JobConfig> jobConfigMap) {
     boolean incomplete = false;
     int failedJobs = 0;
+    TaskState workflowState = ctx.getWorkflowState();
     for (String job : cfg.getJobDag().getAllNodes()) {
       TaskState jobState = ctx.getJobState(job);
       if (jobState == TaskState.FAILED || jobState == TaskState.TIMED_OUT) {
-        failedJobs ++;
-        if (!cfg.isJobQueue() && failedJobs > cfg.getFailureThreshold()) {
+        failedJobs++;
+        if (TaskState.TIMED_OUT.equals(workflowState) || (!cfg.isJobQueue() && failedJobs > cfg
+            .getFailureThreshold())) {
           ctx.setWorkflowState(TaskState.FAILED);
           if (_clusterStatusMonitor != null) {
             _clusterStatusMonitor.updateWorkflowCounters(cfg, TaskState.FAILED);
@@ -86,7 +88,8 @@ public abstract class TaskRebalancer implements Rebalancer, MappingCalculator {
               ctx.setJobState(jobToFail, TaskState.ABORTED);
               // Skip aborted jobs latency since they are not accurate latency for job running time
               if (_clusterStatusMonitor != null) {
-                _clusterStatusMonitor.updateJobCounters(jobConfigMap.get(jobToFail), TaskState.ABORTED);
+                _clusterStatusMonitor
+                    .updateJobCounters(jobConfigMap.get(jobToFail), TaskState.ABORTED);
               }
             }
           }
@@ -102,7 +105,8 @@ public abstract class TaskRebalancer implements Rebalancer, MappingCalculator {
     if (!incomplete && cfg.isTerminable()) {
       ctx.setWorkflowState(TaskState.COMPLETED);
       if (_clusterStatusMonitor != null) {
-        _clusterStatusMonitor.updateWorkflowCounters(cfg, TaskState.COMPLETED, ctx.getFinishTime() - ctx.getStartTime());
+        _clusterStatusMonitor.updateWorkflowCounters(cfg, TaskState.COMPLETED,
+            ctx.getFinishTime() - ctx.getStartTime());
       }
       return true;
     }

http://git-wip-us.apache.org/repos/asf/helix/blob/debb9bb2/helix-core/src/main/java/org/apache/helix/task/WorkflowConfig.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/task/WorkflowConfig.java b/helix-core/src/main/java/org/apache/helix/task/WorkflowConfig.java
index 80e03e5..f9016d1 100644
--- a/helix-core/src/main/java/org/apache/helix/task/WorkflowConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/task/WorkflowConfig.java
@@ -65,7 +65,8 @@ public class  WorkflowConfig extends ResourceConfig {
     IsJobQueue,
     JobPurgeInterval,
     /* Allow multiple jobs in this workflow to be assigned to a same instance or not */
-    AllowOverlapJobAssignment
+    AllowOverlapJobAssignment,
+    Timeout
     }
 
   /* Default values */
@@ -91,7 +92,7 @@ public class  WorkflowConfig extends ResourceConfig {
     this(workflowId, cfg.getJobDag(), cfg.getParallelJobs(), cfg.getTargetState(), cfg.getExpiry(),
         cfg.getFailureThreshold(), cfg.isTerminable(), cfg.getScheduleConfig(), cfg.getCapacity(),
         cfg.getWorkflowType(), cfg.isJobQueue(), cfg.getJobTypes(), cfg.getJobPurgeInterval(),
-        cfg.isAllowOverlapJobAssignment());
+        cfg.isAllowOverlapJobAssignment(), cfg.getTimeout());
   }
 
   /* Member variables */
@@ -100,7 +101,8 @@ public class  WorkflowConfig extends ResourceConfig {
   protected WorkflowConfig(String workflowId, JobDag jobDag, int parallelJobs,
       TargetState targetState, long expiry, int failureThreshold, boolean terminable,
       ScheduleConfig scheduleConfig, int capacity, String workflowType, boolean isJobQueue,
-      Map<String, String> jobTypes, long purgeInterval, boolean allowOverlapJobAssignment) {
+      Map<String, String> jobTypes, long purgeInterval, boolean allowOverlapJobAssignment,
+      long timeout) {
     super(workflowId);
 
     putSimpleConfig(WorkflowConfigProperty.WorkflowID.name(), workflowId);
@@ -123,6 +125,10 @@ public class  WorkflowConfig extends ResourceConfig {
       putSimpleConfig(WorkflowConfigProperty.capacity.name(), String.valueOf(capacity));
     }
 
+    if (timeout != TaskConstants.DEFAULT_NEVER_TIMEOUT) {
+      putSimpleConfig(WorkflowConfigProperty.Timeout.name(), String.valueOf(timeout));
+    }
+
     // Populate schedule if present
     if (scheduleConfig != null) {
       Date startTime = scheduleConfig.getStartTime();
@@ -242,6 +248,11 @@ public class  WorkflowConfig extends ResourceConfig {
         DEFAULT_ALLOW_OVERLAP_JOB_ASSIGNMENT);
   }
 
+  public long getTimeout() {
+    return _record
+        .getLongField(WorkflowConfigProperty.Timeout.name(), TaskConstants.DEFAULT_NEVER_TIMEOUT);
+  }
+
   public static SimpleDateFormat getDefaultDateFormat() {
     SimpleDateFormat defaultDateFormat = new SimpleDateFormat(
         "MM-dd-yyyy HH:mm:ss");
@@ -329,13 +340,14 @@ public class  WorkflowConfig extends ResourceConfig {
     private Map<String, String> _jobTypes;
     private long _jobPurgeInterval = DEFAULT_JOB_PURGE_INTERVAL;
     private boolean _allowOverlapJobAssignment = DEFAULT_ALLOW_OVERLAP_JOB_ASSIGNMENT;
+    private long _timeout = TaskConstants.DEFAULT_NEVER_TIMEOUT;
 
     public WorkflowConfig build() {
       validate();
 
       return new WorkflowConfig(_workflowId, _taskDag, _parallelJobs, _targetState, _expiry,
           _failureThreshold, _isTerminable, _scheduleConfig, _capacity, _workflowType, _isJobQueue,
-          _jobTypes, _jobPurgeInterval, _allowOverlapJobAssignment);
+          _jobTypes, _jobPurgeInterval, _allowOverlapJobAssignment, _timeout);
     }
 
     public Builder() {}
@@ -359,6 +371,7 @@ public class  WorkflowConfig extends ResourceConfig {
       _jobTypes = workflowConfig.getJobTypes();
       _jobPurgeInterval = workflowConfig.getJobPurgeInterval();
       _allowOverlapJobAssignment = workflowConfig.isAllowOverlapJobAssignment();
+      _timeout = workflowConfig.getTimeout();
     }
 
     public Builder setWorkflowId(String v) {
@@ -469,6 +482,16 @@ public class  WorkflowConfig extends ResourceConfig {
     }
 
     /**
+     * ONLY generic workflow can be timeouted. JobQueue does not allow to be timeouted.
+     * @param timeout  The timeout period
+     * @return
+     */
+    public Builder setTimeout(long timeout) {
+      _timeout = timeout;
+      return this;
+    }
+
+    /**
      * Set if allow multiple jobs in one workflow to be assigned on one instance.
      * If not set, default configuration is false.
      * @param allowOverlapJobAssignment true if allow overlap assignment
@@ -559,6 +582,10 @@ public class  WorkflowConfig extends ResourceConfig {
             Boolean.parseBoolean(cfg.get(WorkflowConfigProperty.AllowOverlapJobAssignment.name())));
       }
 
+      if (cfg.containsKey(WorkflowConfigProperty.Timeout.name())) {
+        setTimeout(Long.parseLong(cfg.get(WorkflowConfigProperty.Timeout.name())));
+      }
+
       return this;
     }
 
@@ -598,6 +625,10 @@ public class  WorkflowConfig extends ResourceConfig {
       return _isJobQueue;
     }
 
+    public long getTimeout() {
+      return _timeout;
+    }
+
     public static Builder from(WorkflowBean workflowBean) {
       WorkflowConfig.Builder b = new WorkflowConfig.Builder();
       if (workflowBean.schedule != null) {

http://git-wip-us.apache.org/repos/asf/helix/blob/debb9bb2/helix-core/src/main/java/org/apache/helix/task/WorkflowRebalancer.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/task/WorkflowRebalancer.java b/helix-core/src/main/java/org/apache/helix/task/WorkflowRebalancer.java
index 9e8d4c8..e3214f8 100644
--- a/helix-core/src/main/java/org/apache/helix/task/WorkflowRebalancer.java
+++ b/helix-core/src/main/java/org/apache/helix/task/WorkflowRebalancer.java
@@ -76,6 +76,19 @@ public class WorkflowRebalancer extends TaskRebalancer {
       LOG.debug("Workflow context is created for " + workflow);
     }
 
+    // Only generic workflow get timeouted and schedule rebalance for timeout. Will skip the set if
+    // the workflow already got timeouted. Job Queue will ignore the setup.
+    if (!workflowCfg.isJobQueue() && !TaskState.TIMED_OUT.equals(workflowCtx.getWorkflowState())) {
+      // If timeout point has already been passed, it will not be scheduled
+      scheduleRebalanceForTimeout(workflow, workflowCtx.getStartTime(), workflowCfg.getTimeout());
+
+      if (isTimeout(workflowCtx.getStartTime(), workflowCfg.getTimeout())) {
+        workflowCtx.setWorkflowState(TaskState.TIMED_OUT);
+        clusterData.updateWorkflowContext(workflow, workflowCtx, _manager.getHelixDataAccessor());
+        return buildEmptyAssignment(workflow, currStateOutput);
+      }
+    }
+
     // Clean up if workflow marked for deletion
     TargetState targetState = workflowCfg.getTargetState();
     if (targetState == TargetState.DELETE) {

http://git-wip-us.apache.org/repos/asf/helix/blob/debb9bb2/helix-core/src/test/java/org/apache/helix/integration/task/TestWorkflowTimeout.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/task/TestWorkflowTimeout.java b/helix-core/src/test/java/org/apache/helix/integration/task/TestWorkflowTimeout.java
new file mode 100644
index 0000000..c1f7060
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/integration/task/TestWorkflowTimeout.java
@@ -0,0 +1,84 @@
+package org.apache.helix.integration.task;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
+import java.util.Collections;
+import org.apache.helix.TestHelper;
+import org.apache.helix.model.MasterSlaveSMD;
+import org.apache.helix.task.JobConfig;
+import org.apache.helix.task.JobQueue;
+import org.apache.helix.task.TaskDriver;
+import org.apache.helix.task.TaskState;
+import org.apache.helix.task.TaskUtil;
+import org.apache.helix.task.Workflow;
+import org.apache.helix.task.WorkflowConfig;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+public class TestWorkflowTimeout extends TaskTestBase {
+  private final static String JOB_NAME = "TestJob";
+  private JobConfig.Builder _jobBuilder;
+
+  @BeforeClass
+  public void beforeClass() throws Exception {
+    _numDbs = 1;
+    _numNodes = 3;
+    _numParitions = 5;
+    _numReplicas = 3;
+    super.beforeClass();
+
+    // Create a non-stop job
+    _jobBuilder = new JobConfig.Builder()
+        .setTargetResource(WorkflowGenerator.DEFAULT_TGT_DB)
+        .setTargetPartitionStates(Sets.newHashSet(MasterSlaveSMD.States.MASTER.name()))
+        .setCommand(MockTask.TASK_COMMAND)
+        .setJobCommandConfigMap(ImmutableMap.of(MockTask.TIMEOUT_CONFIG, "99999999"));
+  }
+
+  @Test
+  public void testWorkflowRunningTime() throws InterruptedException {
+    String workflowName = TestHelper.getTestMethodName();
+    _jobBuilder.setWorkflow(workflowName);
+    Workflow.Builder workflowBuilder = new Workflow.Builder(workflowName)
+        .setWorkflowConfig(new WorkflowConfig.Builder(workflowName).setTimeout(1000).build())
+        .addJob(JOB_NAME, _jobBuilder);
+    _driver.start(workflowBuilder.build());
+
+    _driver.pollForWorkflowState(workflowName, 10000L, TaskState.TIMED_OUT);
+  }
+
+  @Test
+  public void testWorkflowPausedTimeout() throws InterruptedException {
+    String workflowName = TestHelper.getTestMethodName();
+    _jobBuilder.setWorkflow(workflowName);
+    Workflow.Builder workflowBuilder = new Workflow.Builder(workflowName)
+        .setWorkflowConfig(new WorkflowConfig.Builder(workflowName).setTimeout(5000).build())
+        .addJob(JOB_NAME, _jobBuilder);
+
+    _driver.start(workflowBuilder.build());
+    // Pause the queue
+    _driver.waitToStop(workflowName, 10000L);
+
+    _driver.pollForWorkflowState(workflowName, 10000L, TaskState.TIMED_OUT);
+  }
+
+  @Test
+  public void testJobQueueNotApplyTimeout() throws InterruptedException {
+    String queueName = TestHelper.getTestMethodName();
+    // Make jobs run success
+    _jobBuilder.setWorkflow(queueName).setJobCommandConfigMap(Collections.EMPTY_MAP);
+    JobQueue.Builder jobQueue = TaskTestUtil.buildJobQueue(queueName);
+    jobQueue.setWorkflowConfig(new WorkflowConfig.Builder(queueName).setTimeout(1000).build())
+        .enqueueJob(JOB_NAME, _jobBuilder).enqueueJob(JOB_NAME + 1, _jobBuilder);
+
+    _driver.start(jobQueue.build());
+
+    _driver.pollForJobState(queueName, TaskUtil.getNamespacedJobName(queueName, JOB_NAME),
+        TaskState.COMPLETED);
+    _driver.pollForJobState(queueName, TaskUtil.getNamespacedJobName(queueName, JOB_NAME + 1),
+        TaskState.COMPLETED);
+
+    // Add back the config
+    _jobBuilder.setJobCommandConfigMap(ImmutableMap.of(MockTask.TIMEOUT_CONFIG, "99999999"));
+  }
+}