You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by jx...@apache.org on 2019/02/26 00:30:37 UTC

[helix] 02/02: Implement Job looping strategy

This is an automated email from the ASF dual-hosted git repository.

jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git

commit af3fb00093df9311d63e62f7f85561de906ded1b
Author: Junkai Xue <jx...@linkedin.com>
AuthorDate: Tue Oct 23 11:30:33 2018 -0700

    Implement Job looping strategy
    
    Last change already made the task scheduling by looping all the workflows. In that case, we still need to integrate the job looping strategy with new workflow looping strategy. With this change, the quota restriction can be applied.
---
 .../apache/helix/common/caches/TaskDataCache.java  | 43 +++++++++--
 .../helix/controller/stages/ClusterDataCache.java  | 13 ++++
 .../stages/task/TaskSchedulingStage.java           | 45 +++++++-----
 .../apache/helix/task/AbstractTaskDispatcher.java  | 12 ++++
 .../java/org/apache/helix/task/JobDispatcher.java  | 16 +----
 .../java/org/apache/helix/task/JobRebalancer.java  |  5 +-
 .../org/apache/helix/task/WorkflowDispatcher.java  | 66 ++++++++++++-----
 .../org/apache/helix/task/WorkflowRebalancer.java  |  7 +-
 .../helix/integration/task/TestDeleteWorkflow.java |  2 +-
 .../task/TestIndependentTaskRebalancer.java        |  2 +-
 .../mbeans/TestClusterAggregateMetrics.java        |  6 +-
 .../apache/helix/task/TestJobStateOnCreation.java  | 84 ----------------------
 12 files changed, 152 insertions(+), 149 deletions(-)

diff --git a/helix-core/src/main/java/org/apache/helix/common/caches/TaskDataCache.java b/helix-core/src/main/java/org/apache/helix/common/caches/TaskDataCache.java
index 285fbe9..95ee62d 100644
--- a/helix-core/src/main/java/org/apache/helix/common/caches/TaskDataCache.java
+++ b/helix-core/src/main/java/org/apache/helix/common/caches/TaskDataCache.java
@@ -56,10 +56,15 @@ public class TaskDataCache extends AbstractDataCache {
   private Set<String> _contextToUpdate = new HashSet<>();
   private Set<String> _contextToRemove = new HashSet<>();
   // The following fields have been added for quota-based task scheduling
-  private final AssignableInstanceManager _assignableInstanceManager = new AssignableInstanceManager();
+  private final AssignableInstanceManager _assignableInstanceManager =
+      new AssignableInstanceManager();
+  // Current usage for this scheduled jobs is used for differentiate the jobs has been processed in
+  // JobDispatcher from RESOURCE_TO_BALANCE to reduce the redundant computation.
+  private Set<String> _dispatchedJobs = new HashSet<>();
 
   /**
    * Original constructor for TaskDataCache.
+   *
    * @param clusterName
    */
   public TaskDataCache(String clusterName) {
@@ -68,7 +73,9 @@ public class TaskDataCache extends AbstractDataCache {
 
   /**
    * This refreshes the cluster data by re-fetching the data from zookeeper in an efficient way
+   *
    * @param accessor
+   *
    * @return
    */
   public synchronized boolean refresh(HelixDataAccessor accessor,
@@ -86,6 +93,8 @@ public class TaskDataCache extends AbstractDataCache {
         _jobConfigMap.put(entry.getKey(), new JobConfig(entry.getValue()));
       }
     }
+    _dispatchedJobs.clear();
+
     return true;
   }
 
@@ -128,6 +137,7 @@ public class TaskDataCache extends AbstractDataCache {
 
   /**
    * Returns job config map
+   *
    * @return
    */
   public Map<String, JobConfig> getJobConfigMap() {
@@ -136,7 +146,9 @@ public class TaskDataCache extends AbstractDataCache {
 
   /**
    * Returns job config
+   *
    * @param resource
+   *
    * @return
    */
   public JobConfig getJobConfig(String resource) {
@@ -145,6 +157,7 @@ public class TaskDataCache extends AbstractDataCache {
 
   /**
    * Returns workflow config map
+   *
    * @return
    */
   public Map<String, WorkflowConfig> getWorkflowConfigMap() {
@@ -153,7 +166,9 @@ public class TaskDataCache extends AbstractDataCache {
 
   /**
    * Returns workflow config
+   *
    * @param resource
+   *
    * @return
    */
   public WorkflowConfig getWorkflowConfig(String resource) {
@@ -162,7 +177,9 @@ public class TaskDataCache extends AbstractDataCache {
 
   /**
    * Return the JobContext by resource name
+   *
    * @param resourceName
+   *
    * @return
    */
   public JobContext getJobContext(String resourceName) {
@@ -174,7 +191,9 @@ public class TaskDataCache extends AbstractDataCache {
 
   /**
    * Return the WorkflowContext by resource name
+   *
    * @param resourceName
+   *
    * @return
    */
   public WorkflowContext getWorkflowContext(String resourceName) {
@@ -249,6 +268,7 @@ public class TaskDataCache extends AbstractDataCache {
 
   /**
    * Return map of WorkflowContexts or JobContexts
+   *
    * @return
    */
   public Map<String, ZNRecord> getContexts() {
@@ -257,6 +277,7 @@ public class TaskDataCache extends AbstractDataCache {
 
   /**
    * Returns the current AssignableInstanceManager instance.
+   *
    * @return
    */
   public AssignableInstanceManager getAssignableInstanceManager() {
@@ -265,6 +286,7 @@ public class TaskDataCache extends AbstractDataCache {
 
   /**
    * Remove Workflow or Job context from cache
+   *
    * @param resourceName
    */
   public void removeContext(String resourceName) {
@@ -276,11 +298,8 @@ public class TaskDataCache extends AbstractDataCache {
 
   @Override
   public String toString() {
-    return "TaskDataCache{"
-        + "_jobConfigMap=" + _jobConfigMap
-        + ", _workflowConfigMap=" + _workflowConfigMap
-        + ", _contextMap=" + _contextMap
-        + ", _clusterName='" + _clusterName
+    return "TaskDataCache{" + "_jobConfigMap=" + _jobConfigMap + ", _workflowConfigMap="
+        + _workflowConfigMap + ", _contextMap=" + _contextMap + ", _clusterName='" + _clusterName
         + '\'' + '}';
   }
 
@@ -288,4 +307,16 @@ public class TaskDataCache extends AbstractDataCache {
     return String.format("/%s/%s%s/%s/%s", _clusterName, PropertyType.PROPERTYSTORE.name(),
         TaskConstants.REBALANCER_CONTEXT_ROOT, resourceName, TaskConstants.CONTEXT_NODE);
   }
+
+  public void dispatchJob(String jobName) {
+    _dispatchedJobs.add(jobName);
+  }
+
+  public void removeDispatchedJob(String jobName) {
+    _dispatchedJobs.remove(jobName);
+  }
+
+  public Set<String> getDispatchedJobs() {
+    return _dispatchedJobs;
+  }
 }
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java b/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java
index e90e63f..571267e 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java
@@ -1022,4 +1022,17 @@ public class ClusterDataCache extends AbstractDataCache {
 
     return sb.toString();
   }
+
+
+  public void dispatchJob(String jobName) {
+    _taskDataCache.dispatchJob(jobName);
+  }
+
+  public void removeDispatchedJob(String jobName) {
+    _taskDataCache.removeDispatchedJob(jobName);
+  }
+
+  public Set<String> getDispatchedJobs() {
+    return _taskDataCache.getDispatchedJobs();
+  }
 }
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/task/TaskSchedulingStage.java b/helix-core/src/main/java/org/apache/helix/controller/stages/task/TaskSchedulingStage.java
index 5399e17..218f470 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/task/TaskSchedulingStage.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/task/TaskSchedulingStage.java
@@ -7,6 +7,8 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.PriorityQueue;
+import java.util.Set;
+import org.apache.helix.HelixConstants;
 import org.apache.helix.HelixManager;
 import org.apache.helix.controller.LogUtil;
 import org.apache.helix.controller.pipeline.AbstractBaseStage;
@@ -73,12 +75,14 @@ public class TaskSchedulingStage extends AbstractBaseStage {
     BestPossibleStateOutput output = new BestPossibleStateOutput();
     final List<String> failureResources = new ArrayList<>();
     // Queues only for Workflows
-    scheduleWorkflows(resourceMap, cache, restOfResources, failureResources);
+    scheduleWorkflows(resourceMap, cache, restOfResources, failureResources, currentStateOutput, output);
+    for (String jobName : cache.getDispatchedJobs()) {
+      updateResourceMap(jobName, resourceMap, output.getPartitionStateMap(jobName).partitionSet());
+      restOfResources.remove(jobName);
+    }
 
-    // Current rest of resources including: jobs + only current state left over ones
-    Iterator<Resource> itr = restOfResources.values().iterator();
-    while (itr.hasNext()) {
-      Resource resource = itr.next();
+    // Current rest of resources including: only current state left over ones
+    for (Resource resource : restOfResources.values()) {
       if (!computeResourceBestPossibleState(event, cache, currentStateOutput, resource, output)) {
         failureResources.add(resource.getResourceName());
         LogUtil.logWarn(logger, _eventId,
@@ -160,7 +164,7 @@ public class TaskSchedulingStage extends AbstractBaseStage {
       rebalancer.init(manager);
         partitionStateAssignment = mappingCalculator
             .computeBestPossiblePartitionState(cache, idealState, resource, currentStateOutput);
-        updateOutput(resource, partitionStateAssignment, output);
+        _workflowDispatcher.updateBestPossibleStateOutput(resource.getResourceName(), partitionStateAssignment, output);
 
         // Check if calculation is done successfully
         return true;
@@ -231,21 +235,12 @@ public class TaskSchedulingStage extends AbstractBaseStage {
     }
     _workflowDispatcher.init(manager);
     _workflowDispatcher.setClusterStatusMonitor(monitor);
-    _workflowDispatcher.updateCache(cache.getTaskDataCache());
-  }
-
-  private void updateOutput(Resource resource, ResourceAssignment partitionStateAssignment,
-      BestPossibleStateOutput output) {
-    // Use the internal MappingCalculator interface to compute the final assignment
-    // The next release will support rebalancers that compute the mapping from start to finish
-    for (Partition partition : resource.getPartitions()) {
-      Map<String, String> newStateMap = partitionStateAssignment.getReplicaMap(partition);
-      output.setState(resource.getResourceName(), partition, newStateMap);
-    }
+    _workflowDispatcher.updateCache(cache);
   }
 
   private void scheduleWorkflows(Map<String, Resource> resourceMap, ClusterDataCache cache,
-      Map<String, Resource> restOfResources, List<String> failureResources) {
+      Map<String, Resource> restOfResources, List<String> failureResources,
+      CurrentStateOutput currentStateOutput, BestPossibleStateOutput bestPossibleOutput) {
     for (PriorityQueue<WorkflowObject> quotaBasedWorkflowPQ : _quotaBasedWorkflowPQs.values()) {
       Iterator<WorkflowObject> it = quotaBasedWorkflowPQ.iterator();
       while (it.hasNext()) {
@@ -260,7 +255,8 @@ public class TaskSchedulingStage extends AbstractBaseStage {
             _workflowDispatcher
                 .updateWorkflowStatus(workflowId, cache.getWorkflowConfig(workflowId), context);
             _workflowDispatcher
-                .assignWorkflow(workflowId, cache.getWorkflowConfig(workflowId), context);
+                .assignWorkflow(workflowId, cache.getWorkflowConfig(workflowId), context,
+                    currentStateOutput, bestPossibleOutput, resourceMap);
             restOfResources.remove(workflowId);
           } catch (Exception e) {
             LogUtil.logError(logger, _eventId,
@@ -271,4 +267,15 @@ public class TaskSchedulingStage extends AbstractBaseStage {
       }
     }
   }
+
+  private void updateResourceMap(String jobName, Map<String, Resource> resourceMap,
+      Set<Partition> partitionSet) {
+    Resource resource = new Resource(jobName);
+    for (Partition partition : partitionSet) {
+      resource.addPartition(partition.getPartitionName());
+    }
+    resource.setStateModelDefRef(TaskConstants.STATE_MODEL_NAME);
+    resource.setStateModelFactoryName(HelixConstants.DEFAULT_STATE_MODEL_FACTORY);
+    resourceMap.put(jobName, resource);
+  }
 }
diff --git a/helix-core/src/main/java/org/apache/helix/task/AbstractTaskDispatcher.java b/helix-core/src/main/java/org/apache/helix/task/AbstractTaskDispatcher.java
index cb721e5..60e4f92 100644
--- a/helix-core/src/main/java/org/apache/helix/task/AbstractTaskDispatcher.java
+++ b/helix-core/src/main/java/org/apache/helix/task/AbstractTaskDispatcher.java
@@ -15,11 +15,13 @@ import org.apache.helix.HelixDefinedState;
 import org.apache.helix.HelixManager;
 import org.apache.helix.common.caches.TaskDataCache;
 import org.apache.helix.controller.rebalancer.util.RebalanceScheduler;
+import org.apache.helix.controller.stages.BestPossibleStateOutput;
 import org.apache.helix.controller.stages.ClusterDataCache;
 import org.apache.helix.controller.stages.CurrentStateOutput;
 import org.apache.helix.model.InstanceConfig;
 import org.apache.helix.model.Message;
 import org.apache.helix.model.Partition;
+import org.apache.helix.model.Resource;
 import org.apache.helix.model.ResourceAssignment;
 import org.apache.helix.monitoring.mbeans.ClusterStatusMonitor;
 import org.apache.helix.task.assigner.AssignableInstance;
@@ -1094,4 +1096,14 @@ public abstract class AbstractTaskDispatcher {
     // Workflow with non-scheduled config or passed start time is ready to schedule.
     return (startTime == null || startTime.getTime() <= System.currentTimeMillis());
   }
+
+  public void updateBestPossibleStateOutput(String resource,
+      ResourceAssignment partitionStateAssignment, BestPossibleStateOutput output) {
+    // Use the internal MappingCalculator interface to compute the final assignment
+    // The next release will support rebalancers that compute the mapping from start to finish
+    for (Partition partition : partitionStateAssignment.getMappedPartitions()) {
+      Map<String, String> newStateMap = partitionStateAssignment.getReplicaMap(partition);
+      output.setState(resource, partition, newStateMap);
+    }
+  }
 }
diff --git a/helix-core/src/main/java/org/apache/helix/task/JobDispatcher.java b/helix-core/src/main/java/org/apache/helix/task/JobDispatcher.java
index 0f2558c..20ce23f 100644
--- a/helix-core/src/main/java/org/apache/helix/task/JobDispatcher.java
+++ b/helix-core/src/main/java/org/apache/helix/task/JobDispatcher.java
@@ -36,7 +36,7 @@ public class JobDispatcher extends AbstractTaskDispatcher {
   }
 
   public ResourceAssignment processJobStatusUpdateandAssignment(String jobName,
-      CurrentStateOutput currStateOutput, IdealState taskIs) {
+      CurrentStateOutput currStateOutput, WorkflowContext workflowCtx) {
     // Fetch job configuration
     JobConfig jobCfg = _clusterDataCache.getJobConfig(jobName);
     if (jobCfg == null) {
@@ -52,7 +52,7 @@ public class JobDispatcher extends AbstractTaskDispatcher {
       return buildEmptyAssignment(jobName, currStateOutput);
     }
 
-    WorkflowContext workflowCtx = _clusterDataCache.getWorkflowContext(workflowResource);
+
     if (workflowCtx == null) {
       LOG.error("Workflow context is NULL for " + jobName);
       return buildEmptyAssignment(jobName, currStateOutput);
@@ -158,18 +158,6 @@ public class JobDispatcher extends AbstractTaskDispatcher {
         computeResourceMapping(jobName, workflowCfg, jobCfg, jobState, jobTgtState, prevAssignment,
             liveInstances, currStateOutput, workflowCtx, jobCtx, partitionsToDrop, _clusterDataCache);
 
-    HelixDataAccessor accessor = _manager.getHelixDataAccessor();
-    PropertyKey propertyKey = accessor.keyBuilder().idealStates(jobName);
-
-    // TODO: will be removed when we are trying to get rid of IdealState
-    taskIs = _clusterDataCache.getIdealState(jobName);
-    if (!partitionsToDrop.isEmpty() && taskIs != null) {
-      for (Integer pId : partitionsToDrop) {
-        taskIs.getRecord().getMapFields().remove(pName(jobName, pId));
-      }
-      accessor.setProperty(propertyKey, taskIs);
-    }
-
     // Update Workflow and Job context in data cache and ZK.
     _clusterDataCache.updateJobContext(jobName, jobCtx);
     _clusterDataCache.updateWorkflowContext(workflowResource, workflowCtx);
diff --git a/helix-core/src/main/java/org/apache/helix/task/JobRebalancer.java b/helix-core/src/main/java/org/apache/helix/task/JobRebalancer.java
index ef4e26a..92cedf5 100644
--- a/helix-core/src/main/java/org/apache/helix/task/JobRebalancer.java
+++ b/helix-core/src/main/java/org/apache/helix/task/JobRebalancer.java
@@ -68,8 +68,9 @@ public class JobRebalancer extends TaskRebalancer {
     _jobDispatcher.init(_manager);
     _jobDispatcher.updateCache(clusterData);
     _jobDispatcher.setClusterStatusMonitor(_clusterStatusMonitor);
-    ResourceAssignment resourceAssignment =
-        _jobDispatcher.processJobStatusUpdateandAssignment(jobName, currStateOutput, taskIs);
+    ResourceAssignment resourceAssignment = _jobDispatcher
+        .processJobStatusUpdateandAssignment(jobName, currStateOutput,
+            clusterData.getWorkflowContext(clusterData.getJobConfig(jobName).getWorkflow()));
     LOG.debug(String.format("JobRebalancer computation takes %d ms for Job %s",
         System.currentTimeMillis() - startTime, jobName));
     return resourceAssignment;
diff --git a/helix-core/src/main/java/org/apache/helix/task/WorkflowDispatcher.java b/helix-core/src/main/java/org/apache/helix/task/WorkflowDispatcher.java
index db85a14..f5b76b1 100644
--- a/helix-core/src/main/java/org/apache/helix/task/WorkflowDispatcher.java
+++ b/helix-core/src/main/java/org/apache/helix/task/WorkflowDispatcher.java
@@ -19,7 +19,13 @@ import org.apache.helix.HelixProperty;
 import org.apache.helix.PropertyKey;
 import org.apache.helix.ZNRecord;
 import org.apache.helix.common.caches.TaskDataCache;
+import org.apache.helix.controller.LogUtil;
+import org.apache.helix.controller.stages.BestPossibleStateOutput;
+import org.apache.helix.controller.stages.ClusterDataCache;
+import org.apache.helix.controller.stages.CurrentStateOutput;
 import org.apache.helix.model.IdealState;
+import org.apache.helix.model.Resource;
+import org.apache.helix.model.ResourceAssignment;
 import org.apache.helix.model.builder.CustomModeISBuilder;
 import org.apache.helix.model.builder.IdealStateBuilder;
 import org.slf4j.Logger;
@@ -29,10 +35,17 @@ public class WorkflowDispatcher extends AbstractTaskDispatcher {
   private static final Logger LOG = LoggerFactory.getLogger(WorkflowDispatcher.class);
   private static final Set<TaskState> finalStates = new HashSet<>(
       Arrays.asList(TaskState.COMPLETED, TaskState.FAILED, TaskState.ABORTED, TaskState.TIMED_OUT));
-  private TaskDataCache _taskDataCache;
+  private ClusterDataCache _clusterDataCache;
+  private JobDispatcher _jobDispatcher;
 
-  public void updateCache(TaskDataCache cache) {
-    _taskDataCache = cache;
+  public void updateCache(ClusterDataCache cache) {
+    _clusterDataCache = cache;
+    if (_jobDispatcher == null) {
+      _jobDispatcher = new JobDispatcher();
+    }
+    _jobDispatcher.init(_manager);
+    _jobDispatcher.updateCache(cache);
+    _jobDispatcher.setClusterStatusMonitor(_clusterStatusMonitor);
   }
 
   // Split it into status update and assign. But there are couple of data need
@@ -63,7 +76,7 @@ public class WorkflowDispatcher extends AbstractTaskDispatcher {
 
       if (!TaskState.TIMED_OUT.equals(workflowCtx.getWorkflowState()) && isTimeout(workflowCtx.getStartTime(), workflowCfg.getTimeout())) {
         workflowCtx.setWorkflowState(TaskState.TIMED_OUT);
-        _taskDataCache.updateWorkflowContext(workflow, workflowCtx);
+        _clusterDataCache.updateWorkflowContext(workflow, workflowCtx);
       }
 
       // We should not return after setting timeout, as in case the workflow is stopped already
@@ -79,7 +92,7 @@ public class WorkflowDispatcher extends AbstractTaskDispatcher {
       LOG.info("Workflow " + workflow + "is marked as stopped.");
       if (isWorkflowStopped(workflowCtx, workflowCfg)) {
         workflowCtx.setWorkflowState(TaskState.STOPPED);
-        _taskDataCache.updateWorkflowContext(workflow, workflowCtx);
+        _clusterDataCache.updateWorkflowContext(workflow, workflowCtx);
       }
       return;
     }
@@ -93,10 +106,10 @@ public class WorkflowDispatcher extends AbstractTaskDispatcher {
     // Note that COMPLETE and FAILED will be marked in markJobComplete / markJobFailed
     // This is to handle TIMED_OUT only
     if (workflowCtx.getFinishTime() == WorkflowContext.UNFINISHED && isWorkflowFinished(workflowCtx,
-        workflowCfg, _taskDataCache.getJobConfigMap(), _taskDataCache)) {
+        workflowCfg, _clusterDataCache.getJobConfigMap(), _clusterDataCache.getTaskDataCache())) {
       workflowCtx.setFinishTime(currentTime);
       updateWorkflowMonitor(workflowCtx, workflowCfg);
-      _taskDataCache.updateWorkflowContext(workflow, workflowCtx);
+      _clusterDataCache.updateWorkflowContext(workflow, workflowCtx);
     }
 
     // Step 5: Handle finished workflows
@@ -125,11 +138,12 @@ public class WorkflowDispatcher extends AbstractTaskDispatcher {
       }
     }
 
-    _taskDataCache.updateWorkflowContext(workflow, workflowCtx);
+    _clusterDataCache.updateWorkflowContext(workflow, workflowCtx);
   }
 
   public void assignWorkflow(String workflow, WorkflowConfig workflowCfg,
-      WorkflowContext workflowCtx) {
+      WorkflowContext workflowCtx, CurrentStateOutput currentStateOutput,
+      BestPossibleStateOutput bestPossibleOutput, Map<String, Resource> resourceMap) {
     // Fetch workflow configuration and context
     if (workflowCfg == null) {
       // Already logged in status update.
@@ -144,16 +158,17 @@ public class WorkflowDispatcher extends AbstractTaskDispatcher {
       return;
     }
 
-
     // Check for readiness, and stop processing if it's not ready
-    boolean isReady = scheduleWorkflowIfReady(workflow, workflowCfg, workflowCtx, _taskDataCache);
+    boolean isReady = scheduleWorkflowIfReady(workflow, workflowCfg, workflowCtx,
+        _clusterDataCache.getTaskDataCache());
     if (isReady) {
       // Schedule jobs from this workflow.
-      scheduleJobs(workflow, workflowCfg, workflowCtx, _taskDataCache.getJobConfigMap(), _taskDataCache);
+      scheduleJobs(workflow, workflowCfg, workflowCtx, _clusterDataCache.getJobConfigMap(),
+          _clusterDataCache, currentStateOutput, bestPossibleOutput, resourceMap);
     } else {
       LOG.debug("Workflow " + workflow + " is not ready to be scheduled.");
     }
-    _taskDataCache.updateWorkflowContext(workflow, workflowCtx);
+    _clusterDataCache.updateWorkflowContext(workflow, workflowCtx);
   }
 
   public WorkflowContext getOrInitializeWorkflowContext(
@@ -174,7 +189,8 @@ public class WorkflowDispatcher extends AbstractTaskDispatcher {
    */
   private void scheduleJobs(String workflow, WorkflowConfig workflowCfg,
       WorkflowContext workflowCtx, Map<String, JobConfig> jobConfigMap,
-      TaskDataCache clusterDataCache) {
+      ClusterDataCache clusterDataCache, CurrentStateOutput currentStateOutput,
+      BestPossibleStateOutput bestPossibleOutput, Map<String, Resource> resourceMap) {
     ScheduleConfig scheduleConfig = workflowCfg.getScheduleConfig();
     if (scheduleConfig != null && scheduleConfig.isRecurring()) {
       LOG.debug("Jobs from recurring workflow are not schedule-able");
@@ -190,6 +206,7 @@ public class WorkflowDispatcher extends AbstractTaskDispatcher {
         if (LOG.isDebugEnabled()) {
           LOG.debug("Job " + job + " is already started or completed.");
         }
+        processJob(job, currentStateOutput, bestPossibleOutput, workflowCtx);
         continue;
       }
 
@@ -203,7 +220,7 @@ public class WorkflowDispatcher extends AbstractTaskDispatcher {
 
       // check ancestor job status
       if (isJobReadyToSchedule(job, workflowCfg, workflowCtx, inCompleteAllJobCount, jobConfigMap,
-          clusterDataCache)) {
+          clusterDataCache.getTaskDataCache())) {
         JobConfig jobConfig = jobConfigMap.get(job);
         if (jobConfig == null) {
           LOG.error(String.format("The job config is missing for job %s", job));
@@ -231,6 +248,7 @@ public class WorkflowDispatcher extends AbstractTaskDispatcher {
         } else {
           scheduleSingleJob(job, jobConfig);
           workflowCtx.setJobState(job, TaskState.NOT_STARTED);
+          processJob(job, currentStateOutput, bestPossibleOutput, workflowCtx);
           scheduledJobs++;
         }
       }
@@ -243,6 +261,19 @@ public class WorkflowDispatcher extends AbstractTaskDispatcher {
     }
   }
 
+  private void processJob(String job, CurrentStateOutput currentStateOutput,
+      BestPossibleStateOutput bestPossibleOutput, WorkflowContext workflowCtx) {
+    _clusterDataCache.dispatchJob(job);
+    try {
+      ResourceAssignment resourceAssignment =
+          _jobDispatcher.processJobStatusUpdateandAssignment(job, currentStateOutput, workflowCtx);
+      updateBestPossibleStateOutput(job, resourceAssignment, bestPossibleOutput);
+    } catch (Exception e) {
+      LogUtil.logWarn(LOG, _clusterDataCache.getEventId(),
+          String.format("Failed to compute job assignment for job %s", job));
+    }
+  }
+
   /**
    * Posts new job to cluster
    */
@@ -491,7 +522,7 @@ public class WorkflowDispatcher extends AbstractTaskDispatcher {
    */
   private void cleanupWorkflow(String workflow) {
     LOG.info("Cleaning up workflow: " + workflow);
-    WorkflowConfig workflowcfg = _taskDataCache.getWorkflowConfig(workflow);
+    WorkflowConfig workflowcfg = _clusterDataCache.getWorkflowConfig(workflow);
 
     if (workflowcfg.isTerminable() || workflowcfg.getTargetState() == TargetState.DELETE) {
       Set<String> jobs = workflowcfg.getJobDag().getAllNodes();
@@ -507,7 +538,7 @@ public class WorkflowDispatcher extends AbstractTaskDispatcher {
         // Only remove from cache when remove all workflow success. Otherwise, batch write will
         // clean all the contexts even if Configs and IdealStates are exists. Then all the workflows
         // and jobs will rescheduled again.
-        removeContexts(workflow, jobs, _taskDataCache);
+        removeContexts(workflow, jobs, _clusterDataCache.getTaskDataCache());
       }
      } else {
       LOG.info("Did not clean up workflow " + workflow
@@ -523,4 +554,5 @@ public class WorkflowDispatcher extends AbstractTaskDispatcher {
     }
     cache.removeContext(workflow);
   }
+
 }
diff --git a/helix-core/src/main/java/org/apache/helix/task/WorkflowRebalancer.java b/helix-core/src/main/java/org/apache/helix/task/WorkflowRebalancer.java
index c913131..c046bb0 100644
--- a/helix-core/src/main/java/org/apache/helix/task/WorkflowRebalancer.java
+++ b/helix-core/src/main/java/org/apache/helix/task/WorkflowRebalancer.java
@@ -19,6 +19,8 @@ package org.apache.helix.task;
  * under the License.
  */
 
+import java.util.HashMap;
+import org.apache.helix.controller.stages.BestPossibleStateOutput;
 import org.apache.helix.controller.stages.ClusterDataCache;
 import org.apache.helix.controller.stages.CurrentStateOutput;
 import org.apache.helix.model.IdealState;
@@ -46,9 +48,10 @@ public class WorkflowRebalancer extends TaskRebalancer {
     WorkflowConfig workflowCfg = clusterData.getWorkflowConfig(workflow);
 
     _workflowDispatcher.setClusterStatusMonitor(_clusterStatusMonitor);
-    _workflowDispatcher.updateCache(clusterData.getTaskDataCache());
+    _workflowDispatcher.updateCache(clusterData);
     _workflowDispatcher.updateWorkflowStatus(workflow, workflowCfg, workflowCtx);
-    _workflowDispatcher.assignWorkflow(workflow, workflowCfg, workflowCtx);
+    _workflowDispatcher.assignWorkflow(workflow, workflowCfg, workflowCtx, currStateOutput,
+        new BestPossibleStateOutput(), new HashMap<String, Resource>());
 
     LOG.debug(String.format("WorkflowRebalancer computation takes %d ms for workflow %s",
         System.currentTimeMillis() - startTime, workflow));
diff --git a/helix-core/src/test/java/org/apache/helix/integration/task/TestDeleteWorkflow.java b/helix-core/src/test/java/org/apache/helix/integration/task/TestDeleteWorkflow.java
index 828bad3..6a859e1 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/task/TestDeleteWorkflow.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/task/TestDeleteWorkflow.java
@@ -48,7 +48,7 @@ public class TestDeleteWorkflow extends TaskTestBase  {
 
     // Pause the Controller so that the job queue won't get deleted
     admin.enableCluster(CLUSTER_NAME, false);
-
+    Thread.sleep(1000);
     // Attempt the deletion and time out
     try {
       _driver.deleteAndWaitForCompletion(jobQueueName, DELETE_DELAY);
diff --git a/helix-core/src/test/java/org/apache/helix/integration/task/TestIndependentTaskRebalancer.java b/helix-core/src/test/java/org/apache/helix/integration/task/TestIndependentTaskRebalancer.java
index 1b068a0..4f9e012 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/task/TestIndependentTaskRebalancer.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/task/TestIndependentTaskRebalancer.java
@@ -245,7 +245,7 @@ public class TestIndependentTaskRebalancer extends TaskTestBase {
     TaskConfig taskConfig1 = new TaskConfig("TaskOne", taskConfigMap);
     taskConfigs.add(taskConfig1);
     Map<String, String> jobCommandMap = Maps.newHashMap();
-    jobCommandMap.put("Timeout", "1000");
+    jobCommandMap.put(MockTask.JOB_DELAY, "1000");
 
     JobConfig.Builder jobBuilder = new JobConfig.Builder().setCommand("DummyCommand")
         .addTaskConfigs(taskConfigs).setJobCommandConfigMap(jobCommandMap);
diff --git a/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestClusterAggregateMetrics.java b/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestClusterAggregateMetrics.java
index 2cd84fb..fc1df4e 100644
--- a/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestClusterAggregateMetrics.java
+++ b/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestClusterAggregateMetrics.java
@@ -154,7 +154,7 @@ public class TestClusterAggregateMetrics extends ZkTestBase {
       String instanceName = PARTICIPANT_PREFIX + "_" + (START_PORT + i);
       _setupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, instanceName, false);
     }
-    Thread.sleep(500);
+    Thread.sleep(1000);
     updateMetrics();
     Assert.assertEquals(_beanValueMap.get(PARTITION_COUNT), 5L);
     Assert.assertEquals(_beanValueMap.get(ERROR_PARTITION_COUNT), 0L);
@@ -166,7 +166,7 @@ public class TestClusterAggregateMetrics extends ZkTestBase {
       String instanceName = PARTICIPANT_PREFIX + "_" + (START_PORT + i);
       _setupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, instanceName, true);
     }
-    Thread.sleep(500);
+    Thread.sleep(1000);
     updateMetrics();
     Assert.assertEquals(_beanValueMap.get(PARTITION_COUNT), 5L);
     Assert.assertEquals(_beanValueMap.get(ERROR_PARTITION_COUNT), 0L);
@@ -175,7 +175,7 @@ public class TestClusterAggregateMetrics extends ZkTestBase {
 
     // Drop the resource and check that all metrics are zero.
     _setupTool.dropResourceFromCluster(CLUSTER_NAME, TEST_DB);
-    Thread.sleep(500);
+    Thread.sleep(1000);
     updateMetrics();
     Assert.assertEquals(_beanValueMap.get(PARTITION_COUNT), 0L);
     Assert.assertEquals(_beanValueMap.get(ERROR_PARTITION_COUNT), 0L);
diff --git a/helix-core/src/test/java/org/apache/helix/task/TestJobStateOnCreation.java b/helix-core/src/test/java/org/apache/helix/task/TestJobStateOnCreation.java
deleted file mode 100644
index b50197b..0000000
--- a/helix-core/src/test/java/org/apache/helix/task/TestJobStateOnCreation.java
+++ /dev/null
@@ -1,84 +0,0 @@
-package org.apache.helix.task;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import com.google.common.collect.Sets;
-import java.util.Map;
-import org.apache.helix.controller.stages.ClusterDataCache;
-import org.apache.helix.controller.stages.CurrentStateOutput;
-import org.apache.helix.integration.manager.MockParticipantManager;
-import org.apache.helix.integration.task.MockTask;
-import org.apache.helix.integration.task.WorkflowGenerator;
-import org.apache.helix.model.IdealState;
-import org.apache.helix.model.Resource;
-import org.apache.helix.model.ResourceAssignment;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-public class TestJobStateOnCreation extends TaskSynchronizedTestBase {
-
-  private static final String WORKFLOW_NAME = "testWorkflow";
-
-  private ClusterDataCache _cache;
-  private IdealState _idealState;
-  private Resource _resource;
-  private CurrentStateOutput _currStateOutput;
-
-  @BeforeClass
-  public void beforeClass() throws Exception {
-    _cache = new ClusterDataCache();
-    _idealState = new IdealState(WORKFLOW_NAME);
-    _resource = new Resource(WORKFLOW_NAME);
-    _currStateOutput = new CurrentStateOutput();
-    _participants =  new MockParticipantManager[_numNodes];
-
-    _gSetupTool.addCluster(CLUSTER_NAME, true);
-    createManagers();
-  }
-
-  @Test
-  public void testJobStateOnCreation() {
-    Workflow.Builder builder = new Workflow.Builder(WORKFLOW_NAME);
-    JobConfig.Builder jobConfigBuilder = new JobConfig.Builder().setCommand(MockTask.TASK_COMMAND)
-        .setTargetResource(WORKFLOW_NAME).setTargetPartitionStates(Sets.newHashSet("SLAVE","MASTER"))
-        .setJobCommandConfigMap(WorkflowGenerator.DEFAULT_COMMAND_CONFIG);
-    String jobName = "job";
-    builder = builder.addJob(jobName, jobConfigBuilder);
-    Workflow workflow = builder.build();
-    WorkflowConfig workflowConfig = workflow.getWorkflowConfig();
-    JobConfig jobConfig = jobConfigBuilder.build();
-    workflowConfig.getRecord().merge(jobConfig.getRecord());
-
-    _cache.getJobConfigMap().put(WORKFLOW_NAME + "_" + jobName, jobConfig);
-    _cache.getWorkflowConfigMap().put(WORKFLOW_NAME, workflowConfig);
-
-    WorkflowRebalancer workflowRebalancer = new WorkflowRebalancer();
-    workflowRebalancer.init(_manager);
-    ResourceAssignment resourceAssignment = workflowRebalancer
-        .computeBestPossiblePartitionState(_cache, _idealState, _resource, _currStateOutput);
-
-    WorkflowContext workflowContext = _cache.getWorkflowContext(WORKFLOW_NAME);
-    Map<String, TaskState> jobStates = workflowContext.getJobStates();
-    for (String job : jobStates.keySet()) {
-      Assert.assertEquals(jobStates.get(job), TaskState.NOT_STARTED);
-    }
-  }
-}