You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2020/08/26 16:43:39 UTC

[airavata] branch param-sweep updated: Refactoring the post workflow manager

This is an automated email from the ASF dual-hosted git repository.

dimuthuupe pushed a commit to branch param-sweep
in repository https://gitbox.apache.org/repos/asf/airavata.git


The following commit(s) were added to refs/heads/param-sweep by this push:
     new e643cf0  Refactoring the post workflow manager
e643cf0 is described below

commit e643cf02dde076c12d1b7c72ca2e3ca32729ec6d
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Wed Aug 26 12:43:24 2020 -0400

    Refactoring the post workflow manager
---
 .../helix/impl/workflow/PostWorkflowManager.java   | 184 ++++++++++++++++-----
 1 file changed, 143 insertions(+), 41 deletions(-)

diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/workflow/PostWorkflowManager.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/workflow/PostWorkflowManager.java
index 29dc834..ada783a 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/workflow/PostWorkflowManager.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/workflow/PostWorkflowManager.java
@@ -31,9 +31,7 @@ import org.apache.airavata.helix.impl.task.parsing.ParsingTriggeringTask;
 import org.apache.airavata.helix.impl.task.staging.*;
 import org.apache.airavata.model.job.ChildJobModel;
 import org.apache.airavata.model.job.JobModel;
-import org.apache.airavata.model.status.ProcessState;
-import org.apache.airavata.model.status.ProcessStatus;
-import org.apache.airavata.monitor.JobStateValidator;
+import org.apache.airavata.model.status.*;
 import org.apache.airavata.monitor.JobStatusResult;
 import org.apache.airavata.monitor.kafka.JobStatusResultDeserializer;
 import org.apache.airavata.messaging.core.MessageContext;
@@ -42,8 +40,6 @@ import org.apache.airavata.model.messaging.event.JobIdentifier;
 import org.apache.airavata.model.messaging.event.JobStatusChangeEvent;
 import org.apache.airavata.model.messaging.event.MessageType;
 import org.apache.airavata.model.process.ProcessModel;
-import org.apache.airavata.model.status.JobState;
-import org.apache.airavata.model.status.JobStatus;
 import org.apache.airavata.model.task.DataStagingTaskModel;
 import org.apache.airavata.model.task.TaskModel;
 import org.apache.airavata.model.task.TaskTypes;
@@ -146,6 +142,8 @@ public class PostWorkflowManager extends WorkflowManager {
 
                 if (!jobStatusResult.isChildJob()) {
                     saveAndPublishJobStatus(jobStatusResult.getJobId(), task, processId, experimentId, gateway, jobStatusResult.getState());
+                } else {
+                    saveAndPublishChildJobStatus(jobStatusResult.getJobId(), jobStatusResult.getState());
                 }
 
                 // TODO get cluster lock before that
@@ -177,15 +175,15 @@ public class PostWorkflowManager extends WorkflowManager {
                         logger.info("Job " + jobStatusResult.getJobId() + " was completed");
 
                         if ("one_pass".equals(experimentModel.getExecutionType())) {
-                            executePostWorkflow(processId, gateway, false, 0, false, false);
+                            executePostWorkflowForProcess(processId, false);
                         } else {
 
                             if (jobStatusResult.isChildJob()) {
                                 logger.info("Running post workflow for child job index {} in job {} and task {}",
                                     jobStatusResult.getJobIndex(), jobStatusResult.getJobId(), task);
-                                executePostWorkflow(processId, gateway, true, jobStatusResult.getJobIndex(), true, false);
-
+                                executePostWorkflowForChildJob(processId, jobStatusResult.getJobIndex(), true);
                             } else {
+                                // Main job was completed. Check for all child job statuses and run missing ones.
                                 List<ChildJobModel> childJobsOfJob = registryClient.getChildJobsOfJob(jobStatusResult.getJobId(), task);
                                 Map<Integer, ChildJobModel> childMap = new HashMap<>();
                                 childJobsOfJob.forEach(child -> childMap.put(child.getJobIndex(), child));
@@ -197,12 +195,13 @@ public class PostWorkflowManager extends WorkflowManager {
                                                     .noneMatch(st -> (st.getJobState() == JobState.COMPLETE) || (st.getJobState() == JobState.FAILED))) {
                                         logger.info("Running post workflow for job {} index {} and task {}",
                                                 jobModel.getJobId(), i, jobModel.getTaskId());
-                                        executePostWorkflow(processId, gateway, false, i, false, true);
+                                        executePostWorkflowForChildJob(processId, i, true);
                                     } else {
                                         logger.info("Ignoring post workflow for job {} index {} in task {}",
                                                 jobModel.getJobId(), i, jobModel.getTaskId());
                                     }
                                 }
+                                executePostWorkflowForProcess(processId, true);
                             }
                         }
                     } else if (jobStatusResult.getState() == JobState.CANCELED) {
@@ -227,8 +226,7 @@ public class PostWorkflowManager extends WorkflowManager {
         }
     }
 
-    private void executePostWorkflow(String processId, String gateway, boolean forceRun, int jobIndex, boolean isChildJob, boolean excludeDataStage) throws Exception {
-
+    private void executePostWorkflowForChildJob(String processId, int jobIndex, boolean forceRun) throws Exception {
         RegistryService.Client registryClient = getRegistryClientPool().getResource();
 
         ProcessModel processModel;
@@ -270,22 +268,16 @@ public class PostWorkflowManager extends WorkflowManager {
                 AiravataTask airavataTask = null;
                 if (taskModel.getTaskType() == TaskTypes.JOB_SUBMISSION) {
                     jobSubmissionFound = true;
-                } else if (taskModel.getTaskType() == TaskTypes.DATA_STAGING && !excludeDataStage) {
+                } else if (taskModel.getTaskType() == TaskTypes.DATA_STAGING) {
                     if (jobSubmissionFound) {
                         DataStagingTaskModel subTaskModel = (DataStagingTaskModel) ThriftUtils.getSubTaskModel(taskModel);
                         assert subTaskModel != null;
                         switch (subTaskModel.getType()) {
                             case OUPUT:
-                                if ("one_pass".equals(experimentModel.getExecutionType())) {
-                                    airavataTask = new OutputDataStagingTask();
-                                    airavataTask.setForceRunTask(true);
-                                }
-                                if ("param_sweep".equals(experimentModel.getExecutionType())) {
-                                    SweepingOutputDataStagingTask sweepTask = new SweepingOutputDataStagingTask();
-                                    sweepTask.setJobIndex(jobIndex);
-                                    sweepTask.setForceRunTask(true);
-                                    airavataTask = sweepTask;
-                                }
+                                SweepingOutputDataStagingTask sweepTask = new SweepingOutputDataStagingTask();
+                                sweepTask.setJobIndex(jobIndex);
+                                sweepTask.setForceRunTask(true);
+                                airavataTask = sweepTask;
                                 break;
                             case ARCHIVE_OUTPUT:
                                 airavataTask = new ArchiveTask();
@@ -309,27 +301,118 @@ public class PostWorkflowManager extends WorkflowManager {
             }
         }
 
-        if (!isChildJob) {
-            CompletingTask completingTask = new CompletingTask();
-            completingTask.setGatewayId(experimentModel.getGatewayId());
-            completingTask.setExperimentId(experimentModel.getExperimentId());
-            completingTask.setProcessId(processModel.getProcessId());
-            completingTask.setTaskId("Completing-Task-" + UUID.randomUUID().toString() + "-");
-            completingTask.setForceRunTask(forceRun);
-            completingTask.setSkipTaskStatusPublish(true);
-            if (allTasks.size() > 0) {
-                allTasks.get(allTasks.size() - 1).setNextTask(new OutPort(completingTask.getTaskId(), completingTask));
+        ParsingTriggeringTask parsingTriggeringTask = new ParsingTriggeringTask();
+        parsingTriggeringTask.setGatewayId(experimentModel.getGatewayId());
+        parsingTriggeringTask.setExperimentId(experimentModel.getExperimentId());
+        parsingTriggeringTask.setProcessId(processModel.getProcessId());
+        parsingTriggeringTask.setTaskId("Parsing-Triggering-Task-" + UUID.randomUUID().toString() + "-");
+        parsingTriggeringTask.setSkipTaskStatusPublish(true);
+        parsingTriggeringTask.setOutputVersion(jobIndex);
+
+        if (allTasks.size() > 0) {
+            allTasks.get(allTasks.size() - 1).setNextTask(new OutPort(parsingTriggeringTask.getTaskId(), parsingTriggeringTask));
+        }
+        allTasks.add(parsingTriggeringTask);
+
+        String workflowName = getWorkflowOperator().launchWorkflow(processId + "-POST-" + UUID.randomUUID().toString(),
+                new ArrayList<>(allTasks), true, false);
+
+        registerWorkflowForProcess(processId, workflowName, "POST");
+
+    }
+
+    private void executePostWorkflowForProcess(String processId, boolean forceRun) throws Exception {
+        RegistryService.Client registryClient = getRegistryClientPool().getResource();
+
+        ProcessModel processModel;
+        ExperimentModel experimentModel;
+        try {
+            processModel = registryClient.getProcess(processId);
+            experimentModel = registryClient.getExperiment(processModel.getExperimentId());
+            getRegistryClientPool().returnResource(registryClient);
+
+        } catch (Exception e) {
+            logger.error("Failed to fetch experiment or process from registry associated with process id " + processId, e);
+            getRegistryClientPool().returnResource(registryClient);
+            throw new Exception("Failed to fetch experiment or process from registry associated with process id " + processId, e);
+        }
+
+        String taskDag = processModel.getTaskDag();
+        List<TaskModel> taskList = processModel.getTasks();
+
+        String[] taskIds = taskDag.split(",");
+        final List<AiravataTask> allTasks = new ArrayList<>();
+
+        JobVerificationTask jobVerificationTask = new JobVerificationTask();
+        jobVerificationTask.setGatewayId(experimentModel.getGatewayId());
+        jobVerificationTask.setExperimentId(experimentModel.getExperimentId());
+        jobVerificationTask.setProcessId(processModel.getProcessId());
+        jobVerificationTask.setTaskId("Job-Verification-Task-" + UUID.randomUUID().toString() +"-");
+        jobVerificationTask.setForceRunTask(forceRun);
+        jobVerificationTask.setSkipTaskStatusPublish(true);
+
+        allTasks.add(jobVerificationTask);
+
+        boolean jobSubmissionFound = false;
+
+        for (String taskId : taskIds) {
+            Optional<TaskModel> model = taskList.stream().filter(taskModel -> taskModel.getTaskId().equals(taskId)).findFirst();
+
+            if (model.isPresent()) {
+                TaskModel taskModel = model.get();
+                AiravataTask airavataTask = null;
+                if (taskModel.getTaskType() == TaskTypes.JOB_SUBMISSION) {
+                    jobSubmissionFound = true;
+                } else if (taskModel.getTaskType() == TaskTypes.DATA_STAGING) {
+                    if (jobSubmissionFound) {
+                        DataStagingTaskModel subTaskModel = (DataStagingTaskModel) ThriftUtils.getSubTaskModel(taskModel);
+                        assert subTaskModel != null;
+                        switch (subTaskModel.getType()) {
+                            case OUPUT:
+                                airavataTask = new OutputDataStagingTask();
+                                airavataTask.setForceRunTask(true);
+                                break;
+                            case ARCHIVE_OUTPUT:
+                                airavataTask = new ArchiveTask();
+                                airavataTask.setForceRunTask(true);
+                                break;
+                        }
+                    }
+                }
+
+                if (airavataTask != null) {
+                    airavataTask.setGatewayId(experimentModel.getGatewayId());
+                    airavataTask.setExperimentId(experimentModel.getExperimentId());
+                    airavataTask.setProcessId(processModel.getProcessId());
+                    airavataTask.setTaskId(taskModel.getTaskId());
+                    airavataTask.setRetryCount(taskModel.getMaxRetry());
+                    if (allTasks.size() > 0) {
+                        allTasks.get(allTasks.size() - 1).setNextTask(new OutPort(airavataTask.getTaskId(), airavataTask));
+                    }
+                    allTasks.add(airavataTask);
+                }
             }
-            allTasks.add(completingTask);
         }
 
+        CompletingTask completingTask = new CompletingTask();
+        completingTask.setGatewayId(experimentModel.getGatewayId());
+        completingTask.setExperimentId(experimentModel.getExperimentId());
+        completingTask.setProcessId(processModel.getProcessId());
+        completingTask.setTaskId("Completing-Task-" + UUID.randomUUID().toString() + "-");
+        completingTask.setForceRunTask(forceRun);
+        completingTask.setSkipTaskStatusPublish(true);
+        if (allTasks.size() > 0) {
+            allTasks.get(allTasks.size() - 1).setNextTask(new OutPort(completingTask.getTaskId(), completingTask));
+        }
+        allTasks.add(completingTask);
+
         ParsingTriggeringTask parsingTriggeringTask = new ParsingTriggeringTask();
         parsingTriggeringTask.setGatewayId(experimentModel.getGatewayId());
         parsingTriggeringTask.setExperimentId(experimentModel.getExperimentId());
         parsingTriggeringTask.setProcessId(processModel.getProcessId());
-        parsingTriggeringTask.setTaskId("Parsing-Triggering-Task");
+        parsingTriggeringTask.setTaskId("Parsing-Triggering-Task-" + UUID.randomUUID().toString() + "-");
         parsingTriggeringTask.setSkipTaskStatusPublish(true);
-        parsingTriggeringTask.setOutputVersion(jobIndex);
+        parsingTriggeringTask.setOutputVersion(-1);
 
         if (allTasks.size() > 0) {
             allTasks.get(allTasks.size() - 1).setNextTask(new OutPort(parsingTriggeringTask.getTaskId(), parsingTriggeringTask));
@@ -386,6 +469,31 @@ public class PostWorkflowManager extends WorkflowManager {
         }).start();
     }
 
+    private void saveAndPublishChildJobStatus(String jobId, JobState jobState) throws Exception {
+
+        try {
+            RegistryService.Client registryClient = getRegistryClientPool().getResource();
+
+            ChildJobStatus childJobStatus = new ChildJobStatus();
+            childJobStatus.setJobState(jobState);
+            childJobStatus.setReason(jobState.name());
+            childJobStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
+
+            try {
+                registryClient.addChildJobStatus(childJobStatus, jobId);
+                getRegistryClientPool().returnResource(registryClient);
+
+            } catch (Exception e) {
+                logger.error("Failed to add child job status " + jobId, e);
+                getRegistryClientPool().returnBrokenResource(registryClient);
+            }
+
+        } catch (Exception e) {
+            logger.error("Error persisting child job status {}", jobId, e);
+            throw e;
+        }
+    }
+
     private void saveAndPublishJobStatus(String jobId, String taskId, String processId, String experimentId, String gateway,
                                         JobState jobState) throws Exception {
         try {
@@ -395,12 +503,6 @@ public class PostWorkflowManager extends WorkflowManager {
             jobStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
             jobStatus.setJobState(jobState);
 
-            if (jobStatus.getTimeOfStateChange() == 0 || jobStatus.getTimeOfStateChange() > 0 ) {
-                jobStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
-            } else {
-                jobStatus.setTimeOfStateChange(jobStatus.getTimeOfStateChange());
-            }
-
             RegistryService.Client registryClient = getRegistryClientPool().getResource();
 
             try {