You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2020/09/02 15:40:36 UTC
[airavata] 02/02: Logging improvements for parser wm
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch param-sweep
in repository https://gitbox.apache.org/repos/asf/airavata.git
commit ad8b7cc27988dcd2a5feaeeb3794ddaf99000680
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Wed Sep 2 11:40:18 2020 -0400
Logging improvements for parser wm
---
.../helix/impl/workflow/ParserWorkflowManager.java | 35 +++++++++++-----------
1 file changed, 17 insertions(+), 18 deletions(-)
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/workflow/ParserWorkflowManager.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/workflow/ParserWorkflowManager.java
index 9f562e5..00945ff 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/workflow/ParserWorkflowManager.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/workflow/ParserWorkflowManager.java
@@ -88,22 +88,20 @@ public class ParserWorkflowManager extends WorkflowManager {
try {
ProcessModel processModel;
- ApplicationInterfaceDescription appDescription;
try {
processModel = registryClient.getProcess(completionMessage.getProcessId());
- appDescription = registryClient.getApplicationInterface(processModel.getApplicationInterfaceId());
-
} catch (Exception e) {
logger.error("Failed to fetch process or application description from registry associated with process id " + completionMessage.getProcessId(), e);
throw new Exception("Failed to fetch process or application description from registry associated with process id " + completionMessage.getProcessId(), e);
}
+ logger.info("Fetching parsing templates for thr experiment {} and the output version {}", completionMessage.getExperimentId(), completionMessage.getOutputVersion());
// All the templates should be run
// FIXME is it ApplicationInterfaceId or ApplicationName
List<ParsingTemplate> parsingTemplates = registryClient.getParsingTemplatesForExperiment(completionMessage.getExperimentId(),
completionMessage.getGatewayId());
- logger.info("Found " + parsingTemplates.size() + " parsing template for experiment " + completionMessage.getExperimentId());
+ logger.info("Found {} parsing template for experiment {}", parsingTemplates.size(), completionMessage.getExperimentId());
Map<String, Map<String, Set<ParserConnector>>> parentToChildParsers = new HashMap<>();
@@ -132,7 +130,7 @@ public class ParserWorkflowManager extends WorkflowManager {
for (ParsingTemplate template : parsingTemplates) {
- logger.info("Launching parsing template " + template.getId());
+ logger.info("Loading inputs for template {}", template.getId());
ParserInput parserInput = registryClient.getParserInput(template.getInitialInputs().get(0).getTargetInputId(), template.getGatewayId());
String parentParserId = parserInput.getParserId();
@@ -157,26 +155,32 @@ public class ParserWorkflowManager extends WorkflowManager {
throw new Exception("Could not find a parent parser for template " + template.getId());
}
+ logger.info("Loading the parent parser for template {}", template.getId());
Parser parentParser = registryClient.getParser(parentParserId, completionMessage.getGatewayId());
+ logger.info("Parent parser is {} for template {}", parentParser.getId(), template.getId());
+
DataParsingTask parentParserTask = createParentTask(parentParser, completionMessage, template.getInitialInputs(), registryClient);
List<AbstractTask> allTasks = new ArrayList<>();
allTasks.add(parentParserTask);
if (parentToChildParsers.containsKey(template.getId())) {
- createParserDagRecursively(allTasks, parentParser, parentParserTask, parentToChildParsers.get(template.getId()), completionMessage, registryClient);
+ logger.info("Creating the child parser dag recursively for template {}", template.getId());
+ createParserDagRecursively(allTasks, parentParser, parentParserTask,
+ parentToChildParsers.get(template.getId()), completionMessage, processModel, registryClient);
}
+
+ logger.info("Launching the parsing template {}", template.getId());
+
String workflow = getWorkflowOperator().launchWorkflow("Parser-" + completionMessage.getProcessId() + UUID.randomUUID().toString(),
allTasks, true, false);
// TODO: figure out processId and register
// registerWorkflowForProcess(processId, workflow, "PARSER");
- logger.info("Launched workflow " + workflow);
+ logger.info("Launched workflow {} for parsing template {} for experiment {}", workflow, template.getId(), completionMessage.getExperimentId());
}
getRegistryClientPool().returnResource(registryClient);
-
-
return true;
@@ -321,10 +325,11 @@ public class ParserWorkflowManager extends WorkflowManager {
}
private void createParserDagRecursively(List<AbstractTask> allTasks, Parser parentParserInfo, DataParsingTask parentTask, Map<String, Set<ParserConnector>> parentToChild,
- ProcessCompletionMessage completionMessage, RegistryService.Client registryClient) throws Exception {
+ ProcessCompletionMessage completionMessage, ProcessModel processModel, RegistryService.Client registryClient) throws Exception {
if (parentToChild.containsKey(parentParserInfo.getId())) {
for (ParserConnector connector : parentToChild.get(parentParserInfo.getId())) {
+ logger.info("Loading the child parser {}", connector.getChildParserId());
Parser childParserInfo = registryClient.getParser(connector.getChildParserId(), completionMessage.getGatewayId());
DataParsingTask parsingTask = new DataParsingTask();
parsingTask.setTaskId(normalizeTaskId(completionMessage.getExperimentId() + "-" + childParserInfo.getId() + "-" + UUID.randomUUID().toString()));
@@ -333,12 +338,7 @@ public class ParserWorkflowManager extends WorkflowManager {
parsingTask.setLocalDataDir("/tmp");
parentTask.setProcessId(completionMessage.getProcessId());
parentTask.setOutputVersion(completionMessage.getOutputVersion());
- try {
- parsingTask.setGroupResourceProfileId(registryClient.getProcess(completionMessage.getProcessId()).getGroupResourceProfileId());
- } catch (TException e) {
- logger.error("Failed while fetching process model for process id " + completionMessage.getProcessId());
- throw new Exception("Failed while fetching process model for process id " + completionMessage.getProcessId());
- }
+ parsingTask.setGroupResourceProfileId(processModel.getGroupResourceProfileId());
ParsingTaskInputs inputs = new ParsingTaskInputs();
for(ParserConnectorInput connectorInput : connector.getConnectorInputs()) {
@@ -377,7 +377,7 @@ public class ParserWorkflowManager extends WorkflowManager {
parentTask.setNextTask(new OutPort(parsingTask.getTaskId(), parentTask));
allTasks.add(parsingTask);
- createParserDagRecursively(allTasks, childParserInfo, parsingTask, parentToChild, completionMessage, registryClient);
+ createParserDagRecursively(allTasks, childParserInfo, parsingTask, parentToChild, completionMessage, processModel, registryClient);
}
}
}
@@ -415,5 +415,4 @@ public class ParserWorkflowManager extends WorkflowManager {
consumer.commitAsync();
}
}
-
}