You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2020/09/02 15:40:34 UTC
[airavata] branch param-sweep updated (fd76113 -> ad8b7cc)
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a change to branch param-sweep
in repository https://gitbox.apache.org/repos/asf/airavata.git.
from fd76113 Further fixes for https://github.com/spotify/docker-client/issues/513
new ff82311 Updating sshj client
new ad8b7cc Logging improvements for parser wm
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../airavata-helix/agent-impl/sshj-agent/pom.xml | 2 +-
.../helix/impl/workflow/ParserWorkflowManager.java | 35 +++++++++++-----------
2 files changed, 18 insertions(+), 19 deletions(-)
[airavata] 02/02: Logging improvements for parser wm
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch param-sweep
in repository https://gitbox.apache.org/repos/asf/airavata.git
commit ad8b7cc27988dcd2a5feaeeb3794ddaf99000680
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Wed Sep 2 11:40:18 2020 -0400
Logging improvements for parser wm
---
.../helix/impl/workflow/ParserWorkflowManager.java | 35 +++++++++++-----------
1 file changed, 17 insertions(+), 18 deletions(-)
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/workflow/ParserWorkflowManager.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/workflow/ParserWorkflowManager.java
index 9f562e5..00945ff 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/workflow/ParserWorkflowManager.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/workflow/ParserWorkflowManager.java
@@ -88,22 +88,20 @@ public class ParserWorkflowManager extends WorkflowManager {
try {
ProcessModel processModel;
- ApplicationInterfaceDescription appDescription;
try {
processModel = registryClient.getProcess(completionMessage.getProcessId());
- appDescription = registryClient.getApplicationInterface(processModel.getApplicationInterfaceId());
-
} catch (Exception e) {
logger.error("Failed to fetch process or application description from registry associated with process id " + completionMessage.getProcessId(), e);
throw new Exception("Failed to fetch process or application description from registry associated with process id " + completionMessage.getProcessId(), e);
}
+ logger.info("Fetching parsing templates for thr experiment {} and the output version {}", completionMessage.getExperimentId(), completionMessage.getOutputVersion());
// All the templates should be run
// FIXME is it ApplicationInterfaceId or ApplicationName
List<ParsingTemplate> parsingTemplates = registryClient.getParsingTemplatesForExperiment(completionMessage.getExperimentId(),
completionMessage.getGatewayId());
- logger.info("Found " + parsingTemplates.size() + " parsing template for experiment " + completionMessage.getExperimentId());
+ logger.info("Found {} parsing template for experiment {}", parsingTemplates.size(), completionMessage.getExperimentId());
Map<String, Map<String, Set<ParserConnector>>> parentToChildParsers = new HashMap<>();
@@ -132,7 +130,7 @@ public class ParserWorkflowManager extends WorkflowManager {
for (ParsingTemplate template : parsingTemplates) {
- logger.info("Launching parsing template " + template.getId());
+ logger.info("Loading inputs for template {}", template.getId());
ParserInput parserInput = registryClient.getParserInput(template.getInitialInputs().get(0).getTargetInputId(), template.getGatewayId());
String parentParserId = parserInput.getParserId();
@@ -157,26 +155,32 @@ public class ParserWorkflowManager extends WorkflowManager {
throw new Exception("Could not find a parent parser for template " + template.getId());
}
+ logger.info("Loading the parent parser for template {}", template.getId());
Parser parentParser = registryClient.getParser(parentParserId, completionMessage.getGatewayId());
+ logger.info("Parent parser is {} for template {}", parentParser.getId(), template.getId());
+
DataParsingTask parentParserTask = createParentTask(parentParser, completionMessage, template.getInitialInputs(), registryClient);
List<AbstractTask> allTasks = new ArrayList<>();
allTasks.add(parentParserTask);
if (parentToChildParsers.containsKey(template.getId())) {
- createParserDagRecursively(allTasks, parentParser, parentParserTask, parentToChildParsers.get(template.getId()), completionMessage, registryClient);
+ logger.info("Creating the child parser dag recursively for template {}", template.getId());
+ createParserDagRecursively(allTasks, parentParser, parentParserTask,
+ parentToChildParsers.get(template.getId()), completionMessage, processModel, registryClient);
}
+
+ logger.info("Launching the parsing template {}", template.getId());
+
String workflow = getWorkflowOperator().launchWorkflow("Parser-" + completionMessage.getProcessId() + UUID.randomUUID().toString(),
allTasks, true, false);
// TODO: figure out processId and register
// registerWorkflowForProcess(processId, workflow, "PARSER");
- logger.info("Launched workflow " + workflow);
+ logger.info("Launched workflow {} for parsing template {} for experiment {}", workflow, template.getId(), completionMessage.getExperimentId());
}
getRegistryClientPool().returnResource(registryClient);
-
-
return true;
@@ -321,10 +325,11 @@ public class ParserWorkflowManager extends WorkflowManager {
}
private void createParserDagRecursively(List<AbstractTask> allTasks, Parser parentParserInfo, DataParsingTask parentTask, Map<String, Set<ParserConnector>> parentToChild,
- ProcessCompletionMessage completionMessage, RegistryService.Client registryClient) throws Exception {
+ ProcessCompletionMessage completionMessage, ProcessModel processModel, RegistryService.Client registryClient) throws Exception {
if (parentToChild.containsKey(parentParserInfo.getId())) {
for (ParserConnector connector : parentToChild.get(parentParserInfo.getId())) {
+ logger.info("Loading the child parser {}", connector.getChildParserId());
Parser childParserInfo = registryClient.getParser(connector.getChildParserId(), completionMessage.getGatewayId());
DataParsingTask parsingTask = new DataParsingTask();
parsingTask.setTaskId(normalizeTaskId(completionMessage.getExperimentId() + "-" + childParserInfo.getId() + "-" + UUID.randomUUID().toString()));
@@ -333,12 +338,7 @@ public class ParserWorkflowManager extends WorkflowManager {
parsingTask.setLocalDataDir("/tmp");
parentTask.setProcessId(completionMessage.getProcessId());
parentTask.setOutputVersion(completionMessage.getOutputVersion());
- try {
- parsingTask.setGroupResourceProfileId(registryClient.getProcess(completionMessage.getProcessId()).getGroupResourceProfileId());
- } catch (TException e) {
- logger.error("Failed while fetching process model for process id " + completionMessage.getProcessId());
- throw new Exception("Failed while fetching process model for process id " + completionMessage.getProcessId());
- }
+ parsingTask.setGroupResourceProfileId(processModel.getGroupResourceProfileId());
ParsingTaskInputs inputs = new ParsingTaskInputs();
for(ParserConnectorInput connectorInput : connector.getConnectorInputs()) {
@@ -377,7 +377,7 @@ public class ParserWorkflowManager extends WorkflowManager {
parentTask.setNextTask(new OutPort(parsingTask.getTaskId(), parentTask));
allTasks.add(parsingTask);
- createParserDagRecursively(allTasks, childParserInfo, parsingTask, parentToChild, completionMessage, registryClient);
+ createParserDagRecursively(allTasks, childParserInfo, parsingTask, parentToChild, completionMessage, processModel, registryClient);
}
}
}
@@ -415,5 +415,4 @@ public class ParserWorkflowManager extends WorkflowManager {
consumer.commitAsync();
}
}
-
}
[airavata] 01/02: Updating sshj client
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch param-sweep
in repository https://gitbox.apache.org/repos/asf/airavata.git
commit ff823118247422fe76270a490aea8109263a0c9c
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Wed Sep 2 11:39:56 2020 -0400
Updating sshj client
---
modules/airavata-helix/agent-impl/sshj-agent/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/modules/airavata-helix/agent-impl/sshj-agent/pom.xml b/modules/airavata-helix/agent-impl/sshj-agent/pom.xml
index aa689b2..2a67379 100644
--- a/modules/airavata-helix/agent-impl/sshj-agent/pom.xml
+++ b/modules/airavata-helix/agent-impl/sshj-agent/pom.xml
@@ -34,7 +34,7 @@
<dependency>
<groupId>com.hierynomus</groupId>
<artifactId>sshj</artifactId>
- <version>0.23.0</version>
+ <version>0.30.0</version>
</dependency>
<dependency>
<groupId>org.apache.airavata</groupId>