You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@taverna.apache.org by st...@apache.org on 2018/08/06 15:22:56 UTC
[08/13] incubator-taverna-language git commit: Clean up and add tests
for the nested workflows
Clean up and add tests for the nested workflows
Project: http://git-wip-us.apache.org/repos/asf/incubator-taverna-language/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-taverna-language/commit/8aa74225
Tree: http://git-wip-us.apache.org/repos/asf/incubator-taverna-language/tree/8aa74225
Diff: http://git-wip-us.apache.org/repos/asf/incubator-taverna-language/diff/8aa74225
Branch: refs/heads/master
Commit: 8aa7422573e81b580a5bfaed9a1ed7da7b37b930
Parents: 5557bc6
Author: Majdi Haouech <m....@criteo.com>
Authored: Fri Jul 27 00:09:04 2018 +0200
Committer: Majdi Haouech <m....@criteo.com>
Committed: Fri Jul 27 00:09:04 2018 +0200
----------------------------------------------------------------------
.../apache/taverna/scufl2/cwl/Converter.java | 4 -
.../scufl2/cwl/components/CommandLineTool.java | 4 +
.../scufl2/cwl/components/WorkflowProcess.java | 77 ++++++++--------
.../taverna/scufl2/cwl/TestWorkflowNesting.java | 95 ++++++++++++++++++++
.../taverna/scufl2/cwl/TestWorkflowProcess.java | 2 +-
.../test/resources/workflow_with_workflow.cwl | 48 ++++++++++
.../src/test/resources/worklflow2.cwl | 44 ---------
7 files changed, 186 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-taverna-language/blob/8aa74225/taverna-scufl2-cwl/src/main/java/org/apache/taverna/scufl2/cwl/Converter.java
----------------------------------------------------------------------
diff --git a/taverna-scufl2-cwl/src/main/java/org/apache/taverna/scufl2/cwl/Converter.java b/taverna-scufl2-cwl/src/main/java/org/apache/taverna/scufl2/cwl/Converter.java
index eff7ef6..687a278 100644
--- a/taverna-scufl2-cwl/src/main/java/org/apache/taverna/scufl2/cwl/Converter.java
+++ b/taverna-scufl2-cwl/src/main/java/org/apache/taverna/scufl2/cwl/Converter.java
@@ -81,10 +81,6 @@ public class Converter {
}
public JsonNode convertWorkflowProcessToJsonNode(WorkflowProcess workflow) {
- Map<String, Processor> workflowProcessors = workflow.getWorkflowProcessors();
- Map<String, InputProcessorPort> processorInputs = workflow.getProcessorInputs();
- Map<String, OutputProcessorPort> processorOutputs = workflow.getProcessorOutputs();
-
ObjectNode result = jsonNodeFactory.objectNode();
ObjectNode inputs = convertInputWorkflows(workflow.getWorkflowInputs());
ObjectNode outputs = convertOutputWorkflows(workflow.getWorkflowOutputs());
http://git-wip-us.apache.org/repos/asf/incubator-taverna-language/blob/8aa74225/taverna-scufl2-cwl/src/main/java/org/apache/taverna/scufl2/cwl/components/CommandLineTool.java
----------------------------------------------------------------------
diff --git a/taverna-scufl2-cwl/src/main/java/org/apache/taverna/scufl2/cwl/components/CommandLineTool.java b/taverna-scufl2-cwl/src/main/java/org/apache/taverna/scufl2/cwl/components/CommandLineTool.java
index ca551fa..70d9098 100644
--- a/taverna-scufl2-cwl/src/main/java/org/apache/taverna/scufl2/cwl/components/CommandLineTool.java
+++ b/taverna-scufl2-cwl/src/main/java/org/apache/taverna/scufl2/cwl/components/CommandLineTool.java
@@ -74,4 +74,8 @@ public class CommandLineTool implements Process {
processorOutputs.put(portId, port);
}
}
+
+ public String getBaseCommand() {
+ return baseCommand;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-taverna-language/blob/8aa74225/taverna-scufl2-cwl/src/main/java/org/apache/taverna/scufl2/cwl/components/WorkflowProcess.java
----------------------------------------------------------------------
diff --git a/taverna-scufl2-cwl/src/main/java/org/apache/taverna/scufl2/cwl/components/WorkflowProcess.java b/taverna-scufl2-cwl/src/main/java/org/apache/taverna/scufl2/cwl/components/WorkflowProcess.java
index 481c93f..1b660dd 100644
--- a/taverna-scufl2-cwl/src/main/java/org/apache/taverna/scufl2/cwl/components/WorkflowProcess.java
+++ b/taverna-scufl2-cwl/src/main/java/org/apache/taverna/scufl2/cwl/components/WorkflowProcess.java
@@ -19,8 +19,6 @@
package org.apache.taverna.scufl2.cwl;
-import java.io.File;
-import java.io.IOException;
import java.util.Set;
import java.util.HashSet;
import java.util.Map;
@@ -31,7 +29,6 @@ import org.apache.taverna.scufl2.api.core.Processor;
import org.apache.taverna.scufl2.api.core.DataLink;
import org.apache.taverna.scufl2.api.core.Workflow;
-import org.apache.taverna.scufl2.api.container.WorkflowBundle;
import org.apache.taverna.scufl2.api.port.InputWorkflowPort;
import org.apache.taverna.scufl2.api.port.OutputWorkflowPort;
import org.apache.taverna.scufl2.api.port.InputProcessorPort;
@@ -39,9 +36,6 @@ import org.apache.taverna.scufl2.api.port.OutputProcessorPort;
import org.apache.taverna.scufl2.api.port.SenderPort;
import org.apache.taverna.scufl2.api.port.ReceiverPort;
-import org.apache.taverna.scufl2.api.io.WorkflowBundleIO;
-import org.apache.taverna.scufl2.api.io.WriterException;
-
import com.fasterxml.jackson.databind.JsonNode;
public class WorkflowProcess implements Process {
@@ -55,6 +49,8 @@ public class WorkflowProcess implements Process {
private Map<String, OutputProcessorPort> processorOutputs = new HashMap<>();
private Set<DataLink> dataLinks = new HashSet<>();
+ private Set<Process> processes = new HashSet<>();
+
private Converter converter = new Converter();
public WorkflowProcess(JsonNode node) {
@@ -66,25 +62,9 @@ public class WorkflowProcess implements Process {
parseInputs();
parseOutputs();
Set<Step> cwlSteps = cwlParser.parseSteps();
- parseProcessors(cwlSteps);
- parseDataLinks(cwlSteps);
-
- Workflow workflow = new Workflow();
- Set<InputWorkflowPort> inputs = new HashSet<>(workflowInputs.values());
- Set<OutputWorkflowPort> outputs = new HashSet<>(workflowOutputs.values());
- Set<Processor> processors = new HashSet<>(workflowProcessors.values());
-
- workflow.setInputPorts(inputs);
- workflow.setOutputPorts(outputs);
- workflow.setProcessors(processors);
-
-
-// System.out.println("DEBUG WORKFLOW");
-// System.out.println(workflow.getInputPorts());
-// System.out.println(workflow.getOutputPorts());
-// System.out.println(workflow.getProcessors());
-
+ this.processes = parseProcessors(cwlSteps);
}
+
public void parseInputs() {
Set<PortDetail> cwlInputs = cwlParser.parseInputs();
for (PortDetail port: cwlInputs) {
@@ -103,23 +83,34 @@ public class WorkflowProcess implements Process {
}
}
- public void parseProcessors(Set<Step> cwlSteps) {
- for(Step step: cwlSteps) {
-
- Processor processor = converter.convertStepToProcessor(step);
- workflowProcessors.put(step.getId(), processor);
+ public Set<Process> parseProcessors(Set<Step> cwlSteps) {
+ Set<Process> result = new HashSet<>();
- // TODO: Add only receiver and sender ports from the Process interface
- for(StepInput stepInput: step.getInputs()) {
- InputProcessorPort processorPort = new InputProcessorPort(processor, stepInput.getId());
- processorInputs.put(stepInput.getId(), processorPort);
- }
- for(StepOutput stepOutput: step.getOutputs()) {
- OutputProcessorPort processorPort = new OutputProcessorPort(processor, stepOutput.getId());
- processorOutputs.put(stepOutput.getId(), processorPort);
- }
+ for(Step step: cwlSteps) {
+ Process process = step.getRun();
+ result.add(process);
}
- }
+
+ return result;
+ }
+
+// public void parseProcessors(Set<Step> cwlSteps) {
+// for(Step step: cwlSteps) {
+//
+// Processor processor = converter.convertStepToProcessor(step);
+// workflowProcessors.put(step.getId(), processor);
+//
+// // TODO: Add only receiver and sender ports from the Process interface
+// for(StepInput stepInput: step.getInputs()) {
+// InputProcessorPort processorPort = new InputProcessorPort(processor, stepInput.getId());
+// processorInputs.put(stepInput.getId(), processorPort);
+// }
+// for(StepOutput stepOutput: step.getOutputs()) {
+// OutputProcessorPort processorPort = new OutputProcessorPort(processor, stepOutput.getId());
+// processorOutputs.put(stepOutput.getId(), processorPort);
+// }
+// }
+// }
public void parseDataLinks(Set<Step> cwlSteps) {
for(Step step: cwlSteps) {
@@ -198,4 +189,12 @@ public class WorkflowProcess implements Process {
public void setDataLinks(Set<DataLink> dataLinks) {
this.dataLinks = dataLinks;
}
+
+ public Set<Process> getProcesses() {
+ return processes;
+ }
+
+ public void setProcesses(Set<Process> processes) {
+ this.processes = processes;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-taverna-language/blob/8aa74225/taverna-scufl2-cwl/src/test/java/org/apache/taverna/scufl2/cwl/TestWorkflowNesting.java
----------------------------------------------------------------------
diff --git a/taverna-scufl2-cwl/src/test/java/org/apache/taverna/scufl2/cwl/TestWorkflowNesting.java b/taverna-scufl2-cwl/src/test/java/org/apache/taverna/scufl2/cwl/TestWorkflowNesting.java
new file mode 100644
index 0000000..d0f3e88
--- /dev/null
+++ b/taverna-scufl2-cwl/src/test/java/org/apache/taverna/scufl2/cwl/TestWorkflowNesting.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.taverna.scufl2.cwl;
+
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.HashMap;
+
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
+import org.yaml.snakeyaml.Yaml;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+
+import org.apache.taverna.scufl2.api.core.Workflow;
+
+import org.apache.taverna.scufl2.api.port.InputWorkflowPort;
+import org.apache.taverna.scufl2.api.port.OutputWorkflowPort;
+
+import org.apache.taverna.scufl2.api.io.WorkflowBundleIO;
+import org.apache.taverna.scufl2.api.io.WriterException;
+
+import org.apache.taverna.scufl2.api.container.WorkflowBundle;
+
+public class TestWorkflowNesting {
+ private static final String HELLO_WORLD_CWL = "/hello_world.cwl";
+ private static final String WORKFLOW_WITH_COMMAND = "/workflow_with_command.cwl";
+ private static final String WORKFLOW_WITH_WORKFLOW = "/workflow_with_workflow.cwl";
+
+ private static JsonNode cwlFile;
+
+ public JsonNode loadYamlFile(String filename) {
+
+ Yaml reader = new Yaml();
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode node = mapper.valueToTree(reader.load(TestWorkflowProcess.class.getResourceAsStream(filename)));
+
+ return node;
+ }
+
+ @Test
+ public void testWorkflowWithProcessor() {
+ JsonNode cwlFile = loadYamlFile(WORKFLOW_WITH_COMMAND);
+
+ WorkflowProcess workflow = (WorkflowProcess) ProcessFactory.createProcess(cwlFile);
+ Set<Process> processes = workflow.getProcesses();
+ assertEquals(processes.size(), 1);
+ Process child = processes.iterator().next();
+ assert(child instanceof CommandLineTool);
+ assertEquals(((CommandLineTool) child).getBaseCommand(), "echo");
+ }
+
+ @Test
+ public void testWorkflowWithWorkflow() {
+ JsonNode cwlFile = loadYamlFile(WORKFLOW_WITH_WORKFLOW);
+
+ WorkflowProcess workflow = (WorkflowProcess) ProcessFactory.createProcess(cwlFile);
+ Set<Process> processes = workflow.getProcesses();
+ assert(processes.size() == 1);
+ Process child = processes.iterator().next();
+ assert(child instanceof WorkflowProcess);
+ WorkflowProcess childWorkflow = (WorkflowProcess) child;
+ assertEquals(childWorkflow.getProcesses().size(), 1);
+
+ Process reference = childWorkflow.getProcesses().iterator().next();
+
+ assert(reference instanceof Reference);
+ assertEquals(((Reference) reference).toString(), "example.cwl");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-taverna-language/blob/8aa74225/taverna-scufl2-cwl/src/test/java/org/apache/taverna/scufl2/cwl/TestWorkflowProcess.java
----------------------------------------------------------------------
diff --git a/taverna-scufl2-cwl/src/test/java/org/apache/taverna/scufl2/cwl/TestWorkflowProcess.java b/taverna-scufl2-cwl/src/test/java/org/apache/taverna/scufl2/cwl/TestWorkflowProcess.java
index fc386ac..4b3bba7 100644
--- a/taverna-scufl2-cwl/src/test/java/org/apache/taverna/scufl2/cwl/TestWorkflowProcess.java
+++ b/taverna-scufl2-cwl/src/test/java/org/apache/taverna/scufl2/cwl/TestWorkflowProcess.java
@@ -91,7 +91,7 @@ public class TestWorkflowProcess {
}
@Test
- public void testWorkflowSteps() {
+ public void testProcessType() {
cwlFile = loadYamlFile(WORKFLOW_WITH_COMMAND);
Process workflow = ProcessFactory.createProcess(cwlFile);
http://git-wip-us.apache.org/repos/asf/incubator-taverna-language/blob/8aa74225/taverna-scufl2-cwl/src/test/resources/workflow_with_workflow.cwl
----------------------------------------------------------------------
diff --git a/taverna-scufl2-cwl/src/test/resources/workflow_with_workflow.cwl b/taverna-scufl2-cwl/src/test/resources/workflow_with_workflow.cwl
new file mode 100644
index 0000000..58323c5
--- /dev/null
+++ b/taverna-scufl2-cwl/src/test/resources/workflow_with_workflow.cwl
@@ -0,0 +1,48 @@
+#!/usr/bin/env cwl-runner
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+cwlVersion: v1.0
+class: Workflow
+
+inputs:
+ message: string
+
+outputs:
+ download:
+ type: File
+ outputSource: "#step1/curl"
+
+steps:
+ step1:
+ run:
+ class: Workflow
+ inputs:
+ name: string
+ outputs:
+ output1: string
+ steps:
+ step2:
+ run: example.cwl
+ inputs:
+ - id: text
+ source: "#x/name"
+ outputs: []
+ in:
+ text: message
+
+ out: [curl]
http://git-wip-us.apache.org/repos/asf/incubator-taverna-language/blob/8aa74225/taverna-scufl2-cwl/src/test/resources/worklflow2.cwl
----------------------------------------------------------------------
diff --git a/taverna-scufl2-cwl/src/test/resources/worklflow2.cwl b/taverna-scufl2-cwl/src/test/resources/worklflow2.cwl
deleted file mode 100644
index fc5e846..0000000
--- a/taverna-scufl2-cwl/src/test/resources/worklflow2.cwl
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env cwl-runner
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-cwlVersion: v1.0
-class: Workflow
-
-inputs:
- message: string
-
-outputs:
- download:
- type: File
- outputSource: "#step1/curl"
-
-steps:
- step1:
- run:
- class: CommandLineTool
- baseCommand: echo
- inputs:
- text:
- type: string
- inputBinding:
- position: 1
- outputs: []
- in:
- text: message
-
- out: [curl]