You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2021/08/11 11:13:25 UTC
[airavata-data-lake] branch master updated: Workflow distribution
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
The following commit(s) were added to refs/heads/master by this push:
new 65d01f2 Workflow distribution
65d01f2 is described below
commit 65d01f2e2600a976acdbdcb95ec6ffd8fc6a30e1
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Wed Aug 11 07:13:14 2021 -0400
Workflow distribution
---
.../datasync-workflow-manager/pom.xml | 23 +++++
.../src/main/dist/bin/setenv.sh | 46 +++++++++
.../src/main/dist/bin/wf-manager-daemon.sh | 113 +++++++++++++++++++++
.../src/main/dist/bin/wf-manager.sh | 71 +++++++++++++
.../src/main/dist/conf/application.properties | 48 +++++++++
.../src/main/dist/conf/logback.xml | 50 +++++++++
.../src/main/dist/wf-manager-assembly.xml | 85 ++++++++++++++++
.../wm/datasync/DataParsingWorkflowManager.java | 6 +-
.../workflow-engine-controller/pom.xml | 23 +++++
.../src/main/dist/bin/setenv.sh | 46 +++++++++
.../src/main/dist/bin/wf-controller-daemon.sh | 113 +++++++++++++++++++++
.../src/main/dist/bin/wf-controller.sh | 71 +++++++++++++
.../src/main/dist/conf/application.properties | 26 +++++
.../src/main/dist/conf/logback.xml | 50 +++++++++
.../src/main/dist/wf-controller-assembly.xml | 85 ++++++++++++++++
.../src/main/resources/logback.xml | 50 +++++++++
.../workflow-engine/workflow-engine-worker/pom.xml | 23 +++++
.../src/main/dist/bin/setenv.sh | 46 +++++++++
.../src/main/dist/bin/wf-worker-daemon.sh | 113 +++++++++++++++++++++
.../src/main/dist/bin/wf-worker.sh | 71 +++++++++++++
.../src/main/dist/conf/application.properties | 26 +++++
.../src/main/dist/conf/logback.xml | 50 +++++++++
.../src/main/dist/conf/task-list.yaml | 10 ++
.../src/main/dist/wf-worker-assembly.xml | 85 ++++++++++++++++
24 files changed, 1328 insertions(+), 2 deletions(-)
diff --git a/data-orchestrator/workflow-engine/datasync-workflow-manager/pom.xml b/data-orchestrator/workflow-engine/datasync-workflow-manager/pom.xml
index 59ecffd..2b1486b 100644
--- a/data-orchestrator/workflow-engine/datasync-workflow-manager/pom.xml
+++ b/data-orchestrator/workflow-engine/datasync-workflow-manager/pom.xml
@@ -63,6 +63,29 @@
<artifactId>spring-boot-maven-plugin</artifactId>
<version>2.4.1</version>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven.assembly.plugin}</version>
+ <executions>
+ <execution>
+ <id>wf-manager-package</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <tarLongFileMode>posix</tarLongFileMode>
+ <finalName>WF-Manager-${project.version}</finalName>
+ <descriptors>
+ <descriptor>src/main/dist/wf-manager-assembly.xml</descriptor>
+ </descriptors>
+ <attach>false</attach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
diff --git a/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/dist/bin/setenv.sh b/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/dist/bin/setenv.sh
new file mode 100755
index 0000000..9e894e1
--- /dev/null
+++ b/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/dist/bin/setenv.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# resolve links - $0 may be a softlink
+PRG="$0"
+
+while [ -h "$PRG" ]; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '.*/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "$PRG"`/"$link"
+ fi
+done
+
+PRGDIR=`dirname "$PRG"`
+
+# Only set AIRAVATA_HOME if not already set
+[ -z "$AIRAVATA_HOME" ] && AIRAVATA_HOME=`cd "$PRGDIR/.." ; pwd`
+
+AIRAVATA_CLASSPATH=""
+
+for f in "$AIRAVATA_HOME"/lib/*.jar
+do
+ AIRAVATA_CLASSPATH="$AIRAVATA_CLASSPATH":$f
+done
+
+export AIRAVATA_HOME
+export AIRAVATA_CLASSPATH
diff --git a/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/dist/bin/wf-manager-daemon.sh b/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/dist/bin/wf-manager-daemon.sh
new file mode 100644
index 0000000..36a60f9
--- /dev/null
+++ b/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/dist/bin/wf-manager-daemon.sh
@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+. `dirname $0`/setenv.sh
+# Capture user's working dir before changing directory
+CWD="$PWD"
+cd ${AIRAVATA_HOME}/bin
+LOGO_FILE="logo.txt"
+
+JAVA_OPTS="-Dspring.config.location=${AIRAVATA_HOME}/conf/ -Dairavata.home=${AIRAVATA_HOME} -Dlogback.configurationFile=file:${AIRAVATA_HOME}/conf/logback.xml"
+AIRAVATA_COMMAND=""
+EXTRA_ARGS=""
+SERVERS=""
+LOGO=true
+IS_SUBSET=false
+SUBSET=""
+DEFAULT_LOG_FILE="${AIRAVATA_HOME}/logs/airavata-daemon.out"
+LOG_FILE=$DEFAULT_LOG_FILE
+
+SERVICE_NAME="Workflow Manager"
+PID_PATH_NAME="${AIRAVATA_HOME}/bin/service-pid"
+
+case $1 in
+ start)
+ echo "Starting $SERVICE_NAME ..."
+ if [ ! -f $PID_PATH_NAME ]; then
+ nohup java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.airavata.datalake.workflow.engine.wm.datasync.WorkflowManagerRunner ${AIRAVATA_COMMAND} $* > $LOG_FILE 2>&1 &
+ echo $! > $PID_PATH_NAME
+ echo "$SERVICE_NAME started ..."
+ else
+ echo "$SERVICE_NAME is already running ..."
+ fi
+ ;;
+ stop)
+ if [ -f $PID_PATH_NAME ]; then
+ PID=$(cat $PID_PATH_NAME);
+ echo "$SERVICE_NAME stoping ..."
+ kill $PID;
+ RETRY=0
+ while kill -0 $PID 2> /dev/null; do
+ echo "Waiting for the process $PID to be stopped"
+ RETRY=`expr ${RETRY} + 1`
+ if [ "${RETRY}" -gt "20" ]
+ then
+ echo "Forcefully killing the process as it is not responding ..."
+ kill -9 $PID
+ fi
+ sleep 1
+ done
+ echo "$SERVICE_NAME stopped ..."
+ rm $PID_PATH_NAME
+ else
+ echo "$SERVICE_NAME is not running ..."
+ fi
+ ;;
+ restart)
+ if [ -f $PID_PATH_NAME ]; then
+ PID=$(cat $PID_PATH_NAME);
+ echo "$SERVICE_NAME stopping ...";
+ kill $PID;
+ RETRY=0
+ while kill -0 $PID 2> /dev/null; do
+ echo "Waiting for the process $PID to be stopped"
+ RETRY=`expr ${RETRY} + 1`
+ if [ "${RETRY}" -gt "20" ]
+ then
+ echo "Forcefully killing the process as it is not responding ..."
+ kill -9 $PID
+ fi
+ sleep 1
+ done
+ echo "$SERVICE_NAME stopped ...";
+ rm $PID_PATH_NAME
+ echo "$SERVICE_NAME starting ..."
+ nohup java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.airavata.datalake.workflow.engine.wm.datasync.WorkflowManagerRunner ${AIRAVATA_COMMAND} $* > $LOG_FILE 2>&1 &
+ echo $! > $PID_PATH_NAME
+ echo "$SERVICE_NAME started ..."
+ else
+ echo "$SERVICE_NAME is not running ..."
+ fi
+ ;;
+ -h)
+ echo "Usage: wf-manager-daemon.sh"
+
+ echo "command options:"
+ echo " start Start server in daemon mode"
+ echo " stop Stop server running in daemon mode"
+ echo " restart Restart server in daemon mode"
+ echo " -log <LOG_FILE> Where to redirect stdout/stderr (defaults to $DEFAULT_LOG_FILE)"
+ echo " -h Display this help and exit"
+ shift
+ exit 0
+ ;;
+esac
+
diff --git a/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/dist/bin/wf-manager.sh b/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/dist/bin/wf-manager.sh
new file mode 100644
index 0000000..36f682c
--- /dev/null
+++ b/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/dist/bin/wf-manager.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+. `dirname $0`/setenv.sh
+# Capture user's working dir before changing directory
+CWD="$PWD"
+cd ${AIRAVATA_HOME}/bin
+LOGO_FILE="logo.txt"
+
+JAVA_OPTS="-Dspring.config.location=${AIRAVATA_HOME}/conf/ -Dairavata.home=${AIRAVATA_HOME} -Dlogback.configurationFile=file:${AIRAVATA_HOME}/conf/logback.xml"
+AIRAVATA_COMMAND=""
+EXTRA_ARGS=""
+SERVERS=""
+IS_SUBSET=false
+SUBSET=""
+DEFAULT_LOG_FILE="${AIRAVATA_HOME}/logs/console.out"
+LOG_FILE=$DEFAULT_LOG_FILE
+
+# parse command arguments
+for var in "$@"
+do
+ case ${var} in
+ -xdebug)
+ AIRAVATA_COMMAND="${AIRAVATA_COMMAND}"
+ JAVA_OPTS="$JAVA_OPTS -Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,server=y,address=8000"
+ shift
+ ;;
+ -log)
+ shift
+ LOG_FILE="$1"
+ shift
+ # If relative path, expand to absolute path using the user's $CWD
+ if [ -z "`echo "$LOG_FILE" | egrep "^/"`" ]; then
+ LOG_FILE="${CWD}/${LOG_FILE}"
+ fi
+ ;;
+ -h)
+ echo "Usage: wf-manager.sh"
+
+ echo "command options:"
+ echo " -xdebug Start Workflow Manager under JPDA debugger"
+ echo " -h Display this help and exit"
+ shift
+ exit 0
+ ;;
+ *)
+ EXTRA_ARGS="${EXTRA_ARGS} ${var}"
+ shift
+ ;;
+ esac
+done
+
+java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.airavata.datalake.workflow.engine.wm.datasync.WorkflowManagerRunner ${AIRAVATA_COMMAND} $*
+
diff --git a/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/dist/conf/application.properties b/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/dist/conf/application.properties
new file mode 100644
index 0000000..d6521a1
--- /dev/null
+++ b/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/dist/conf/application.properties
@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+cluster.name=datalake
+zookeeper.connection=localhost:2181
+
+datasync.wm.name=datasync_wf
+parsing.wm.name=parsing_wf
+
+kafka.url=localhost:9092
+kafka.mft.publisher.name=mft-status-publisher
+kafka.mft.status.publish.topic=mft-status-topic
+kafka.mft.status.consumer.group=mft-even-group
+
+datasync.wm.grpc.host=localhost
+datasync.wm.grpc.port=6565
+spring.config.use-legacy-processing=true
+
+mft.callback.url=localhost:33335
+mft.host=localhost
+mft.port=7004
+mft.clientId=CHANGE_ME
+mft.clientSecret=CHANGE_ME
+
+custos.host=custos.scigap.org
+custos.port=31499
+custos.id=CHANGE_ME
+custos.secret=CHANGE_ME
+
+drms.host=localhost
+drms.port=7070
+server.port=88991
\ No newline at end of file
diff --git a/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/dist/conf/logback.xml b/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/dist/conf/logback.xml
new file mode 100644
index 0000000..3afe661
--- /dev/null
+++ b/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/dist/conf/logback.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<configuration>
+
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d [%t] %-5p %c{30} %m [%X]%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="LOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <File>../logs/airavata.log</File>
+ <Append>true</Append>
+ <encoder>
+ <pattern>%d [%t] %-5p %c{30} %m [%X]%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>../logs/airavata.log.%d{yyyy-MM-dd}</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ <totalSizeCap>1GB</totalSizeCap>
+ </rollingPolicy>
+ </appender>
+
+ <logger name="ch.qos.logback" level="WARN"/>
+ <logger name="org.apache.helix" level="WARN"/>
+ <logger name="org.apache.zookeeper" level="ERROR"/>
+ <logger name="org.apache.helix" level="ERROR"/>
+ <logger name="org.apache.airavata" level="INFO"/>
+ <logger name="org.hibernate" level="ERROR"/>
+ <logger name="net.schmizz.sshj" level="WARN"/>
+ <root level="INFO">
+ <appender-ref ref="CONSOLE"/>
+ <appender-ref ref="LOGFILE"/>
+ </root>
+</configuration>
\ No newline at end of file
diff --git a/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/dist/wf-manager-assembly.xml b/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/dist/wf-manager-assembly.xml
new file mode 100644
index 0000000..551916c
--- /dev/null
+++ b/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/dist/wf-manager-assembly.xml
@@ -0,0 +1,85 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<!DOCTYPE assembly [
+ <!ELEMENT assembly (id|includeBaseDirectory|baseDirectory|formats|fileSets|dependencySets)*>
+ <!ELEMENT id (#PCDATA)>
+ <!ELEMENT includeBaseDirectory (#PCDATA)>
+ <!ELEMENT baseDirectory (#PCDATA)>
+ <!ELEMENT formats (format)*>
+ <!ELEMENT format (#PCDATA)>
+ <!ELEMENT fileSets (fileSet)*>
+ <!ELEMENT fileSet (directory|outputDirectory|fileMode|includes)*>
+ <!ELEMENT directory (#PCDATA)>
+ <!ELEMENT outputDirectory (#PCDATA)>
+ <!ELEMENT includes (include)*>
+ <!ELEMENT include (#PCDATA)>
+ <!ELEMENT dependencySets (dependencySet)*>
+ <!ELEMENT dependencySet (outputDirectory|outputFileNameMapping|includes)*>
+ ]>
+<assembly>
+
+ <id>bin</id>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>WF-Manager-${project.version}</baseDirectory>
+ <formats>
+ <format>tar.gz</format>
+ <format>zip</format>
+ </formats>
+
+ <fileSets>
+ <fileSet>
+ <directory>src/main/dist/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>777</fileMode>
+ <includes>
+ <include>*.sh</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/dist/conf</directory>
+ <outputDirectory>conf</outputDirectory>
+ <includes>
+ <include>application.properties</include>
+ <include>logback.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>./</directory>
+ <outputDirectory>logs</outputDirectory>
+ <excludes>
+ <exclude>*/**</exclude>
+ </excludes>
+ </fileSet>
+ <fileSet>
+ <directory>target</directory>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>*.jar</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>*</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
\ No newline at end of file
diff --git a/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/java/org/apache/airavata/datalake/workflow/engine/wm/datasync/DataParsingWorkflowManager.java b/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/java/org/apache/airavata/datalake/workflow/engine/wm/datasync/DataParsingWorkflowManager.java
index 75eff4c..2abb3c2 100644
--- a/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/java/org/apache/airavata/datalake/workflow/engine/wm/datasync/DataParsingWorkflowManager.java
+++ b/data-orchestrator/workflow-engine/datasync-workflow-manager/src/main/java/org/apache/airavata/datalake/workflow/engine/wm/datasync/DataParsingWorkflowManager.java
@@ -69,9 +69,11 @@ public class DataParsingWorkflowManager {
@org.springframework.beans.factory.annotation.Value("${drms.port}")
private int drmsPort;
- private String mftClientId = "mft-agent";
+ @org.springframework.beans.factory.annotation.Value("mft.clientId")
+ private String mftClientId;
- private String mftClientSecret = "kHqH27BloDCbLvwUA8ZYRlHcJxXZyby9PB90bTdU";
+ @org.springframework.beans.factory.annotation.Value("mft.clientSecret")
+ private String mftClientSecret;
@Autowired
diff --git a/data-orchestrator/workflow-engine/workflow-engine-controller/pom.xml b/data-orchestrator/workflow-engine/workflow-engine-controller/pom.xml
index e23c6af..eed6f5e 100644
--- a/data-orchestrator/workflow-engine/workflow-engine-controller/pom.xml
+++ b/data-orchestrator/workflow-engine/workflow-engine-controller/pom.xml
@@ -36,6 +36,29 @@
<artifactId>spring-boot-maven-plugin</artifactId>
<version>2.4.1</version>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven.assembly.plugin}</version>
+ <executions>
+ <execution>
+ <id>wf-controller-package</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <tarLongFileMode>posix</tarLongFileMode>
+ <finalName>WF-Controller-${project.version}</finalName>
+ <descriptors>
+ <descriptor>src/main/dist/wf-controller-assembly.xml</descriptor>
+ </descriptors>
+ <attach>false</attach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
\ No newline at end of file
diff --git a/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/dist/bin/setenv.sh b/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/dist/bin/setenv.sh
new file mode 100755
index 0000000..9e894e1
--- /dev/null
+++ b/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/dist/bin/setenv.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# resolve links - $0 may be a softlink
+PRG="$0"
+
+while [ -h "$PRG" ]; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '.*/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "$PRG"`/"$link"
+ fi
+done
+
+PRGDIR=`dirname "$PRG"`
+
+# Only set AIRAVATA_HOME if not already set
+[ -z "$AIRAVATA_HOME" ] && AIRAVATA_HOME=`cd "$PRGDIR/.." ; pwd`
+
+AIRAVATA_CLASSPATH=""
+
+for f in "$AIRAVATA_HOME"/lib/*.jar
+do
+ AIRAVATA_CLASSPATH="$AIRAVATA_CLASSPATH":$f
+done
+
+export AIRAVATA_HOME
+export AIRAVATA_CLASSPATH
diff --git a/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/dist/bin/wf-controller-daemon.sh b/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/dist/bin/wf-controller-daemon.sh
new file mode 100644
index 0000000..0ad0e3a
--- /dev/null
+++ b/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/dist/bin/wf-controller-daemon.sh
@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+. `dirname $0`/setenv.sh
+# Capture user's working dir before changing directory
+CWD="$PWD"
+cd ${AIRAVATA_HOME}/bin
+LOGO_FILE="logo.txt"
+
+JAVA_OPTS="-Dspring.config.location=${AIRAVATA_HOME}/conf/ -Dairavata.home=${AIRAVATA_HOME} -Dlogback.configurationFile=file:${AIRAVATA_HOME}/conf/logback.xml"
+AIRAVATA_COMMAND=""
+EXTRA_ARGS=""
+SERVERS=""
+LOGO=true
+IS_SUBSET=false
+SUBSET=""
+DEFAULT_LOG_FILE="${AIRAVATA_HOME}/logs/airavata-daemon.out"
+LOG_FILE=$DEFAULT_LOG_FILE
+
+SERVICE_NAME="Workflow Controller"
+PID_PATH_NAME="${AIRAVATA_HOME}/bin/service-pid"
+
+case $1 in
+ start)
+ echo "Starting $SERVICE_NAME ..."
+ if [ ! -f $PID_PATH_NAME ]; then
+ nohup java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.airavata.datalake.workflow.engine.controller.Controller ${AIRAVATA_COMMAND} $* > $LOG_FILE 2>&1 &
+ echo $! > $PID_PATH_NAME
+ echo "$SERVICE_NAME started ..."
+ else
+ echo "$SERVICE_NAME is already running ..."
+ fi
+ ;;
+ stop)
+ if [ -f $PID_PATH_NAME ]; then
+ PID=$(cat $PID_PATH_NAME);
+ echo "$SERVICE_NAME stoping ..."
+ kill $PID;
+ RETRY=0
+ while kill -0 $PID 2> /dev/null; do
+ echo "Waiting for the process $PID to be stopped"
+ RETRY=`expr ${RETRY} + 1`
+ if [ "${RETRY}" -gt "20" ]
+ then
+ echo "Forcefully killing the process as it is not responding ..."
+ kill -9 $PID
+ fi
+ sleep 1
+ done
+ echo "$SERVICE_NAME stopped ..."
+ rm $PID_PATH_NAME
+ else
+ echo "$SERVICE_NAME is not running ..."
+ fi
+ ;;
+ restart)
+ if [ -f $PID_PATH_NAME ]; then
+ PID=$(cat $PID_PATH_NAME);
+ echo "$SERVICE_NAME stopping ...";
+ kill $PID;
+ RETRY=0
+ while kill -0 $PID 2> /dev/null; do
+ echo "Waiting for the process $PID to be stopped"
+ RETRY=`expr ${RETRY} + 1`
+ if [ "${RETRY}" -gt "20" ]
+ then
+ echo "Forcefully killing the process as it is not responding ..."
+ kill -9 $PID
+ fi
+ sleep 1
+ done
+ echo "$SERVICE_NAME stopped ...";
+ rm $PID_PATH_NAME
+ echo "$SERVICE_NAME starting ..."
+ nohup java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.airavata.datalake.workflow.engine.controller.Controller ${AIRAVATA_COMMAND} $* > $LOG_FILE 2>&1 &
+ echo $! > $PID_PATH_NAME
+ echo "$SERVICE_NAME started ..."
+ else
+ echo "$SERVICE_NAME is not running ..."
+ fi
+ ;;
+ -h)
+ echo "Usage: wf-controller-daemon.sh"
+
+ echo "command options:"
+ echo " start Start server in daemon mode"
+ echo " stop Stop server running in daemon mode"
+ echo " restart Restart server in daemon mode"
+ echo " -log <LOG_FILE> Where to redirect stdout/stderr (defaults to $DEFAULT_LOG_FILE)"
+ echo " -h Display this help and exit"
+ shift
+ exit 0
+ ;;
+esac
+
diff --git a/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/dist/bin/wf-controller.sh b/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/dist/bin/wf-controller.sh
new file mode 100644
index 0000000..d6c252f
--- /dev/null
+++ b/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/dist/bin/wf-controller.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+. `dirname $0`/setenv.sh
+# Capture user's working dir before changing directory
+CWD="$PWD"
+cd ${AIRAVATA_HOME}/bin
+LOGO_FILE="logo.txt"
+
+JAVA_OPTS="-Dspring.config.location=${AIRAVATA_HOME}/conf/ -Dairavata.home=${AIRAVATA_HOME} -Dlogback.configurationFile=file:${AIRAVATA_HOME}/conf/logback.xml"
+AIRAVATA_COMMAND=""
+EXTRA_ARGS=""
+SERVERS=""
+IS_SUBSET=false
+SUBSET=""
+DEFAULT_LOG_FILE="${AIRAVATA_HOME}/logs/console.out"
+LOG_FILE=$DEFAULT_LOG_FILE
+
+# parse command arguments
+for var in "$@"
+do
+ case ${var} in
+ -xdebug)
+ AIRAVATA_COMMAND="${AIRAVATA_COMMAND}"
+ JAVA_OPTS="$JAVA_OPTS -Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,server=y,address=8000"
+ shift
+ ;;
+ -log)
+ shift
+ LOG_FILE="$1"
+ shift
+ # If relative path, expand to absolute path using the user's $CWD
+ if [ -z "`echo "$LOG_FILE" | egrep "^/"`" ]; then
+ LOG_FILE="${CWD}/${LOG_FILE}"
+ fi
+ ;;
+ -h)
+ echo "Usage: wf-controller.sh"
+
+ echo "command options:"
+ echo " -xdebug Start Workflow Controller under JPDA debugger"
+ echo " -h Display this help and exit"
+ shift
+ exit 0
+ ;;
+ *)
+ EXTRA_ARGS="${EXTRA_ARGS} ${var}"
+ shift
+ ;;
+ esac
+done
+
+java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.airavata.datalake.workflow.engine.controller.Controller ${AIRAVATA_COMMAND} $*
+
diff --git a/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/dist/conf/application.properties b/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/dist/conf/application.properties
new file mode 100644
index 0000000..9caa8a2
--- /dev/null
+++ b/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/dist/conf/application.properties
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+cluster.name=datalake
+zookeeper.connection=localhost:2181
+
+controller.name=helix_controller
+task.list.file=task-list.yaml
+
+server.port=33335
\ No newline at end of file
diff --git a/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/dist/conf/logback.xml b/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/dist/conf/logback.xml
new file mode 100644
index 0000000..3afe661
--- /dev/null
+++ b/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/dist/conf/logback.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<configuration>
+
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d [%t] %-5p %c{30} %m [%X]%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="LOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <File>../logs/airavata.log</File>
+ <Append>true</Append>
+ <encoder>
+ <pattern>%d [%t] %-5p %c{30} %m [%X]%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>../logs/airavata.log.%d{yyyy-MM-dd}</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ <totalSizeCap>1GB</totalSizeCap>
+ </rollingPolicy>
+ </appender>
+
+ <logger name="ch.qos.logback" level="WARN"/>
+ <logger name="org.apache.helix" level="WARN"/>
+ <logger name="org.apache.zookeeper" level="ERROR"/>
+ <logger name="org.apache.helix" level="ERROR"/>
+ <logger name="org.apache.airavata" level="INFO"/>
+ <logger name="org.hibernate" level="ERROR"/>
+ <logger name="net.schmizz.sshj" level="WARN"/>
+ <root level="INFO">
+ <appender-ref ref="CONSOLE"/>
+ <appender-ref ref="LOGFILE"/>
+ </root>
+</configuration>
\ No newline at end of file
diff --git a/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/dist/wf-controller-assembly.xml b/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/dist/wf-controller-assembly.xml
new file mode 100644
index 0000000..800ea0a
--- /dev/null
+++ b/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/dist/wf-controller-assembly.xml
@@ -0,0 +1,85 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<!DOCTYPE assembly [
+ <!ELEMENT assembly (id|includeBaseDirectory|baseDirectory|formats|fileSets|dependencySets)*>
+ <!ELEMENT id (#PCDATA)>
+ <!ELEMENT includeBaseDirectory (#PCDATA)>
+ <!ELEMENT baseDirectory (#PCDATA)>
+ <!ELEMENT formats (format)*>
+ <!ELEMENT format (#PCDATA)>
+ <!ELEMENT fileSets (fileSet)*>
+ <!ELEMENT fileSet (directory|outputDirectory|fileMode|includes)*>
+ <!ELEMENT directory (#PCDATA)>
+ <!ELEMENT outputDirectory (#PCDATA)>
+ <!ELEMENT includes (include)*>
+ <!ELEMENT include (#PCDATA)>
+ <!ELEMENT dependencySets (dependencySet)*>
+ <!ELEMENT dependencySet (outputDirectory|outputFileNameMapping|includes)*>
+ ]>
+<assembly>
+
+ <id>bin</id>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>WF-Controller-${project.version}</baseDirectory>
+ <formats>
+ <format>tar.gz</format>
+ <format>zip</format>
+ </formats>
+
+ <fileSets>
+ <fileSet>
+ <directory>src/main/dist/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>777</fileMode>
+ <includes>
+ <include>*.sh</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/dist/conf</directory>
+ <outputDirectory>conf</outputDirectory>
+ <includes>
+ <include>application.properties</include>
+ <include>logback.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>./</directory>
+ <outputDirectory>logs</outputDirectory>
+ <excludes>
+ <exclude>*/**</exclude>
+ </excludes>
+ </fileSet>
+ <fileSet>
+ <directory>target</directory>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>*.jar</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>*</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
\ No newline at end of file
diff --git a/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/resources/logback.xml b/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/resources/logback.xml
new file mode 100644
index 0000000..3afe661
--- /dev/null
+++ b/data-orchestrator/workflow-engine/workflow-engine-controller/src/main/resources/logback.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<configuration>
+
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d [%t] %-5p %c{30} %m [%X]%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="LOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <File>../logs/airavata.log</File>
+ <Append>true</Append>
+ <encoder>
+ <pattern>%d [%t] %-5p %c{30} %m [%X]%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>../logs/airavata.log.%d{yyyy-MM-dd}</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ <totalSizeCap>1GB</totalSizeCap>
+ </rollingPolicy>
+ </appender>
+
+ <logger name="ch.qos.logback" level="WARN"/>
+ <logger name="org.apache.helix" level="WARN"/>
+ <logger name="org.apache.zookeeper" level="ERROR"/>
+ <logger name="org.apache.helix" level="ERROR"/>
+ <logger name="org.apache.airavata" level="INFO"/>
+ <logger name="org.hibernate" level="ERROR"/>
+ <logger name="net.schmizz.sshj" level="WARN"/>
+ <root level="INFO">
+ <appender-ref ref="CONSOLE"/>
+ <appender-ref ref="LOGFILE"/>
+ </root>
+</configuration>
\ No newline at end of file
diff --git a/data-orchestrator/workflow-engine/workflow-engine-worker/pom.xml b/data-orchestrator/workflow-engine/workflow-engine-worker/pom.xml
index be3d01d..c3420e3 100644
--- a/data-orchestrator/workflow-engine/workflow-engine-worker/pom.xml
+++ b/data-orchestrator/workflow-engine/workflow-engine-worker/pom.xml
@@ -37,6 +37,29 @@
<artifactId>spring-boot-maven-plugin</artifactId>
<version>2.4.1</version>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven.assembly.plugin}</version>
+ <executions>
+ <execution>
+ <id>wf-worker-package</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <tarLongFileMode>posix</tarLongFileMode>
+ <finalName>WF-Worker-${project.version}</finalName>
+ <descriptors>
+ <descriptor>src/main/dist/wf-worker-assembly.xml</descriptor>
+ </descriptors>
+ <attach>false</attach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
diff --git a/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/bin/setenv.sh b/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/bin/setenv.sh
new file mode 100755
index 0000000..9e894e1
--- /dev/null
+++ b/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/bin/setenv.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# resolve links - $0 may be a softlink
+PRG="$0"
+
+while [ -h "$PRG" ]; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '.*/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "$PRG"`/"$link"
+ fi
+done
+
+PRGDIR=`dirname "$PRG"`
+
+# Only set AIRAVATA_HOME if not already set
+[ -z "$AIRAVATA_HOME" ] && AIRAVATA_HOME=`cd "$PRGDIR/.." ; pwd`
+
+AIRAVATA_CLASSPATH=""
+
+for f in "$AIRAVATA_HOME"/lib/*.jar
+do
+ AIRAVATA_CLASSPATH="$AIRAVATA_CLASSPATH":$f
+done
+
+export AIRAVATA_HOME
+export AIRAVATA_CLASSPATH
diff --git a/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/bin/wf-worker-daemon.sh b/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/bin/wf-worker-daemon.sh
new file mode 100644
index 0000000..2340ef6
--- /dev/null
+++ b/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/bin/wf-worker-daemon.sh
@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+. `dirname $0`/setenv.sh
+# Capture user's working dir before changing directory
+CWD="$PWD"
+cd ${AIRAVATA_HOME}/bin
+LOGO_FILE="logo.txt"
+
+JAVA_OPTS="-Dspring.config.location=${AIRAVATA_HOME}/conf/ -Dairavata.home=${AIRAVATA_HOME} -Dlogback.configurationFile=file:${AIRAVATA_HOME}/conf/logback.xml"
+AIRAVATA_COMMAND=""
+EXTRA_ARGS=""
+SERVERS=""
+LOGO=true
+IS_SUBSET=false
+SUBSET=""
+DEFAULT_LOG_FILE="${AIRAVATA_HOME}/logs/airavata-daemon.out"
+LOG_FILE=$DEFAULT_LOG_FILE
+
+SERVICE_NAME="Workflow Worker"
+PID_PATH_NAME="${AIRAVATA_HOME}/bin/service-pid"
+
+case $1 in
+ start)
+ echo "Starting $SERVICE_NAME ..."
+ if [ ! -f $PID_PATH_NAME ]; then
+ nohup java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.airavata.datalake.workflow.engine.worker.Participant ${AIRAVATA_COMMAND} $* > $LOG_FILE 2>&1 &
+ echo $! > $PID_PATH_NAME
+ echo "$SERVICE_NAME started ..."
+ else
+ echo "$SERVICE_NAME is already running ..."
+ fi
+ ;;
+ stop)
+ if [ -f $PID_PATH_NAME ]; then
+ PID=$(cat $PID_PATH_NAME);
+ echo "$SERVICE_NAME stoping ..."
+ kill $PID;
+ RETRY=0
+ while kill -0 $PID 2> /dev/null; do
+ echo "Waiting for the process $PID to be stopped"
+ RETRY=`expr ${RETRY} + 1`
+ if [ "${RETRY}" -gt "20" ]
+ then
+ echo "Forcefully killing the process as it is not responding ..."
+ kill -9 $PID
+ fi
+ sleep 1
+ done
+ echo "$SERVICE_NAME stopped ..."
+ rm $PID_PATH_NAME
+ else
+ echo "$SERVICE_NAME is not running ..."
+ fi
+ ;;
+ restart)
+ if [ -f $PID_PATH_NAME ]; then
+ PID=$(cat $PID_PATH_NAME);
+ echo "$SERVICE_NAME stopping ...";
+ kill $PID;
+ RETRY=0
+ while kill -0 $PID 2> /dev/null; do
+ echo "Waiting for the process $PID to be stopped"
+ RETRY=`expr ${RETRY} + 1`
+ if [ "${RETRY}" -gt "20" ]
+ then
+ echo "Forcefully killing the process as it is not responding ..."
+ kill -9 $PID
+ fi
+ sleep 1
+ done
+ echo "$SERVICE_NAME stopped ...";
+ rm $PID_PATH_NAME
+ echo "$SERVICE_NAME starting ..."
+ nohup java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.airavata.datalake.workflow.engine.worker.Participant ${AIRAVATA_COMMAND} $* > $LOG_FILE 2>&1 &
+ echo $! > $PID_PATH_NAME
+ echo "$SERVICE_NAME started ..."
+ else
+ echo "$SERVICE_NAME is not running ..."
+ fi
+ ;;
+ -h)
+ echo "Usage: wf-worker-daemon.sh"
+
+ echo "command options:"
+ echo " start Start server in daemon mode"
+ echo " stop Stop server running in daemon mode"
+ echo " restart Restart server in daemon mode"
+ echo " -log <LOG_FILE> Where to redirect stdout/stderr (defaults to $DEFAULT_LOG_FILE)"
+ echo " -h Display this help and exit"
+ shift
+ exit 0
+ ;;
+esac
+
diff --git a/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/bin/wf-worker.sh b/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/bin/wf-worker.sh
new file mode 100644
index 0000000..4060535
--- /dev/null
+++ b/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/bin/wf-worker.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+. `dirname $0`/setenv.sh
+# Capture user's working dir before changing directory
+CWD="$PWD"
+cd ${AIRAVATA_HOME}/bin
+LOGO_FILE="logo.txt"
+
+JAVA_OPTS="-Dspring.config.location=${AIRAVATA_HOME}/conf/ -Dairavata.home=${AIRAVATA_HOME} -Dlogback.configurationFile=file:${AIRAVATA_HOME}/conf/logback.xml"
+AIRAVATA_COMMAND=""
+EXTRA_ARGS=""
+SERVERS=""
+IS_SUBSET=false
+SUBSET=""
+DEFAULT_LOG_FILE="${AIRAVATA_HOME}/logs/console.out"
+LOG_FILE=$DEFAULT_LOG_FILE
+
+# parse command arguments
+for var in "$@"
+do
+ case ${var} in
+ -xdebug)
+ AIRAVATA_COMMAND="${AIRAVATA_COMMAND}"
+ JAVA_OPTS="$JAVA_OPTS -Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,server=y,address=8000"
+ shift
+ ;;
+ -log)
+ shift
+ LOG_FILE="$1"
+ shift
+ # If relative path, expand to absolute path using the user's $CWD
+ if [ -z "`echo "$LOG_FILE" | egrep "^/"`" ]; then
+ LOG_FILE="${CWD}/${LOG_FILE}"
+ fi
+ ;;
+ -h)
+ echo "Usage: wf-worker.sh"
+
+ echo "command options:"
+ echo " -xdebug Start Workflow Worker under JPDA debugger"
+ echo " -h Display this help and exit"
+ shift
+ exit 0
+ ;;
+ *)
+ EXTRA_ARGS="${EXTRA_ARGS} ${var}"
+ shift
+ ;;
+ esac
+done
+
+java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.airavata.datalake.workflow.engine.worker.Participant ${AIRAVATA_COMMAND} $*
+
diff --git a/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/conf/application.properties b/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/conf/application.properties
new file mode 100644
index 0000000..690ca48
--- /dev/null
+++ b/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/conf/application.properties
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+cluster.name=datalake
+zookeeper.connection=localhost:2181
+
+participant.name=datalake_participant
+task.list.file=task-list.yaml
+
+server.port=33335
\ No newline at end of file
diff --git a/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/conf/logback.xml b/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/conf/logback.xml
new file mode 100644
index 0000000..3afe661
--- /dev/null
+++ b/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/conf/logback.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<configuration>
+
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d [%t] %-5p %c{30} %m [%X]%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="LOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <File>../logs/airavata.log</File>
+ <Append>true</Append>
+ <encoder>
+ <pattern>%d [%t] %-5p %c{30} %m [%X]%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>../logs/airavata.log.%d{yyyy-MM-dd}</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ <totalSizeCap>1GB</totalSizeCap>
+ </rollingPolicy>
+ </appender>
+
+ <logger name="ch.qos.logback" level="WARN"/>
+ <logger name="org.apache.helix" level="WARN"/>
+ <logger name="org.apache.zookeeper" level="ERROR"/>
+ <logger name="org.apache.helix" level="ERROR"/>
+ <logger name="org.apache.airavata" level="INFO"/>
+ <logger name="org.hibernate" level="ERROR"/>
+ <logger name="net.schmizz.sshj" level="WARN"/>
+ <root level="INFO">
+ <appender-ref ref="CONSOLE"/>
+ <appender-ref ref="LOGFILE"/>
+ </root>
+</configuration>
\ No newline at end of file
diff --git a/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/conf/task-list.yaml b/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/conf/task-list.yaml
new file mode 100644
index 0000000..1c167e0
--- /dev/null
+++ b/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/conf/task-list.yaml
@@ -0,0 +1,10 @@
+tasks:
+ blocking:
+ - org.apache.airavata.datalake.orchestrator.workflow.engine.task.impl.ExampleBlockingTask
+ - org.apache.airavata.datalake.orchestrator.workflow.engine.task.impl.SyncLocalDataDownloadTask
+ - org.apache.airavata.datalake.orchestrator.workflow.engine.task.impl.GenericDataParsingTask
+ - org.apache.airavata.datalake.orchestrator.workflow.engine.task.impl.MetadataPersistTask
+ nonBlocking:
+ - org.apache.airavata.datalake.orchestrator.workflow.engine.task.impl.ExampleNonBlockingTask
+ - org.apache.airavata.datalake.orchestrator.workflow.engine.task.impl.AsyncDataTransferTask
+ - org.apache.airavata.datalake.orchestrator.workflow.engine.task.impl.DataTransferPreValidationTask
diff --git a/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/wf-worker-assembly.xml b/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/wf-worker-assembly.xml
new file mode 100644
index 0000000..77164c3
--- /dev/null
+++ b/data-orchestrator/workflow-engine/workflow-engine-worker/src/main/dist/wf-worker-assembly.xml
@@ -0,0 +1,85 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<!DOCTYPE assembly [
+ <!ELEMENT assembly (id|includeBaseDirectory|baseDirectory|formats|fileSets|dependencySets)*>
+ <!ELEMENT id (#PCDATA)>
+ <!ELEMENT includeBaseDirectory (#PCDATA)>
+ <!ELEMENT baseDirectory (#PCDATA)>
+ <!ELEMENT formats (format)*>
+ <!ELEMENT format (#PCDATA)>
+ <!ELEMENT fileSets (fileSet)*>
+ <!ELEMENT fileSet (directory|outputDirectory|fileMode|includes)*>
+ <!ELEMENT directory (#PCDATA)>
+ <!ELEMENT outputDirectory (#PCDATA)>
+ <!ELEMENT includes (include)*>
+ <!ELEMENT include (#PCDATA)>
+ <!ELEMENT dependencySets (dependencySet)*>
+ <!ELEMENT dependencySet (outputDirectory|outputFileNameMapping|includes)*>
+ ]>
+<assembly>
+
+ <id>bin</id>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>WF-Worker-${project.version}</baseDirectory>
+ <formats>
+ <format>tar.gz</format>
+ <format>zip</format>
+ </formats>
+
+ <fileSets>
+ <fileSet>
+ <directory>src/main/dist/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>777</fileMode>
+ <includes>
+ <include>*.sh</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/dist/conf</directory>
+ <outputDirectory>conf</outputDirectory>
+ <includes>
+ <include>application.properties</include>
+ <include>logback.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>./</directory>
+ <outputDirectory>logs</outputDirectory>
+ <excludes>
+ <exclude>*/**</exclude>
+ </excludes>
+ </fileSet>
+ <fileSet>
+ <directory>target</directory>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>*.jar</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>*</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
\ No newline at end of file