You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2021/07/04 15:11:09 UTC
[airavata-data-lake] branch master updated: Drms synchronizer
distribution
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
The following commit(s) were added to refs/heads/master by this push:
new bbb0ac1 Drms synchronizer distribution
bbb0ac1 is described below
commit bbb0ac1330193b76db16c20c32f4683ce84ad0c5
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Sun Jul 4 11:10:56 2021 -0400
Drms synchronizer distribution
---
.../drms-custos-synchronizer/pom.xml | 21 ++--
.../dist/bin/drms-custos-synchronizer-daemon.sh | 113 +++++++++++++++++++++
.../src/main/dist/bin/drms-custos-synchronizer.sh | 71 +++++++++++++
.../src/main/dist/bin/setenv.sh | 46 +++++++++
.../src/main/dist/conf/application.properties | 2 +
.../src/main/dist/conf/config.yml | 17 ++++
.../src/main/dist/conf/logback.xml | 45 ++++++++
.../dist/drms-custos-synchronizer-assembly.xml | 86 ++++++++++++++++
.../custos/synchronizer/CustosSynchronizer.java | 20 ++--
9 files changed, 400 insertions(+), 21 deletions(-)
diff --git a/data-resource-management-service/drms-custos-synchronizer/pom.xml b/data-resource-management-service/drms-custos-synchronizer/pom.xml
index 727d1b4..b507eca 100644
--- a/data-resource-management-service/drms-custos-synchronizer/pom.xml
+++ b/data-resource-management-service/drms-custos-synchronizer/pom.xml
@@ -104,17 +104,24 @@
<build>
<plugins>
<plugin>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-maven-plugin</artifactId>
- <configuration>
- <fork>true</fork>
- <mainClass>org.apache.airavata.drms.custos.synchronizer.CustosSynchronizer</mainClass>
- </configuration>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven.assembly.plugin}</version>
<executions>
<execution>
+ <id>drms-custos-synchronizer-package</id>
+ <phase>package</phase>
<goals>
- <goal>repackage</goal>
+ <goal>single</goal>
</goals>
+ <configuration>
+ <tarLongFileMode>posix</tarLongFileMode>
+ <finalName>drms-custos-synchronizer</finalName>
+ <descriptors>
+ <descriptor>src/main/dist/drms-custos-synchronizer-assembly.xml</descriptor>
+ </descriptors>
+ <attach>false</attach>
+ </configuration>
</execution>
</executions>
</plugin>
diff --git a/data-resource-management-service/drms-custos-synchronizer/src/main/dist/bin/drms-custos-synchronizer-daemon.sh b/data-resource-management-service/drms-custos-synchronizer/src/main/dist/bin/drms-custos-synchronizer-daemon.sh
new file mode 100644
index 0000000..ec5c564
--- /dev/null
+++ b/data-resource-management-service/drms-custos-synchronizer/src/main/dist/bin/drms-custos-synchronizer-daemon.sh
@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+. `dirname $0`/setenv.sh
+# Capture user's working dir before changing directory
+CWD="$PWD"
+cd ${AIRAVATA_HOME}/bin
+LOGO_FILE="logo.txt"
+
+JAVA_OPTS="-Dspring.config.location=${AIRAVATA_HOME}/conf/ -Dairavata.home=${AIRAVATA_HOME} -Dlogback.configurationFile=file:${AIRAVATA_HOME}/conf/logback.xml"
+AIRAVATA_COMMAND=""
+EXTRA_ARGS=""
+SERVERS=""
+LOGO=true
+IS_SUBSET=false
+SUBSET=""
+DEFAULT_LOG_FILE="${AIRAVATA_HOME}/logs/airavata-daemon.out"
+LOG_FILE=$DEFAULT_LOG_FILE
+
+SERVICE_NAME="DRMS Custos Synchronizer"
+PID_PATH_NAME="${AIRAVATA_HOME}/bin/service-pid"
+
+case $1 in
+ start)
+ echo "Starting $SERVICE_NAME ..."
+ if [ ! -f $PID_PATH_NAME ]; then
+ nohup java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.airavata.drms.custos.synchronizer.CustosSynchronizer ${AIRAVATA_COMMAND} $* > $LOG_FILE 2>&1 &
+ echo $! > $PID_PATH_NAME
+ echo "$SERVICE_NAME started ..."
+ else
+ echo "$SERVICE_NAME is already running ..."
+ fi
+ ;;
+ stop)
+ if [ -f $PID_PATH_NAME ]; then
+ PID=$(cat $PID_PATH_NAME);
+ echo "$SERVICE_NAME stoping ..."
+ kill $PID;
+ RETRY=0
+ while kill -0 $PID 2> /dev/null; do
+ echo "Waiting for the process $PID to be stopped"
+ RETRY=`expr ${RETRY} + 1`
+ if [ "${RETRY}" -gt "20" ]
+ then
+ echo "Forcefully killing the process as it is not responding ..."
+ kill -9 $PID
+ fi
+ sleep 1
+ done
+ echo "$SERVICE_NAME stopped ..."
+ rm $PID_PATH_NAME
+ else
+ echo "$SERVICE_NAME is not running ..."
+ fi
+ ;;
+ restart)
+ if [ -f $PID_PATH_NAME ]; then
+ PID=$(cat $PID_PATH_NAME);
+ echo "$SERVICE_NAME stopping ...";
+ kill $PID;
+ RETRY=0
+ while kill -0 $PID 2> /dev/null; do
+ echo "Waiting for the process $PID to be stopped"
+ RETRY=`expr ${RETRY} + 1`
+ if [ "${RETRY}" -gt "20" ]
+ then
+ echo "Forcefully killing the process as it is not responding ..."
+ kill -9 $PID
+ fi
+ sleep 1
+ done
+ echo "$SERVICE_NAME stopped ...";
+ rm $PID_PATH_NAME
+ echo "$SERVICE_NAME starting ..."
+ nohup java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.airavata.drms.custos.synchronizer.CustosSynchronizer ${AIRAVATA_COMMAND} $* > $LOG_FILE 2>&1 &
+ echo $! > $PID_PATH_NAME
+ echo "$SERVICE_NAME started ..."
+ else
+ echo "$SERVICE_NAME is not running ..."
+ fi
+ ;;
+ -h)
+ echo "Usage: drms-custos-synchronizer-daemon.sh"
+
+ echo "command options:"
+ echo " start Start server in daemon mode"
+ echo " stop Stop server running in daemon mode"
+ echo " restart Restart server in daemon mode"
+ echo " -log <LOG_FILE> Where to redirect stdout/stderr (defaults to $DEFAULT_LOG_FILE)"
+ echo " -h Display this help and exit"
+ shift
+ exit 0
+ ;;
+esac
+
diff --git a/data-resource-management-service/drms-custos-synchronizer/src/main/dist/bin/drms-custos-synchronizer.sh b/data-resource-management-service/drms-custos-synchronizer/src/main/dist/bin/drms-custos-synchronizer.sh
new file mode 100644
index 0000000..cfe3b49
--- /dev/null
+++ b/data-resource-management-service/drms-custos-synchronizer/src/main/dist/bin/drms-custos-synchronizer.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+. `dirname $0`/setenv.sh
+# Capture user's working dir before changing directory
+CWD="$PWD"
+cd ${AIRAVATA_HOME}/bin
+LOGO_FILE="logo.txt"
+
+JAVA_OPTS="-Dspring.config.location=${AIRAVATA_HOME}/conf/ -Dairavata.home=${AIRAVATA_HOME} -Dlogback.configurationFile=file:${AIRAVATA_HOME}/conf/logback.xml"
+AIRAVATA_COMMAND=""
+EXTRA_ARGS=""
+SERVERS=""
+IS_SUBSET=false
+SUBSET=""
+DEFAULT_LOG_FILE="${AIRAVATA_HOME}/logs/console.out"
+LOG_FILE=$DEFAULT_LOG_FILE
+
+# parse command arguments
+for var in "$@"
+do
+ case ${var} in
+ -xdebug)
+ AIRAVATA_COMMAND="${AIRAVATA_COMMAND}"
+ JAVA_OPTS="$JAVA_OPTS -Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,server=y,address=8000"
+ shift
+ ;;
+ -log)
+ shift
+ LOG_FILE="$1"
+ shift
+ # If relative path, expand to absolute path using the user's $CWD
+ if [ -z "`echo "$LOG_FILE" | egrep "^/"`" ]; then
+ LOG_FILE="${CWD}/${LOG_FILE}"
+ fi
+ ;;
+ -h)
+ echo "Usage: drms.sh"
+
+ echo "command options:"
+ echo " -xdebug Start DRMS Custos Synchronizer under JPDA debugger"
+ echo " -h Display this help and exit"
+ shift
+ exit 0
+ ;;
+ *)
+ EXTRA_ARGS="${EXTRA_ARGS} ${var}"
+ shift
+ ;;
+ esac
+done
+
+java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.airavata.drms.custos.synchronizer.CustosSynchronizer ${AIRAVATA_COMMAND} $*
+
diff --git a/data-resource-management-service/drms-custos-synchronizer/src/main/dist/bin/setenv.sh b/data-resource-management-service/drms-custos-synchronizer/src/main/dist/bin/setenv.sh
new file mode 100755
index 0000000..9e894e1
--- /dev/null
+++ b/data-resource-management-service/drms-custos-synchronizer/src/main/dist/bin/setenv.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# resolve links - $0 may be a softlink
+PRG="$0"
+
+while [ -h "$PRG" ]; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '.*/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "$PRG"`/"$link"
+ fi
+done
+
+PRGDIR=`dirname "$PRG"`
+
+# Only set AIRAVATA_HOME if not already set
+[ -z "$AIRAVATA_HOME" ] && AIRAVATA_HOME=`cd "$PRGDIR/.." ; pwd`
+
+AIRAVATA_CLASSPATH=""
+
+for f in "$AIRAVATA_HOME"/lib/*.jar
+do
+ AIRAVATA_CLASSPATH="$AIRAVATA_CLASSPATH":$f
+done
+
+export AIRAVATA_HOME
+export AIRAVATA_CLASSPATH
diff --git a/data-resource-management-service/drms-custos-synchronizer/src/main/dist/conf/application.properties b/data-resource-management-service/drms-custos-synchronizer/src/main/dist/conf/application.properties
new file mode 100644
index 0000000..0c2b1b7
--- /dev/null
+++ b/data-resource-management-service/drms-custos-synchronizer/src/main/dist/conf/application.properties
@@ -0,0 +1,2 @@
+config.path=${AIRAVATA_HOME}/conf/config.yml
+spring.main.web-application-type=none
\ No newline at end of file
diff --git a/data-resource-management-service/drms-custos-synchronizer/src/main/dist/conf/config.yml b/data-resource-management-service/drms-custos-synchronizer/src/main/dist/conf/config.yml
new file mode 100644
index 0000000..9fc265d
--- /dev/null
+++ b/data-resource-management-service/drms-custos-synchronizer/src/main/dist/conf/config.yml
@@ -0,0 +1,17 @@
+pollingInterval: 60
+dataResourceManagementService:
+ dbURI: "bolt://192.168.0.14:7687"
+ dbUser: "neo4j"
+ dbPassword: "123456"
+custos:
+ host: "custos.scigap.org"
+ port: 31499
+ custosId: "custos-plgz25nnaqzkfhbmux9c-10002709"
+ custosSec: "NvecnxCK9mpM7AjKalnCRRrer7mFGjSPd3zbv2I3"
+ custosBrokerURL: "149.165.156.200:9092"
+ consumerGroup: "custosEventsGroup"
+ maxPollRecordsConfig: 10
+ topics:
+ - "10002708-754b89b5-3a57-496c-aa34-8e2b4916fbc"
+ tenantsToBeSynced:
+ - "custos-plgz25nnaqzkfhbmux9c-10002709"
diff --git a/data-resource-management-service/drms-custos-synchronizer/src/main/dist/conf/logback.xml b/data-resource-management-service/drms-custos-synchronizer/src/main/dist/conf/logback.xml
new file mode 100644
index 0000000..f4ac5de
--- /dev/null
+++ b/data-resource-management-service/drms-custos-synchronizer/src/main/dist/conf/logback.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<configuration debug="true">
+
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d [%t] %-5p %c{30} %m [%X]%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="LOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <File>../logs/custos_synchronizer.log</File>
+ <Append>true</Append>
+ <encoder>
+ <pattern>%d [%t] %-5p %c{30} %m [%X]%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>../logs/custos_synchronizer.log.%d{yyyy-MM-dd}</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ <totalSizeCap>1GB</totalSizeCap>
+ </rollingPolicy>
+ </appender>
+
+ <logger name="ch.qos.logback" level="WARN"/>
+ <logger name="org.apache.airavata" level="INFO"/>
+ <root level="INFO">
+ <appender-ref ref="CONSOLE"/>
+ <appender-ref ref="LOGFILE"/>
+ </root>
+</configuration>
\ No newline at end of file
diff --git a/data-resource-management-service/drms-custos-synchronizer/src/main/dist/drms-custos-synchronizer-assembly.xml b/data-resource-management-service/drms-custos-synchronizer/src/main/dist/drms-custos-synchronizer-assembly.xml
new file mode 100644
index 0000000..287808f
--- /dev/null
+++ b/data-resource-management-service/drms-custos-synchronizer/src/main/dist/drms-custos-synchronizer-assembly.xml
@@ -0,0 +1,86 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<!DOCTYPE assembly [
+ <!ELEMENT assembly (id|includeBaseDirectory|baseDirectory|formats|fileSets|dependencySets)*>
+ <!ELEMENT id (#PCDATA)>
+ <!ELEMENT includeBaseDirectory (#PCDATA)>
+ <!ELEMENT baseDirectory (#PCDATA)>
+ <!ELEMENT formats (format)*>
+ <!ELEMENT format (#PCDATA)>
+ <!ELEMENT fileSets (fileSet)*>
+ <!ELEMENT fileSet (directory|outputDirectory|fileMode|includes)*>
+ <!ELEMENT directory (#PCDATA)>
+ <!ELEMENT outputDirectory (#PCDATA)>
+ <!ELEMENT includes (include)*>
+ <!ELEMENT include (#PCDATA)>
+ <!ELEMENT dependencySets (dependencySet)*>
+ <!ELEMENT dependencySet (outputDirectory|outputFileNameMapping|includes)*>
+ ]>
+<assembly>
+
+ <id>bin</id>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>drms-custos-synchronizer</baseDirectory>
+ <formats>
+ <format>tar.gz</format>
+ <format>zip</format>
+ </formats>
+
+ <fileSets>
+ <fileSet>
+ <directory>src/main/dist/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>777</fileMode>
+ <includes>
+ <include>*.sh</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/dist/conf</directory>
+ <outputDirectory>conf</outputDirectory>
+ <includes>
+ <include>application.properties</include>
+ <include>logback.xml</include>
+ <include>config.yml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>./</directory>
+ <outputDirectory>logs</outputDirectory>
+ <excludes>
+ <exclude>*/**</exclude>
+ </excludes>
+ </fileSet>
+ <fileSet>
+ <directory>target</directory>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>*.jar</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>*</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
\ No newline at end of file
diff --git a/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/CustosSynchronizer.java b/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/CustosSynchronizer.java
index 2631ba7..a12a7d8 100644
--- a/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/CustosSynchronizer.java
+++ b/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/CustosSynchronizer.java
@@ -17,8 +17,9 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
public class CustosSynchronizer implements CommandLineRunner {
private static final Logger LOGGER = LoggerFactory.getLogger(CustosSynchronizer.class);
- private static String configFilePath;
+ @org.springframework.beans.factory.annotation.Value("${config.path}")
+ private String configPath;
public static void main(String[] args) {
SpringApplication.run(CustosSynchronizer.class, args);
@@ -27,18 +28,11 @@ public class CustosSynchronizer implements CommandLineRunner {
@Override
public void run(String... args) throws Exception {
LOGGER.info("Starting Custos synchronizer ...");
- if (args.length > 0) {
- configFilePath = args[0];
- } else {
- configFilePath = "/Users/isururanawaka/Documents/Airavata_Repository/airavata-data-lake" +
- "/data-resource-management-service/drms-custos-synchronizer/src/main/resources/config.yml";
- }
-
- LOGGER.info("Configuring scheduler ...");
- Utils.initializeConnectors(Utils.loadConfiguration(configFilePath));
- configureScheduler(configFilePath);
- configureEventListener(configFilePath);
+ LOGGER.info("Configuring scheduler using file {}...", configPath);
+ Utils.initializeConnectors(Utils.loadConfiguration(configPath));
+ configureScheduler(configPath);
+ configureEventListener(configPath);
}
@@ -78,6 +72,4 @@ public class CustosSynchronizer implements CommandLineRunner {
}
});
}
-
-
}