You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2024/12/16 13:11:15 UTC
(airavata) 01/01: File server initial implementation
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch file-server
in repository https://gitbox.apache.org/repos/asf/airavata.git
commit e18975880ce038d205a9046b406fea11e8b58079
Author: DImuthuUpe <di...@gmail.com>
AuthorDate: Mon Dec 16 08:11:04 2024 -0500
File server initial implementation
---
modules/agent-framework/connection-service/pom.xml | 3 +-
.../airavata/helix/impl/task/AiravataTask.java | 2 +-
modules/file-server/pom.xml | 81 +++++++++++++++
.../main/assembly/file-service-bin-assembly.xml | 90 +++++++++++++++++
.../file/server/FileServerApplication.java | 16 +++
.../file/server/FileServerConfiguration.java | 47 +++++++++
.../file/server/controller/FileController.java | 87 ++++++++++++++++
.../file/server/model/AiravataDirectory.java | 49 +++++++++
.../airavata/file/server/model/AiravataFile.java | 47 +++++++++
.../file/server/model/FileUploadResponse.java | 47 +++++++++
.../file/server/service/AirvataFileService.java | 88 ++++++++++++++++
.../file/server/service/ProcessDataManager.java | 57 +++++++++++
.../src/main/resources/application.properties | 12 +++
.../distribution/bin/file-service-daemon.sh | 112 +++++++++++++++++++++
.../resources/distribution/bin/file-service.sh | 70 +++++++++++++
.../src/main/resources/distribution/bin/setenv.sh | 46 +++++++++
.../distribution/conf/application.properties | 12 +++
.../main/resources/distribution/conf/log4j2.xml | 53 ++++++++++
pom.xml | 2 +
19 files changed, 918 insertions(+), 3 deletions(-)
diff --git a/modules/agent-framework/connection-service/pom.xml b/modules/agent-framework/connection-service/pom.xml
index 619c872692..3ee06a2ccb 100644
--- a/modules/agent-framework/connection-service/pom.xml
+++ b/modules/agent-framework/connection-service/pom.xml
@@ -15,7 +15,6 @@
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <spring.boot.version>3.2.4</spring.boot.version>
<jul-to-slf4j.version>1.7.0</jul-to-slf4j.version>
<json.version>20240303</json.version>
<javax.version>2.0.1.Final</javax.version>
@@ -159,4 +158,4 @@
</plugins>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java
index d2208d5293..0715a17bdb 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java
@@ -496,7 +496,7 @@ public abstract class AiravataTask extends AbstractTask {
}
}
- private void loadContext() throws TaskOnFailException {
+ protected void loadContext() throws TaskOnFailException {
try {
logger.info("Loading context for task " + getTaskId());
processModel = getRegistryServiceClient().getProcess(processId);
diff --git a/modules/file-server/pom.xml b/modules/file-server/pom.xml
new file mode 100644
index 0000000000..a7065bdff3
--- /dev/null
+++ b/modules/file-server/pom.xml
@@ -0,0 +1,81 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata</artifactId>
+ <version>0.21-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <artifactId>file-server</artifactId>
+ <name>airavata-file-server</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ <version>${spring.boot.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-to-slf4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>task-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>helix-spectator</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.8.1</version>
+ <configuration>
+ <compilerArgs>
+ <arg>-parameters</arg>
+ </compilerArgs>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven.assembly.plugin}</version>
+ <executions>
+ <execution>
+ <id>file-service-distribution-package</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <tarLongFileMode>posix</tarLongFileMode>
+ <finalName>${file.service.dist.name}</finalName>
+ <descriptors>
+ <descriptor>src/main/assembly/file-service-bin-assembly.xml</descriptor>
+ </descriptors>
+ <attach>false</attach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <properties>
+ <file.service.dist.name>File-Service-0.01</file.service.dist.name>
+ </properties>
+</project>
diff --git a/modules/file-server/src/main/assembly/file-service-bin-assembly.xml b/modules/file-server/src/main/assembly/file-service-bin-assembly.xml
new file mode 100644
index 0000000000..8d72d47391
--- /dev/null
+++ b/modules/file-server/src/main/assembly/file-service-bin-assembly.xml
@@ -0,0 +1,90 @@
+
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<!DOCTYPE assembly [
+ <!ELEMENT assembly (id|includeBaseDirectory|baseDirectory|formats|fileSets|dependencySets)*>
+ <!ELEMENT id (#PCDATA)>
+ <!ELEMENT includeBaseDirectory (#PCDATA)>
+ <!ELEMENT baseDirectory (#PCDATA)>
+ <!ELEMENT formats (format)*>
+ <!ELEMENT format (#PCDATA)>
+ <!ELEMENT fileSets (fileSet)*>
+ <!ELEMENT fileSet (directory|outputDirectory|fileMode|includes)*>
+ <!ELEMENT directory (#PCDATA)>
+ <!ELEMENT outputDirectory (#PCDATA)>
+ <!ELEMENT includes (include)*>
+ <!ELEMENT include (#PCDATA)>
+ <!ELEMENT dependencySets (dependencySet)*>
+ <!ELEMENT dependencySet (outputDirectory|outputFileNameMapping|includes)*>
+ ]>
+<assembly>
+ <id>bin</id>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>${file.service.dist.name}</baseDirectory>
+ <formats>
+ <format>tar.gz</format>
+ <format>zip</format>
+ </formats>
+
+ <fileSets>
+ <fileSet>
+ <directory>src/main/resources/distribution/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>777</fileMode>
+ <includes>
+ <include>*.sh</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/resources/distribution/conf</directory>
+ <outputDirectory>conf</outputDirectory>
+ <includes>
+ <include>application.properties</include>
+ <include>log4j2.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>./</directory>
+ <outputDirectory>logs</outputDirectory>
+ <excludes>
+ <exclude>*/**</exclude>
+ </excludes>
+ </fileSet>
+ <fileSet>
+ <directory>target</directory>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>*.jar</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>*</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+
+</assembly>
diff --git a/modules/file-server/src/main/java/org/apache/airavata/file/server/FileServerApplication.java b/modules/file-server/src/main/java/org/apache/airavata/file/server/FileServerApplication.java
new file mode 100644
index 0000000000..42bf099184
--- /dev/null
+++ b/modules/file-server/src/main/java/org/apache/airavata/file/server/FileServerApplication.java
@@ -0,0 +1,16 @@
+package org.apache.airavata.file.server;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.groovy.template.GroovyTemplateAutoConfiguration;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.ComponentScan;
+
+@SpringBootApplication(scanBasePackages = "org.apache.airavata.file.server",
+ exclude = {GroovyTemplateAutoConfiguration.class})
+@EnableConfigurationProperties(FileServerConfiguration.class)
+public class FileServerApplication {
+ public static void main(String[] args) {
+ SpringApplication.run(FileServerApplication.class, args);
+ }
+}
diff --git a/modules/file-server/src/main/java/org/apache/airavata/file/server/FileServerConfiguration.java b/modules/file-server/src/main/java/org/apache/airavata/file/server/FileServerConfiguration.java
new file mode 100644
index 0000000000..e4b6df32d8
--- /dev/null
+++ b/modules/file-server/src/main/java/org/apache/airavata/file/server/FileServerConfiguration.java
@@ -0,0 +1,47 @@
+package org.apache.airavata.file.server;
+
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.common.utils.ThriftClientPool;
+import org.apache.airavata.helix.core.support.adaptor.AdaptorSupportImpl;
+import org.apache.airavata.helix.task.api.support.AdaptorSupport;
+import org.apache.airavata.registry.api.RegistryService;
+import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+
+@Component
+@ConfigurationProperties
+public class FileServerConfiguration {
+
+ @Bean
+ public AdaptorSupport adaptorSupport() {
+ return AdaptorSupportImpl.getInstance();
+ }
+
+ //regserver.server.host
+ @Value("${regserver.server.host:localhost}")
+ private String registryServerHost;
+ //regserver.server.port
+
+ @Value("${regserver.server.port:8970}")
+ private int registryServerPort;
+
+ @Bean
+ public ThriftClientPool<RegistryService.Client> registryClientPool() {
+ GenericObjectPoolConfig<RegistryService.Client> poolConfig = new GenericObjectPoolConfig<>();
+ poolConfig.setMaxTotal(100);
+ poolConfig.setMinIdle(5);
+ poolConfig.setBlockWhenExhausted(true);
+ poolConfig.setTestOnBorrow(true);
+ poolConfig.setTestWhileIdle(true);
+ // must set timeBetweenEvictionRunsMillis since eviction doesn't run unless that is positive
+ poolConfig.setTimeBetweenEvictionRunsMillis(5L * 60L * 1000L);
+ poolConfig.setNumTestsPerEvictionRun(10);
+ poolConfig.setMaxWaitMillis(3000);
+
+ return new ThriftClientPool<>(
+ RegistryService.Client::new, poolConfig, registryServerHost, registryServerPort);
+ }
+}
diff --git a/modules/file-server/src/main/java/org/apache/airavata/file/server/controller/FileController.java b/modules/file-server/src/main/java/org/apache/airavata/file/server/controller/FileController.java
new file mode 100644
index 0000000000..b735da7765
--- /dev/null
+++ b/modules/file-server/src/main/java/org/apache/airavata/file/server/controller/FileController.java
@@ -0,0 +1,87 @@
+package org.apache.airavata.file.server.controller;
+
+import org.apache.airavata.file.server.model.AiravataDirectory;
+import org.apache.airavata.file.server.model.FileUploadResponse;
+import org.apache.airavata.file.server.service.AirvataFileService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.UrlResource;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.multipart.MultipartFile;
+import org.springframework.web.servlet.support.ServletUriComponentsBuilder;
+
+import java.io.File;
+import java.nio.file.Path;
+
+@Controller
+public class FileController {
+
+ private final static Logger logger = LoggerFactory.getLogger(FileController.class);
+
+ @Autowired
+ private AirvataFileService fileService;
+
+ @GetMapping("/list/{live}/{processId}")
+ @ResponseBody
+ public Object listFilesRoot(@PathVariable String live,
+ @PathVariable String processId) throws Exception {
+ try {
+ return fileService.listFiles(processId, "");
+ } catch (Exception e) {
+ logger.error("Failed to list files for path {} in process {}", "root path", processId, e);
+ throw e;
+ }
+ }
+ @GetMapping("/list/{live}/{processId}/{subPath}")
+ @ResponseBody
+ public Object listFiles(@PathVariable String live,
+ @PathVariable String processId,
+ @PathVariable String subPath) throws Exception {
+ try {
+ return fileService.listFiles(processId, subPath);
+ } catch (Exception e) {
+ logger.error("Failed to list files for path {} in process {}", subPath, processId, e);
+ throw e;
+ }
+ }
+
+ @GetMapping("/download/{isLive}/{processId}/{subPath}")
+ @ResponseBody
+ public ResponseEntity downloadFile(@PathVariable String isLive,
+ @PathVariable String processId,
+ @PathVariable String subPath) {
+
+ try {
+ Path localPath = fileService.downloadFile(processId, subPath);
+ Resource resource = new UrlResource(localPath.toUri());
+ return ResponseEntity.ok()
+ .header(HttpHeaders.CONTENT_DISPOSITION,
+ "attachment; filename=\"" + new File(subPath).getName() + "\"")
+ .body(resource);
+ } catch (Exception e) {
+ return ResponseEntity.internalServerError()
+ .body("An internal server error occurred: " + e.getMessage());
+ }
+ }
+
+
+ @PostMapping("/upload-file")
+ @ResponseBody
+ public FileUploadResponse uploadFile(@RequestParam("file") MultipartFile file) {
+ String name = "";
+
+ String uri = ServletUriComponentsBuilder.fromCurrentContextPath()
+ .path("/download/")
+ .path(name)
+ .toUriString();
+
+ return new FileUploadResponse(name, uri, file.getContentType(), file.getSize());
+ }
+
+
+}
diff --git a/modules/file-server/src/main/java/org/apache/airavata/file/server/model/AiravataDirectory.java b/modules/file-server/src/main/java/org/apache/airavata/file/server/model/AiravataDirectory.java
new file mode 100644
index 0000000000..bed98f1ffb
--- /dev/null
+++ b/modules/file-server/src/main/java/org/apache/airavata/file/server/model/AiravataDirectory.java
@@ -0,0 +1,49 @@
+package org.apache.airavata.file.server.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class AiravataDirectory {
+ private String directoryName;
+ private long createdTime;
+
+ private List<AiravataFile> innerFiles = new ArrayList<>();
+ private List<AiravataDirectory> innerDirectories = new ArrayList<>();
+
+ public AiravataDirectory(String directoryName, long createdTime) {
+ this.directoryName = directoryName;
+ this.createdTime = createdTime;
+ }
+
+ public String getDirectoryName() {
+ return directoryName;
+ }
+
+ public void setDirectoryName(String directoryName) {
+ this.directoryName = directoryName;
+ }
+
+ public long getCreatedTime() {
+ return createdTime;
+ }
+
+ public void setCreatedTime(long createdTime) {
+ this.createdTime = createdTime;
+ }
+
+ public List<AiravataFile> getInnerFiles() {
+ return innerFiles;
+ }
+
+ public void setInnerFiles(List<AiravataFile> innerFiles) {
+ this.innerFiles = innerFiles;
+ }
+
+ public List<AiravataDirectory> getInnerDirectories() {
+ return innerDirectories;
+ }
+
+ public void setInnerDirectories(List<AiravataDirectory> innerDirectories) {
+ this.innerDirectories = innerDirectories;
+ }
+}
diff --git a/modules/file-server/src/main/java/org/apache/airavata/file/server/model/AiravataFile.java b/modules/file-server/src/main/java/org/apache/airavata/file/server/model/AiravataFile.java
new file mode 100644
index 0000000000..f2a174f6a7
--- /dev/null
+++ b/modules/file-server/src/main/java/org/apache/airavata/file/server/model/AiravataFile.java
@@ -0,0 +1,47 @@
+package org.apache.airavata.file.server.model;
+
+public class AiravataFile {
+ private String fileName;
+ private long fileSize;
+ private long createdTime;
+ private long updatedTime;
+
+ public AiravataFile(String fileName, long fileSize, long createdTime, long updatedTime) {
+ this.fileName = fileName;
+ this.fileSize = fileSize;
+ this.createdTime = createdTime;
+ this.updatedTime = updatedTime;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public void setFileName(String fileName) {
+ this.fileName = fileName;
+ }
+
+ public long getFileSize() {
+ return fileSize;
+ }
+
+ public void setFileSize(long fileSize) {
+ this.fileSize = fileSize;
+ }
+
+ public long getCreatedTime() {
+ return createdTime;
+ }
+
+ public void setCreatedTime(long createdTime) {
+ this.createdTime = createdTime;
+ }
+
+ public long getUpdatedTime() {
+ return updatedTime;
+ }
+
+ public void setUpdatedTime(long updatedTime) {
+ this.updatedTime = updatedTime;
+ }
+}
diff --git a/modules/file-server/src/main/java/org/apache/airavata/file/server/model/FileUploadResponse.java b/modules/file-server/src/main/java/org/apache/airavata/file/server/model/FileUploadResponse.java
new file mode 100644
index 0000000000..0f061cad2b
--- /dev/null
+++ b/modules/file-server/src/main/java/org/apache/airavata/file/server/model/FileUploadResponse.java
@@ -0,0 +1,47 @@
+package org.apache.airavata.file.server.model;
+
+public class FileUploadResponse {
+ private String name;
+ private String uri;
+ private String type;
+ private long size;
+
+ public FileUploadResponse(String name, String uri, String type, long size) {
+ this.name = name;
+ this.uri = uri;
+ this.type = type;
+ this.size = size;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getUri() {
+ return uri;
+ }
+
+ public void setUri(String uri) {
+ this.uri = uri;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public long getSize() {
+ return size;
+ }
+
+ public void setSize(long size) {
+ this.size = size;
+ }
+}
diff --git a/modules/file-server/src/main/java/org/apache/airavata/file/server/service/AirvataFileService.java b/modules/file-server/src/main/java/org/apache/airavata/file/server/service/AirvataFileService.java
new file mode 100644
index 0000000000..949d59c77b
--- /dev/null
+++ b/modules/file-server/src/main/java/org/apache/airavata/file/server/service/AirvataFileService.java
@@ -0,0 +1,88 @@
+package org.apache.airavata.file.server.service;
+
+import org.apache.airavata.agents.api.AgentAdaptor;
+import org.apache.airavata.agents.api.FileMetadata;
+import org.apache.airavata.common.utils.ThriftClientPool;
+import org.apache.airavata.file.server.model.AiravataDirectory;
+import org.apache.airavata.file.server.model.AiravataFile;
+import org.apache.airavata.helix.task.api.support.AdaptorSupport;
+import org.apache.airavata.registry.api.RegistryService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.File;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+@Service
+public class AirvataFileService {
+
+ private final static Logger logger = LoggerFactory.getLogger(AirvataFileService.class);
+
+ @Autowired
+ private AdaptorSupport adaptorSupport;
+
+ @Autowired
+ ThriftClientPool<RegistryService.Client> registryClientPool;
+
+ private AgentAdaptor getAgentAdaptor(ProcessDataManager dataManager, String processId) throws Exception {
+ AgentAdaptor agentAdaptor;
+ try {
+ agentAdaptor = dataManager.getAgentAdaptor();
+ } catch (Exception e) {
+ logger.error("Failed to fetch adaptor for process {}", processId, e);
+ throw new Exception("Failed to fetch adaptor for process " + processId, e);
+ }
+ return agentAdaptor;
+ }
+ public AiravataDirectory listFiles(String processId, String subPath) throws Exception {
+ ProcessDataManager dataManager = new ProcessDataManager(registryClientPool, processId, adaptorSupport);
+
+ AgentAdaptor agentAdaptor = getAgentAdaptor(dataManager, processId);
+
+ AiravataDirectory airavataDirectory = new AiravataDirectory("root", System.currentTimeMillis()); // TODO: set dir name
+
+ subPath = dataManager.getBaseDir() + (subPath.isEmpty()? "" : "/" + subPath);
+ logger.info("Listing files in path {}", subPath);
+ List<String> fileList = agentAdaptor.listDirectory(subPath); // TODO: Validate if this is a file or dir
+ for (String fileOrDir : fileList) {
+ logger.info("Processing file {}", fileOrDir);
+ FileMetadata fileMetadata = agentAdaptor.getFileMetadata(subPath + "/" + fileOrDir);
+ airavataDirectory.getInnerFiles().add(new AiravataFile(fileMetadata.getName(),
+ fileMetadata.getSize(), System.currentTimeMillis(), System.currentTimeMillis())); // TODO: update created and updated time
+ }
+
+ return airavataDirectory;
+ }
+ public Path downloadFile(String processId, String subPath) throws Exception {
+
+ ProcessDataManager dataManager = new ProcessDataManager(registryClientPool, processId, adaptorSupport);
+
+ AgentAdaptor agentAdaptor = getAgentAdaptor(dataManager, processId);
+ subPath = dataManager.getBaseDir() + (subPath.isEmpty()? "" : "/" + subPath);
+
+ if (agentAdaptor.doesFileExist(subPath)) {
+ Path tempFile = Files.createTempFile("tempfile_", ".data");
+ tempFile.toFile().deleteOnExit();
+ try {
+ agentAdaptor.downloadFile(subPath, tempFile.toFile().getAbsolutePath());
+ return tempFile;
+ } catch (Exception e) {
+ logger.error("Failed to download file {} from process {} to local path {}",
+ subPath, processId, tempFile.toFile().getAbsolutePath());
+ try {
+ tempFile.toFile().delete();
+ } catch (Exception ignore) {
+ // Ignore
+ }
+ throw e;
+ }
+ } else {
+ throw new Exception("File " + subPath + " does not exist in process " + processId);
+ }
+ }
+}
diff --git a/modules/file-server/src/main/java/org/apache/airavata/file/server/service/ProcessDataManager.java b/modules/file-server/src/main/java/org/apache/airavata/file/server/service/ProcessDataManager.java
new file mode 100644
index 0000000000..b253ed11ee
--- /dev/null
+++ b/modules/file-server/src/main/java/org/apache/airavata/file/server/service/ProcessDataManager.java
@@ -0,0 +1,57 @@
+package org.apache.airavata.file.server.service;
+
+import org.apache.airavata.agents.api.AgentAdaptor;
+import org.apache.airavata.common.utils.ThriftClientPool;
+import org.apache.airavata.helix.impl.task.TaskOnFailException;
+import org.apache.airavata.helix.impl.task.staging.OutputDataStagingTask;
+import org.apache.airavata.helix.task.api.support.AdaptorSupport;
+import org.apache.airavata.model.experiment.ExperimentModel;
+import org.apache.airavata.model.process.ProcessModel;
+import org.apache.airavata.registry.api.RegistryService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.UUID;
+
+public class ProcessDataManager extends OutputDataStagingTask {
+
+ private final static Logger logger = LoggerFactory.getLogger(ProcessDataManager.class);
+
+ private String processId;
+ private AdaptorSupport adaptorSupport;
+
+ private ProcessModel process;
+ ExperimentModel experiment;
+
+ public ProcessDataManager(ThriftClientPool<RegistryService.Client> registryClientPool,
+ String processId, AdaptorSupport adaptorSupport) throws Exception {
+
+ this.adaptorSupport = adaptorSupport;
+ RegistryService.Client regClient = registryClientPool.getResource();
+ try {
+ process = regClient.getProcess(processId);
+ experiment = regClient.getExperiment(process.getExperimentId());
+
+ setTaskId(UUID.randomUUID().toString());
+ setProcessId(processId);
+ setExperimentId(process.getExperimentId());
+ setGatewayId(experiment.getGatewayId());
+ loadContext();
+
+ registryClientPool.returnResource(regClient);
+ } catch (Exception e) {
+ logger.error("Failed to initialize the output data mover for process {}", processId, e);
+ registryClientPool.returnBrokenResource(regClient);
+ throw e;
+ }
+ this.processId = processId;
+ }
+
+ public AgentAdaptor getAgentAdaptor() throws TaskOnFailException {
+ return getComputeResourceAdaptor(adaptorSupport);
+ }
+
+ public String getBaseDir() throws Exception {
+ return getTaskContext().getWorkingDir();
+ }
+}
diff --git a/modules/file-server/src/main/resources/application.properties b/modules/file-server/src/main/resources/application.properties
new file mode 100644
index 0000000000..3d5477b4dd
--- /dev/null
+++ b/modules/file-server/src/main/resources/application.properties
@@ -0,0 +1,12 @@
+spring.servlet.multipart.max-file-size=10MB
+# max request size
+spring.servlet.multipart.max-request-size=10MB
+# files storage location (stores all files uploaded via REST API)
+storage.location=./uploads
+
+regserver.server.host=localhost
+regserver.server.port=8970
+credential.store.server.host=localhost
+credential.store.server.port=8960
+
+server.port=8050
diff --git a/modules/file-server/src/main/resources/distribution/bin/file-service-daemon.sh b/modules/file-server/src/main/resources/distribution/bin/file-service-daemon.sh
new file mode 100644
index 0000000000..131d99df54
--- /dev/null
+++ b/modules/file-server/src/main/resources/distribution/bin/file-service-daemon.sh
@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+. `dirname $0`/setenv.sh
+# Capture user's working dir before changing directory
+CWD="$PWD"
+cd ${AIRAVATA_HOME}/bin
+LOGO_FILE="logo.txt"
+
+JAVA_OPTS=" -Dspring.config.location=${AIRAVATA_HOME}/conf/ -Dairavata.home=${AIRAVATA_HOME} -Dlog4j.configurationFile=file:${AIRAVATA_HOME}/conf/log4j2.xml"
+AIRAVATA_COMMAND=""
+EXTRA_ARGS=""
+SERVERS=""
+LOGO=true
+IS_SUBSET=false
+SUBSET=""
+DEFAULT_LOG_FILE="${AIRAVATA_HOME}/logs/airavata-daemon.out"
+LOG_FILE=$DEFAULT_LOG_FILE
+
+SERVICE_NAME="File Service"
+PID_PATH_NAME="${AIRAVATA_HOME}/bin/service-pid"
+
+case $1 in
+ start)
+ echo "Starting $SERVICE_NAME ..."
+ if [ ! -f $PID_PATH_NAME ]; then
+ nohup java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.airavata.file.server.FileServerApplication ${AIRAVATA_COMMAND} $* > $LOG_FILE 2>&1 &
+ echo $! > $PID_PATH_NAME
+ echo "$SERVICE_NAME started ..."
+ else
+ echo "$SERVICE_NAME is already running ..."
+ fi
+ ;;
+ stop)
+ if [ -f $PID_PATH_NAME ]; then
+ PID=$(cat $PID_PATH_NAME);
+ echo "$SERVICE_NAME stopping ..."
+ kill $PID;
+ RETRY=0
+ while kill -0 $PID 2> /dev/null; do
+ echo "Waiting for the process $PID to be stopped"
+ RETRY=`expr ${RETRY} + 1`
+ if [ "${RETRY}" -gt "20" ]
+ then
+ echo "Forcefully killing the process as it is not responding ..."
+ kill -9 $PID
+ fi
+ sleep 1
+ done
+ echo "$SERVICE_NAME stopped ..."
+ rm $PID_PATH_NAME
+ else
+ echo "$SERVICE_NAME is not running ..."
+ fi
+ ;;
+ restart)
+ if [ -f $PID_PATH_NAME ]; then
+ PID=$(cat $PID_PATH_NAME);
+ echo "$SERVICE_NAME stopping ...";
+ kill $PID;
+ RETRY=0
+ while kill -0 $PID 2> /dev/null; do
+ echo "Waiting for the process $PID to be stopped"
+ RETRY=`expr ${RETRY} + 1`
+ if [ "${RETRY}" -gt "20" ]
+ then
+ echo "Forcefully killing the process as it is not responding ..."
+ kill -9 $PID
+ fi
+ sleep 1
+ done
+ echo "$SERVICE_NAME stopped ...";
+ rm $PID_PATH_NAME
+ echo "$SERVICE_NAME starting ..."
+ nohup java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.airavata.file.server.FileServerApplication ${AIRAVATA_COMMAND} $* > $LOG_FILE 2>&1 &
+ echo $! > $PID_PATH_NAME
+ echo "$SERVICE_NAME started ..."
+ else
+ echo "$SERVICE_NAME is not running ..."
+ fi
+ ;;
+ -h)
+ echo "Usage: file-service-daemon.sh"
+
+ echo "command options:"
+ echo " start Start server in daemon mode"
+ echo " stop Stop server running in daemon mode"
+ echo " restart Restart server in daemon mode"
+ echo " -log <LOG_FILE> Where to redirect stdout/stderr (defaults to $DEFAULT_LOG_FILE)"
+ echo " -h Display this help and exit"
+ shift
+ exit 0
+ ;;
+esac
diff --git a/modules/file-server/src/main/resources/distribution/bin/file-service.sh b/modules/file-server/src/main/resources/distribution/bin/file-service.sh
new file mode 100644
index 0000000000..231f5dfa26
--- /dev/null
+++ b/modules/file-server/src/main/resources/distribution/bin/file-service.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+. `dirname $0`/setenv.sh
+# Capture user's working dir before changing directory
+CWD="$PWD"
+cd ${AIRAVATA_HOME}/bin
+LOGO_FILE="logo.txt"
+
+JAVA_OPTS="-Dspring.config.location=${AIRAVATA_HOME}/conf/ -Dairavata.home=${AIRAVATA_HOME} -Dlog4j.configurationFile=file:${AIRAVATA_HOME}/conf/log4j2.xml"
+AIRAVATA_COMMAND=""
+EXTRA_ARGS=""
+SERVERS=""
+IS_SUBSET=false
+SUBSET=""
+DEFAULT_LOG_FILE="${AIRAVATA_HOME}/logs/airavata.out"
+LOG_FILE=$DEFAULT_LOG_FILE
+
+# parse command arguments
+for var in "$@"
+do
+ case ${var} in
+ -xdebug)
+ AIRAVATA_COMMAND="${AIRAVATA_COMMAND}"
+ JAVA_OPTS="$JAVA_OPTS -Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,server=y,address=*:8000"
+ shift
+ ;;
+ -log)
+ shift
+ LOG_FILE="$1"
+ shift
+ # If relative path, expand to absolute path using the user's $CWD
+ if [ -z "`echo "$LOG_FILE" | egrep "^/"`" ]; then
+ LOG_FILE="${CWD}/${LOG_FILE}"
+ fi
+ ;;
+ -h)
+ echo "Usage: file-service.sh"
+
+ echo "command options:"
+ echo " -xdebug Start Connection Service under JPDA debugger"
+ echo " -h Display this help and exit"
+ shift
+ exit 0
+ ;;
+ *)
+ EXTRA_ARGS="${EXTRA_ARGS} ${var}"
+ shift
+ ;;
+ esac
+done
+
+java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.airavata.file.server.FileServerApplication ${AIRAVATA_COMMAND} $*
diff --git a/modules/file-server/src/main/resources/distribution/bin/setenv.sh b/modules/file-server/src/main/resources/distribution/bin/setenv.sh
new file mode 100644
index 0000000000..3f1d7632bf
--- /dev/null
+++ b/modules/file-server/src/main/resources/distribution/bin/setenv.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# resolve links - $0 may be a softlink
+PRG="$0"
+
+while [ -h "$PRG" ]; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '.*/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "$PRG"`/"$link"
+ fi
+done
+
+PRGDIR=`dirname "$PRG"`
+
+# Only set AIRAVATA_HOME if not already set
+[ -z "$AIRAVATA_HOME" ] && AIRAVATA_HOME=`cd "$PRGDIR/.." ; pwd`
+
+AIRAVATA_CLASSPATH=""
+
+for f in "$AIRAVATA_HOME"/lib/*.jar
+do
+ AIRAVATA_CLASSPATH="$AIRAVATA_CLASSPATH":$f
+done
+
+export AIRAVATA_HOME
+export AIRAVATA_CLASSPATH
\ No newline at end of file
diff --git a/modules/file-server/src/main/resources/distribution/conf/application.properties b/modules/file-server/src/main/resources/distribution/conf/application.properties
new file mode 100644
index 0000000000..3d5477b4dd
--- /dev/null
+++ b/modules/file-server/src/main/resources/distribution/conf/application.properties
@@ -0,0 +1,12 @@
+spring.servlet.multipart.max-file-size=10MB
+# max request size
+spring.servlet.multipart.max-request-size=10MB
+# files storage location (stores all files uploaded via REST API)
+storage.location=./uploads
+
+regserver.server.host=localhost
+regserver.server.port=8970
+credential.store.server.host=localhost
+credential.store.server.port=8960
+
+server.port=8050
diff --git a/modules/file-server/src/main/resources/distribution/conf/log4j2.xml b/modules/file-server/src/main/resources/distribution/conf/log4j2.xml
new file mode 100644
index 0000000000..e38f7ff011
--- /dev/null
+++ b/modules/file-server/src/main/resources/distribution/conf/log4j2.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<Configuration status="WARN">
+
+ <Appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <PatternLayout pattern="%d [%t] %-5p %c{30} %X - %m%n"/>
+ </Console>
+ <RollingFile name="RollingFileAppender" fileName="../logs/airavata.log"
+ filePattern="logs/${date:yyyy-MM}/airavata-log-%d{MM-dd-yyyy}-%i.log.gz">
+ <PatternLayout>
+ <Pattern>%d [%t] %-5p %c{30} %X - %m%n</Pattern>
+ </PatternLayout>
+ <Policies>
+ <OnStartupTriggeringPolicy />
+ <TimeBasedTriggeringPolicy />
+ <SizeBasedTriggeringPolicy size="50 MB" />
+ </Policies>
+ <DefaultRolloverStrategy max="20" />
+ </RollingFile>
+ </Appenders>
+ <Loggers>
+ <logger name="ch.qos.logback" level="WARN"/>
+ <logger name="org.apache.helix" level="WARN"/>
+ <logger name="org.apache.zookeeper" level="ERROR"/>
+ <logger name="org.apache.airavata" level="INFO"/>
+ <logger name="org.hibernate" level="ERROR"/>
+ <Root level="INFO">
+ <AppenderRef ref="Console"/>
+ <AppenderRef ref="RollingFileAppender"/>
+ </Root>
+ </Loggers>
+</Configuration>
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 3f57e2f519..f9ff579af0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -102,6 +102,7 @@
<module>modules/agent-framework</module>
<module>tools</module>
<module>modules/ide-integration</module>
+ <module>modules/file-server</module>
</modules>
<properties>
@@ -151,6 +152,7 @@
<org.apache.commons.pool2.version>2.7.0</org.apache.commons.pool2.version>
<helix.version>0.9.7</helix.version>
<httpclient.version>4.5.14</httpclient.version>
+ <spring.boot.version>3.2.4</spring.boot.version>
</properties>
<dependencies>