You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2022/12/07 07:25:24 UTC
[airavata-mft] branch master updated: Removing dependency between agent and secret/resource services
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
The following commit(s) were added to refs/heads/master by this push:
new 60ed0a4 Removing dependency between agent and secret/resource services
60ed0a4 is described below
commit 60ed0a4e3cab4c74cb3dafc9e0e309324d111557
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Wed Dec 7 02:25:14 2022 -0500
Removing dependency between agent and secret/resource services
---
agent/pom.xml | 130 +----
agent/{ => service}/pom.xml | 27 +-
.../src/main/assembly/agent-bin-assembly.xml | 0
.../org/apache/airavata/mft/agent/AgentUtil.java | 0
.../org/apache/airavata/mft/agent/AppConfig.java | 19 +
.../org/apache/airavata/mft/agent/MFTAgent.java | 0
.../airavata/mft/agent/TransferOrchestrator.java | 54 +-
.../airavata/mft/agent/TransportMediator.java | 23 +-
.../mft/agent/http/AgentHttpDownloadData.java | 0
.../apache/airavata/mft/agent/http/HttpServer.java | 0
.../airavata/mft/agent/http/HttpServerHandler.java | 4 +-
.../mft/agent/http/HttpServerInitializer.java | 0
.../mft/agent/http/HttpTransferRequestsStore.java | 0
.../mft/agent/ingress/ConsulIngressHandler.java | 23 +-
.../apache/airavata/mft/agent/rpc/RPCParser.java | 58 +-
.../src/main/resources/application.properties | 0
.../distribution/conf/application.properties | 0
.../main/resources/distribution/conf/log4j2.xml | 0
agent/{ => service}/src/main/resources/log4j2.xml | 0
.../resources/distribution/bin/agent-daemon.sh | 113 ----
agent/src/main/resources/distribution/bin/agent.sh | 71 ---
.../src/main/resources/distribution/bin/setenv.sh | 46 --
agent/stub/pom.xml | 84 +++
agent/stub/src/main/proto/MFTAgentStubs.proto | 125 ++++
.../airavata/mft/api/handler/MFTApiHandler.java | 14 +-
common/common-clients/pom.xml | 16 +
.../mft/admin/ControllerRequestBuilder.java | 255 +++++++++
.../apache/airavata/mft/admin/MFTConsulClient.java | 20 +-
controller/pom.xml | 12 +-
.../apache/airavata/mft/controller/AppConfig.java | 31 +
.../airavata/mft/controller/MFTController.java | 21 +-
.../src/main/resources/application.properties | 6 +-
core/pom.xml | 5 +
.../airavata/mft/core/FileResourceMetadata.java | 1 -
.../airavata/mft/core/api/ConnectorConfig.java | 160 ++----
.../airavata/mft/core/api/MetadataCollector.java | 39 +-
python-cli/README.md | 6 +
python-cli/mft_cli/mft_cli/main.py | 24 +
.../backend/file/FileBasedResourceBackend.java | 626 ---------------------
.../backend/file/FileBasedSecretBackend.java | 386 -------------
.../stub/src/main/proto/gcs/GCSCredential.proto | 11 +-
.../transport/azure/AzureMetadataCollector.java | 149 +++--
.../mft/transport/box/BoxMetadataCollector.java | 134 +++--
.../dropbox/DropboxMetadataCollector.java | 133 +++--
.../mft/transport/ftp/FTPMetadataCollector.java | 131 ++---
transport/gcp-transport/pom.xml | 24 +-
.../transport/gcp/GCSIncomingChunkedConnector.java | 96 ++++
.../gcp/GCSIncomingStreamingConnector.java | 79 +++
.../mft/transport/gcp/GCSMetadataCollector.java | 197 +++----
.../gcp/GCSOutgoingStreamingConnector.java | 78 +++
.../airavata/mft/transport/gcp/GCSReceiver.java | 143 -----
.../airavata/mft/transport/gcp/GCSSender.java | 135 -----
.../apache/airavata/mft/transport/gcp/GCSUtil.java | 26 +-
.../transport/local/LocalMetadataCollector.java | 97 ++--
.../transport/odata/ODataIncomingConnector.java | 26 +-
.../transport/odata/ODataMetadataCollector.java | 86 +--
.../mft/transport/s3/S3IncomingConnector.java | 45 +-
.../mft/transport/s3/S3MetadataCollector.java | 115 ++--
.../mft/transport/s3/S3OutgoingConnector.java | 43 +-
.../transport/s3/S3OutgoingStreamingConnector.java | 44 +-
.../mft/transport/scp/SCPIncomingConnector.java | 18 +-
.../mft/transport/scp/SCPMetadataCollector.java | 211 ++-----
.../mft/transport/scp/SCPOutgoingConnector.java | 21 +-
.../transport/swift/SwiftIncomingConnector.java | 77 +--
.../transport/swift/SwiftMetadataCollector.java | 104 ++--
.../transport/swift/SwiftOutgoingConnector.java | 76 +--
66 files changed, 1785 insertions(+), 2913 deletions(-)
diff --git a/agent/pom.xml b/agent/pom.xml
index f944b2f..676936a 100644
--- a/agent/pom.xml
+++ b/agent/pom.xml
@@ -31,132 +31,20 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>mft-agent</artifactId>
+ <packaging>pom</packaging>
+ <modules>
+ <module>stub</module>
+ <module>service</module>
+ </modules>
<dependencies>
<dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>mft-scp-transport</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.custos</groupId>
- <artifactId>custos-java-sdk</artifactId>
- </exclusion>
- </exclusions>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>${protobuf.java}</version>
</dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>mft-local-transport</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>mft-s3-transport</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>mft-box-transport</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>mft-azure-transport</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>mft-gcp-transport</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>mft-dropbox-transport</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>mft-ftp-transport</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>mft-swift-transport</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>mft-odata-transport</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>mft-common-clients</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.orbitz.consul</groupId>
- <artifactId>consul-client</artifactId>
- <version>${consul.client}</version>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- <version>2.10.0</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>2.12.7.1</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.dataformat</groupId>
- <artifactId>jackson-dataformat-xml</artifactId>
- <version>2.10.0</version>
- </dependency>
- <dependency>
- <groupId>com.sun.activation</groupId>
- <artifactId>javax.activation</artifactId>
- <version>1.2.0</version>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-assembly-plugin</artifactId>
- <version>${maven.assembly.plugin}</version>
- <executions>
- <execution>
- <id>mft-agent-distribution-package</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <tarLongFileMode>posix</tarLongFileMode>
- <finalName>${agent.dist.name}</finalName>
- <descriptors>
- <descriptor>src/main/assembly/agent-bin-assembly.xml</descriptor>
- </descriptors>
- <attach>false</attach>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
+ </dependencies>
<properties>
<agent.dist.name>MFT-Agent-0.01</agent.dist.name>
<maven.assembly.plugin>3.1.1</maven.assembly.plugin>
diff --git a/agent/pom.xml b/agent/service/pom.xml
similarity index 88%
copy from agent/pom.xml
copy to agent/service/pom.xml
index f944b2f..4a5d644 100644
--- a/agent/pom.xml
+++ b/agent/service/pom.xml
@@ -24,15 +24,20 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
- <artifactId>airavata-mft</artifactId>
+ <artifactId>mft-agent</artifactId>
<groupId>org.apache.airavata</groupId>
<version>0.01-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>mft-agent</artifactId>
+ <artifactId>mft-agent-service</artifactId>
<dependencies>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>mft-agent-stub</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>mft-scp-transport</artifactId>
@@ -112,17 +117,22 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
- <version>2.10.0</version>
+ <version>2.12.3</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
- <version>2.12.7.1</version>
+ <version>2.12.3</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-xml</artifactId>
- <version>2.10.0</version>
+ <version>2.12.3</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>2.12.3</version>
</dependency>
<dependency>
<groupId>com.sun.activation</groupId>
@@ -157,8 +167,11 @@
</plugin>
</plugins>
</build>
+
<properties>
- <agent.dist.name>MFT-Agent-0.01</agent.dist.name>
- <maven.assembly.plugin>3.1.1</maven.assembly.plugin>
+ <maven.compiler.source>11</maven.compiler.source>
+ <maven.compiler.target>11</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
+
</project>
\ No newline at end of file
diff --git a/agent/src/main/assembly/agent-bin-assembly.xml b/agent/service/src/main/assembly/agent-bin-assembly.xml
similarity index 100%
rename from agent/src/main/assembly/agent-bin-assembly.xml
rename to agent/service/src/main/assembly/agent-bin-assembly.xml
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/AgentUtil.java b/agent/service/src/main/java/org/apache/airavata/mft/agent/AgentUtil.java
similarity index 100%
rename from agent/src/main/java/org/apache/airavata/mft/agent/AgentUtil.java
rename to agent/service/src/main/java/org/apache/airavata/mft/agent/AgentUtil.java
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/AppConfig.java b/agent/service/src/main/java/org/apache/airavata/mft/agent/AppConfig.java
similarity index 77%
rename from agent/src/main/java/org/apache/airavata/mft/agent/AppConfig.java
rename to agent/service/src/main/java/org/apache/airavata/mft/agent/AppConfig.java
index e7b8fbc..7946613 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/AppConfig.java
+++ b/agent/service/src/main/java/org/apache/airavata/mft/agent/AppConfig.java
@@ -17,12 +17,15 @@
package org.apache.airavata.mft.agent;
+import org.apache.airavata.mft.admin.ControllerRequestBuilder;
import org.apache.airavata.mft.admin.MFTConsulClient;
import org.apache.airavata.mft.agent.http.HttpTransferRequestsStore;
import org.apache.airavata.mft.agent.ingress.ConsulIngressHandler;
import org.apache.airavata.mft.agent.rpc.RPCParser;
import org.apache.airavata.mft.resource.client.StorageServiceClient;
import org.apache.airavata.mft.resource.client.StorageServiceClientBuilder;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -41,6 +44,12 @@ public class AppConfig {
@org.springframework.beans.factory.annotation.Value("${resource.service.port}")
private int resourceServicePort;
+ @org.springframework.beans.factory.annotation.Value("${secret.service.host}")
+ private String secretServiceHost;
+
+ @org.springframework.beans.factory.annotation.Value("${secret.service.port}")
+ private int secretServicePort;
+
@Bean
public MFTConsulClient mftConsulClient() {
return new MFTConsulClient(consulHost, consulPort);
@@ -61,6 +70,11 @@ public class AppConfig {
return StorageServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort);
}
+ @Bean
+ public SecretServiceClient secretServiceClient() {
+ return SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort);
+ }
+
@Bean
public ConsulIngressHandler consulIngressHandler() {
return new ConsulIngressHandler();
@@ -72,4 +86,9 @@ public class AppConfig {
return new TransferOrchestrator();
}
+ @Bean
+ public ControllerRequestBuilder controllerRequestBuilder() {
+ return new ControllerRequestBuilder();
+ }
+
}
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java b/agent/service/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
similarity index 100%
rename from agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
rename to agent/service/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/TransferOrchestrator.java b/agent/service/src/main/java/org/apache/airavata/mft/agent/TransferOrchestrator.java
similarity index 74%
rename from agent/src/main/java/org/apache/airavata/mft/agent/TransferOrchestrator.java
rename to agent/service/src/main/java/org/apache/airavata/mft/agent/TransferOrchestrator.java
index 81c7e66..1fc2347 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/TransferOrchestrator.java
+++ b/agent/service/src/main/java/org/apache/airavata/mft/agent/TransferOrchestrator.java
@@ -18,14 +18,12 @@
package org.apache.airavata.mft.agent;
import org.apache.airavata.mft.admin.models.TransferState;
-import org.apache.airavata.mft.api.service.TransferApiRequest;
-import org.apache.airavata.mft.core.FileResourceMetadata;
+import org.apache.airavata.mft.agent.stub.AgentTransferRequest;
+import org.apache.airavata.mft.agent.stub.ResourceMetadata;
import org.apache.airavata.mft.core.MetadataCollectorResolver;
import org.apache.airavata.mft.core.api.ConnectorConfig;
import org.apache.airavata.mft.core.api.MetadataCollector;
import org.apache.airavata.mft.resource.client.StorageServiceClient;
-import org.apache.airavata.mft.resource.stubs.storage.common.StorageTypeResolveRequest;
-import org.apache.airavata.mft.resource.stubs.storage.common.StorageTypeResolveResponse;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -97,7 +95,7 @@ public class TransferOrchestrator {
logger.info("Transfer orchestrator turned off");
}
- public void submitTransferToProcess(String transferId, TransferApiRequest request,
+ public void submitTransferToProcess(String transferId, AgentTransferRequest request,
BiConsumer<String, TransferState> updateStatus,
Consumer<Boolean> createTransferHook) {
long totalPending = totalPendingTransfers.incrementAndGet();
@@ -105,7 +103,7 @@ public class TransferOrchestrator {
transferRequestExecutor.submit(() -> processTransfer(transferId, request, updateStatus, createTransferHook));
}
- public void processTransfer(String transferId, TransferApiRequest request,
+ public void processTransfer(String transferId, AgentTransferRequest request,
BiConsumer<String, TransferState> updateStatus, Consumer<Boolean> createTransferHook) {
try {
@@ -119,49 +117,29 @@ public class TransferOrchestrator {
.setUpdateTimeMils(System.currentTimeMillis())
.setDescription("Starting the transfer"));
- StorageTypeResolveResponse sourceStorageType = storageServiceClient.common()
- .resolveStorageType(StorageTypeResolveRequest.newBuilder()
- .setStorageId(request.getSourceStorageId()).build());
Optional<MetadataCollector> srcMetadataCollectorOp = MetadataCollectorResolver
- .resolveMetadataCollector(sourceStorageType.getStorageType());
- MetadataCollector srcMetadataCollector = srcMetadataCollectorOp.orElseThrow(() -> new Exception("Could not find a metadata collector for source"));
- srcMetadataCollector.init(resourceServiceHost, resourceServicePort, secretServiceHost, secretServicePort);
-
- StorageTypeResolveResponse destStorageType = storageServiceClient.common()
- .resolveStorageType(StorageTypeResolveRequest.newBuilder()
- .setStorageId(request.getSourceStorageId()).build());
+ .resolveMetadataCollector(request.getSourceStorage().getStorageCase().name());
- FileResourceMetadata srcMetadata = srcMetadataCollector.getFileResourceMetadata(
- request.getMftAuthorizationToken(),
- request.getSourcePath(),
- request.getSourceStorageId(),
- request.getSourceToken());
+ MetadataCollector srcMetadataCollector = srcMetadataCollectorOp.orElseThrow(() -> new Exception("Could not find a metadata collector for source"));
+ srcMetadataCollector.init(request.getSourceStorage(), request.getSourceSecret());
+ ResourceMetadata srcMetadata = srcMetadataCollector.getResourceMetadata(request.getSourcePath());
+ if (srcMetadata.getMetadataCase() != ResourceMetadata.MetadataCase.FILE) {
+ throw new Exception("Expected a file as the source but received " + srcMetadata.getMetadataCase().name());
+ }
ConnectorConfig srcCC = ConnectorConfig.ConnectorConfigBuilder.newBuilder()
- .withAuthToken(request.getMftAuthorizationToken())
- .withResourceServiceHost(resourceServiceHost)
- .withResourceServicePort(resourceServicePort)
- .withSecretServiceHost(secretServiceHost)
- .withSecretServicePort(secretServicePort)
.withTransferId(transferId)
- .withStorageId(request.getSourceStorageId())
+ .withSecret(request.getSourceSecret())
+ .withStorage(request.getSourceStorage())
.withResourcePath(request.getSourcePath())
- .withStorageType(sourceStorageType.getStorageType())
- .withCredentialToken(request.getSourceToken())
.withMetadata(srcMetadata).build();
ConnectorConfig dstCC = ConnectorConfig.ConnectorConfigBuilder.newBuilder()
- .withAuthToken(request.getMftAuthorizationToken())
- .withResourceServiceHost(resourceServiceHost)
- .withResourceServicePort(resourceServicePort)
- .withSecretServiceHost(secretServiceHost)
- .withSecretServicePort(secretServicePort)
.withTransferId(transferId)
- .withStorageId(request.getDestinationStorageId())
+ .withSecret(request.getDestinationSecret())
+ .withStorage(request.getDestinationStorage())
.withResourcePath(request.getDestinationPath())
- .withStorageType(destStorageType.getStorageType())
- .withCredentialToken(request.getDestinationToken())
.withMetadata(srcMetadata).build();
updateStatus.accept(transferId, new TransferState()
@@ -173,7 +151,7 @@ public class TransferOrchestrator {
// Save transfer metadata in scheduled path to recover in case of an Agent failures. Recovery is done from controller
createTransferHook.accept(true);
- mediator.transferSingleThread(transferId, request, srcCC, dstCC, updateStatus,
+ mediator.transferSingleThread(transferId, srcCC, dstCC, updateStatus,
(id, transferSuccess) -> {
try {
// Delete scheduled key as the transfer completed / failed if it was placed in current session
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java b/agent/service/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java
similarity index 94%
rename from agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java
rename to agent/service/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java
index bc524fc..e73f52c 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java
+++ b/agent/service/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java
@@ -62,7 +62,6 @@ public class TransportMediator {
}
public void transferSingleThread(String transferId,
- TransferApiRequest request,
ConnectorConfig srcCC,
ConnectorConfig dstCC,
BiConsumer<String, TransferState> onStatusCallback,
@@ -75,14 +74,14 @@ public class TransportMediator {
logger.info("Stating transfer {}", transferId);
Optional<IncomingStreamingConnector> inStreamingConnectorOp = ConnectorResolver
- .resolveIncomingStreamingConnector(srcCC.getStorageType());
+ .resolveIncomingStreamingConnector(srcCC.getStorage().getStorageCase().name());
Optional<OutgoingStreamingConnector> outStreamingConnectorOp = ConnectorResolver
- .resolveOutgoingStreamingConnector(dstCC.getStorageType());
+ .resolveOutgoingStreamingConnector(dstCC.getStorage().getStorageCase().name());
Optional<IncomingChunkedConnector> inChunkedConnectorOp = ConnectorResolver
- .resolveIncomingChunkedConnector(srcCC.getStorageType());
+ .resolveIncomingChunkedConnector(srcCC.getStorage().getStorageCase().name());
Optional<OutgoingChunkedConnector> outChunkedConnectorOp = ConnectorResolver
- .resolveOutgoingChunkedConnector(dstCC.getStorageType());
+ .resolveOutgoingChunkedConnector(dstCC.getStorage().getStorageCase().name());
@@ -104,15 +103,15 @@ public class TransportMediator {
CompletionService<Integer> completionService = new ExecutorCompletionService<Integer>(chunkedExecutorService);
- long fileLength = srcCC.getMetadata().getResourceSize();
+ long fileLength = srcCC.getMetadata().getFile().getResourceSize();
long uploadLength = 0L;
int chunkIdx = 0;
IncomingChunkedConnector inConnector = inChunkedConnectorOp
- .orElseThrow(() -> new Exception("Could not find an in chunked connector for type " + srcCC.getStorageType()));
+ .orElseThrow(() -> new Exception("Could not find an in chunked connector for type " + srcCC.getStorage().getStorageCase().name()));
OutgoingChunkedConnector outConnector = outChunkedConnectorOp
- .orElseThrow(() -> new Exception("Could not find an out chunked connector for type " + dstCC.getStorageType()));
+ .orElseThrow(() -> new Exception("Could not find an out chunked connector for type " + dstCC.getStorage().getStorageCase().name()));
inConnector.init(srcCC);
outConnector.init(dstCC);
@@ -152,10 +151,10 @@ public class TransportMediator {
logger.info("Starting streaming transfer for transfer {}", transferId);
IncomingStreamingConnector inConnector = inStreamingConnectorOp
- .orElseThrow(() -> new Exception("Could not find an in streaming connector for type " + srcCC.getStorageType()));
+ .orElseThrow(() -> new Exception("Could not find an in streaming connector for type " + srcCC.getStorage().getStorageCase().name()));
OutgoingStreamingConnector outConnector = outStreamingConnectorOp
- .orElseThrow(() -> new Exception("Could not find an out streaming connector for type " + dstCC.getStorageType()));
+ .orElseThrow(() -> new Exception("Could not find an out streaming connector for type " + dstCC.getStorage().getStorageCase().name()));
inConnector.init(srcCC);
outConnector.init(dstCC);
@@ -180,7 +179,7 @@ public class TransportMediator {
logger.info("Status monitor is exiting for transfer {}", transferId);
break;
}
- double transferPercentage = countAtomic.get() * 100.0 / srcCC.getMetadata().getResourceSize();
+ double transferPercentage = countAtomic.get() * 100.0 / srcCC.getMetadata().getFile().getResourceSize();
logger.info("Transfer percentage for transfer {} {}", transferId, transferPercentage);
onStatusCallback.accept(transferId, new TransferState()
.setPercentage(transferPercentage)
@@ -216,7 +215,7 @@ public class TransportMediator {
double time = (endTime - start) / 1000.0;
logger.info("Transfer {} completed. Time {} S. Speed {} MB/s", transferId, time,
- (srcCC.getMetadata().getResourceSize() * 1.0 / time) / (1024 * 1024));
+ (srcCC.getMetadata().getFile().getResourceSize() * 1.0 / time) / (1024 * 1024));
onStatusCallback.accept(transferId, new TransferState()
.setPercentage(100)
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/http/AgentHttpDownloadData.java b/agent/service/src/main/java/org/apache/airavata/mft/agent/http/AgentHttpDownloadData.java
similarity index 100%
rename from agent/src/main/java/org/apache/airavata/mft/agent/http/AgentHttpDownloadData.java
rename to agent/service/src/main/java/org/apache/airavata/mft/agent/http/AgentHttpDownloadData.java
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/http/HttpServer.java b/agent/service/src/main/java/org/apache/airavata/mft/agent/http/HttpServer.java
similarity index 100%
rename from agent/src/main/java/org/apache/airavata/mft/agent/http/HttpServer.java
rename to agent/service/src/main/java/org/apache/airavata/mft/agent/http/HttpServer.java
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/http/HttpServerHandler.java b/agent/service/src/main/java/org/apache/airavata/mft/agent/http/HttpServerHandler.java
similarity index 98%
rename from agent/src/main/java/org/apache/airavata/mft/agent/http/HttpServerHandler.java
rename to agent/service/src/main/java/org/apache/airavata/mft/agent/http/HttpServerHandler.java
index 2bc173f..6366502 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/http/HttpServerHandler.java
+++ b/agent/service/src/main/java/org/apache/airavata/mft/agent/http/HttpServerHandler.java
@@ -67,11 +67,11 @@ public class HttpServerHandler extends SimpleChannelInboundHandler<FullHttpReque
return;
}
- long fileLength = downloadData.getConnectorConfig().getMetadata().getResourceSize();
+ long fileLength = downloadData.getConnectorConfig().getMetadata().getFile().getResourceSize();
HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
HttpUtil.setContentLength(response, fileLength);
- setContentTypeHeader(response, downloadData.getConnectorConfig().getMetadata().getFriendlyName());
+ setContentTypeHeader(response, downloadData.getConnectorConfig().getMetadata().getFile().getFriendlyName());
if (HttpUtil.isKeepAlive(request)) {
response.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE);
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/http/HttpServerInitializer.java b/agent/service/src/main/java/org/apache/airavata/mft/agent/http/HttpServerInitializer.java
similarity index 100%
rename from agent/src/main/java/org/apache/airavata/mft/agent/http/HttpServerInitializer.java
rename to agent/service/src/main/java/org/apache/airavata/mft/agent/http/HttpServerInitializer.java
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/http/HttpTransferRequestsStore.java b/agent/service/src/main/java/org/apache/airavata/mft/agent/http/HttpTransferRequestsStore.java
similarity index 100%
rename from agent/src/main/java/org/apache/airavata/mft/agent/http/HttpTransferRequestsStore.java
rename to agent/service/src/main/java/org/apache/airavata/mft/agent/http/HttpTransferRequestsStore.java
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/ingress/ConsulIngressHandler.java b/agent/service/src/main/java/org/apache/airavata/mft/agent/ingress/ConsulIngressHandler.java
similarity index 92%
rename from agent/src/main/java/org/apache/airavata/mft/agent/ingress/ConsulIngressHandler.java
rename to agent/service/src/main/java/org/apache/airavata/mft/agent/ingress/ConsulIngressHandler.java
index f3ffdd2..30b8f28 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/ingress/ConsulIngressHandler.java
+++ b/agent/service/src/main/java/org/apache/airavata/mft/agent/ingress/ConsulIngressHandler.java
@@ -18,13 +18,14 @@
package org.apache.airavata.mft.agent.ingress;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.protobuf.util.JsonFormat;
+import com.google.protobuf.InvalidProtocolBufferException;
import com.orbitz.consul.ConsulException;
import com.orbitz.consul.cache.ConsulCache;
import com.orbitz.consul.cache.KVCache;
import com.orbitz.consul.model.kv.Value;
import com.orbitz.consul.model.session.ImmutableSession;
import com.orbitz.consul.model.session.SessionCreatedResponse;
+import com.orbitz.consul.option.PutOptions;
import org.apache.airavata.mft.admin.MFTConsulClient;
import org.apache.airavata.mft.admin.MFTConsulClientException;
import org.apache.airavata.mft.admin.models.AgentInfo;
@@ -33,6 +34,7 @@ import org.apache.airavata.mft.admin.models.rpc.SyncRPCResponse;
import org.apache.airavata.mft.agent.AgentUtil;
import org.apache.airavata.mft.agent.TransferOrchestrator;
import org.apache.airavata.mft.agent.rpc.RPCParser;
+import org.apache.airavata.mft.agent.stub.AgentTransferRequest;
import org.apache.airavata.mft.api.service.TransferApiRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -93,26 +95,27 @@ public class ConsulIngressHandler {
transferCacheListener = newValues -> {
newValues.values().forEach(value -> {
- Optional<String> decodedValue = value.getValueAsString();
+ Optional<byte[]> decodedValue = value.getValueAsBytes();
String transferId = value.getKey().substring(value.getKey().lastIndexOf("/") + 1);
- decodedValue.ifPresent(reqJson -> {
+ decodedValue.ifPresent(reqBytes -> {
mftConsulClient.getKvClient().deleteKey(value.getKey());
- TransferApiRequest.Builder builder = TransferApiRequest.newBuilder();
+ AgentTransferRequest.Builder builder = null;
try {
- JsonFormat.parser().merge(reqJson, builder);
-
- } catch (Exception e) {
- logger.error("Failed to parse json string {} into a transfer request", reqJson);
+ builder = AgentTransferRequest.newBuilder().mergeFrom(reqBytes);
+ } catch (InvalidProtocolBufferException e) {
+ logger.error("Failed to merge transfer {} from bytes", transferId, e);
return;
}
- TransferApiRequest request = builder.build();
+
+ AgentTransferRequest request = builder.build();
transferOrchestrator.submitTransferToProcess(transferId, request,
AgentUtil.throwingBiConsumerWrapper((id, st) -> {
mftConsulClient.submitTransferStateToProcess(id, agentId, st.setPublisher(agentId));
}),
AgentUtil.throwingConsumerWrapper(create -> {
if (create) {
- mftConsulClient.getKvClient().putValue(MFTConsulClient.AGENTS_SCHEDULED_PATH + agentId + "/" + session + "/" + transferId, reqJson);
+ mftConsulClient.getKvClient().putValue(MFTConsulClient.AGENTS_SCHEDULED_PATH + agentId + "/" + session + "/" + transferId,
+ reqBytes, 0L, PutOptions.BLANK);
} else {
mftConsulClient.getKvClient().deleteKey(MFTConsulClient.AGENTS_SCHEDULED_PATH + agentId + "/" + session + "/" + transferId);
}
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java b/agent/service/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java
similarity index 78%
rename from agent/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java
rename to agent/service/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java
index f41ecdb..36adad7 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java
+++ b/agent/service/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java
@@ -19,14 +19,14 @@ package org.apache.airavata.mft.agent.rpc;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.protobuf.util.JsonFormat;
+import org.apache.airavata.mft.admin.ControllerRequestBuilder;
import org.apache.airavata.mft.admin.models.rpc.SyncRPCRequest;
import org.apache.airavata.mft.admin.models.rpc.SyncRPCResponse;
import org.apache.airavata.mft.agent.http.AgentHttpDownloadData;
import org.apache.airavata.mft.agent.http.HttpTransferRequestsStore;
+import org.apache.airavata.mft.agent.stub.*;
import org.apache.airavata.mft.common.AuthToken;
import org.apache.airavata.mft.core.ConnectorResolver;
-import org.apache.airavata.mft.core.DirectoryResourceMetadata;
-import org.apache.airavata.mft.core.FileResourceMetadata;
import org.apache.airavata.mft.core.MetadataCollectorResolver;
import org.apache.airavata.mft.core.api.ConnectorConfig;
import org.apache.airavata.mft.core.api.IncomingChunkedConnector;
@@ -35,6 +35,7 @@ import org.apache.airavata.mft.core.api.MetadataCollector;
import org.apache.airavata.mft.resource.client.StorageServiceClient;
import org.apache.airavata.mft.resource.stubs.storage.common.StorageTypeResolveRequest;
import org.apache.airavata.mft.resource.stubs.storage.common.StorageTypeResolveResponse;
+import org.apache.commons.lang3.tuple.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -45,18 +46,6 @@ public class RPCParser {
private static final Logger logger = LoggerFactory.getLogger(RPCParser.class);
- @org.springframework.beans.factory.annotation.Value("${resource.service.host}")
- private String resourceServiceHost;
-
- @org.springframework.beans.factory.annotation.Value("${resource.service.port}")
- private int resourceServicePort;
-
- @org.springframework.beans.factory.annotation.Value("${secret.service.host}")
- private String secretServiceHost;
-
- @org.springframework.beans.factory.annotation.Value("${secret.service.port}")
- private int secretServicePort;
-
@org.springframework.beans.factory.annotation.Value("${agent.advertised.url}")
private String agentAdvertisedUrl;
@@ -66,6 +55,9 @@ public class RPCParser {
@Autowired
private StorageServiceClient storageServiceClient;
+ @Autowired
+ private ControllerRequestBuilder controllerRequestBuilder;
+
public String resolveRPCRequest(SyncRPCRequest request) throws Exception {
// TODO implement using the reflection
ObjectMapper mapper = new ObjectMapper();
@@ -89,9 +81,10 @@ public class RPCParser {
if (metadataCollectorOp.isPresent()) {
MetadataCollector metadataCollector = metadataCollectorOp.get();
- metadataCollector.init(resourceServiceHost, resourceServicePort, secretServiceHost, secretServicePort);
- FileResourceMetadata fileResourceMetadata = metadataCollector
- .getFileResourceMetadata(mftAuthorizationToken, resourcePath, storageId, resourceToken);
+
+ Pair<StorageWrapper, SecretWrapper> credentials = controllerRequestBuilder.createCredentials(storageId, resourceToken);
+ metadataCollector.init(credentials.getLeft() , credentials.getRight());
+ FileMetadata fileResourceMetadata = metadataCollector.getResourceMetadata(resourcePath).getFile();
return mapper.writeValueAsString(fileResourceMetadata);
}
break;
@@ -112,9 +105,12 @@ public class RPCParser {
metadataCollectorOp = MetadataCollectorResolver.resolveMetadataCollector(storageType.getStorageType());
if (metadataCollectorOp.isPresent()) {
MetadataCollector metadataCollector = metadataCollectorOp.get();
- metadataCollector.init(resourceServiceHost, resourceServicePort, secretServiceHost, secretServicePort);
- DirectoryResourceMetadata dirResourceMetadata = metadataCollector
- .getDirectoryResourceMetadata(mftAuthorizationToken, resourcePath, storageId, resourceToken);
+
+ Pair<StorageWrapper, SecretWrapper> credentials = controllerRequestBuilder.createCredentials(storageId, resourceToken);
+ metadataCollector.init(credentials.getLeft() , credentials.getRight());
+
+ DirectoryMetadata dirResourceMetadata = metadataCollector
+ .getResourceMetadata(resourcePath).getDirectory();
return mapper.writeValueAsString(dirResourceMetadata);
}
break;
@@ -138,26 +134,18 @@ public class RPCParser {
if (metadataCollectorOp.isPresent() && (connectorStreamingOp.isPresent() || connectorChunkedOp.isPresent())) {
MetadataCollector metadataCollector = metadataCollectorOp.get();
- metadataCollector.init(resourceServiceHost, resourceServicePort, secretServiceHost, secretServicePort);
- FileResourceMetadata fileResourceMetadata = metadataCollector.getFileResourceMetadata(
- mftAuthorizationToken,
- resourcePath,
- sourceStorageId,
- sourceToken);
+ Pair<StorageWrapper, SecretWrapper> credentials = controllerRequestBuilder.createCredentials(sourceStorageId, sourceToken);
+ metadataCollector.init(credentials.getLeft() , credentials.getRight());
+
+ ResourceMetadata resourceMetadata = metadataCollector.getResourceMetadata(resourcePath);
AgentHttpDownloadData.AgentHttpDownloadDataBuilder agentHttpDownloadDataBuilder = AgentHttpDownloadData.AgentHttpDownloadDataBuilder.newBuilder()
.withConnectorConfig(ConnectorConfig.ConnectorConfigBuilder.newBuilder()
- .withResourceServiceHost(resourceServiceHost)
- .withResourceServicePort(resourceServicePort)
- .withSecretServiceHost(secretServiceHost)
- .withSecretServicePort(secretServicePort)
- .withStorageId(sourceStorageId)
- .withStorageType(storageType.getStorageType())
+ .withStorage(credentials.getLeft())
+ .withSecret(credentials.getRight())
.withResourcePath(resourcePath)
- .withCredentialToken(sourceToken)
- .withAuthToken(mftAuthorizationToken)
- .withMetadata(fileResourceMetadata).build());
+ .withMetadata(resourceMetadata).build());
connectorStreamingOp.ifPresent(agentHttpDownloadDataBuilder::withIncomingStreamingConnector);
connectorChunkedOp.ifPresent(agentHttpDownloadDataBuilder::withIncomingChunkedConnector);
diff --git a/agent/src/main/resources/application.properties b/agent/service/src/main/resources/application.properties
similarity index 100%
rename from agent/src/main/resources/application.properties
rename to agent/service/src/main/resources/application.properties
diff --git a/agent/src/main/resources/distribution/conf/application.properties b/agent/service/src/main/resources/distribution/conf/application.properties
similarity index 100%
rename from agent/src/main/resources/distribution/conf/application.properties
rename to agent/service/src/main/resources/distribution/conf/application.properties
diff --git a/agent/src/main/resources/distribution/conf/log4j2.xml b/agent/service/src/main/resources/distribution/conf/log4j2.xml
similarity index 100%
rename from agent/src/main/resources/distribution/conf/log4j2.xml
rename to agent/service/src/main/resources/distribution/conf/log4j2.xml
diff --git a/agent/src/main/resources/log4j2.xml b/agent/service/src/main/resources/log4j2.xml
similarity index 100%
rename from agent/src/main/resources/log4j2.xml
rename to agent/service/src/main/resources/log4j2.xml
diff --git a/agent/src/main/resources/distribution/bin/agent-daemon.sh b/agent/src/main/resources/distribution/bin/agent-daemon.sh
deleted file mode 100644
index 20272a2..0000000
--- a/agent/src/main/resources/distribution/bin/agent-daemon.sh
+++ /dev/null
@@ -1,113 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-. `dirname $0`/setenv.sh
-# Capture user's working dir before changing directory
-CWD="$PWD"
-cd ${AIRAVATA_HOME}/bin
-LOGO_FILE="logo.txt"
-
-JAVA_OPTS="-Dspring.config.location=${AIRAVATA_HOME}/conf/ -Dairavata.home=${AIRAVATA_HOME} -Dlog4j.configurationFile=file:${AIRAVATA_HOME}/conf/log4j2.xml"
-AIRAVATA_COMMAND=""
-EXTRA_ARGS=""
-SERVERS=""
-LOGO=true
-IS_SUBSET=false
-SUBSET=""
-DEFAULT_LOG_FILE="${AIRAVATA_HOME}/logs/airavata-daemon.out"
-LOG_FILE=$DEFAULT_LOG_FILE
-
-SERVICE_NAME="MFT Agent"
-PID_PATH_NAME="${AIRAVATA_HOME}/bin/service-pid"
-
-case $1 in
- start)
- echo "Starting $SERVICE_NAME ..."
- if [ ! -f $PID_PATH_NAME ]; then
- nohup java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
- org.apache.airavata.mft.agent.MFTAgent ${AIRAVATA_COMMAND} $* > $LOG_FILE 2>&1 &
- echo $! > $PID_PATH_NAME
- echo "$SERVICE_NAME started ..."
- else
- echo "$SERVICE_NAME is already running ..."
- fi
- ;;
- stop)
- if [ -f $PID_PATH_NAME ]; then
- PID=$(cat $PID_PATH_NAME);
- echo "$SERVICE_NAME stoping ..."
- kill $PID;
- RETRY=0
- while kill -0 $PID 2> /dev/null; do
- echo "Waiting for the process $PID to be stopped"
- RETRY=`expr ${RETRY} + 1`
- if [ "${RETRY}" -gt "20" ]
- then
- echo "Forcefully killing the process as it is not responding ..."
- kill -9 $PID
- fi
- sleep 1
- done
- echo "$SERVICE_NAME stopped ..."
- rm $PID_PATH_NAME
- else
- echo "$SERVICE_NAME is not running ..."
- fi
- ;;
- restart)
- if [ -f $PID_PATH_NAME ]; then
- PID=$(cat $PID_PATH_NAME);
- echo "$SERVICE_NAME stopping ...";
- kill $PID;
- RETRY=0
- while kill -0 $PID 2> /dev/null; do
- echo "Waiting for the process $PID to be stopped"
- RETRY=`expr ${RETRY} + 1`
- if [ "${RETRY}" -gt "20" ]
- then
- echo "Forcefully killing the process as it is not responding ..."
- kill -9 $PID
- fi
- sleep 1
- done
- echo "$SERVICE_NAME stopped ...";
- rm $PID_PATH_NAME
- echo "$SERVICE_NAME starting ..."
- nohup java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
- org.apache.airavata.mft.agent.MFTAgent ${AIRAVATA_COMMAND} $* > $LOG_FILE 2>&1 &
- echo $! > $PID_PATH_NAME
- echo "$SERVICE_NAME started ..."
- else
- echo "$SERVICE_NAME is not running ..."
- fi
- ;;
- -h)
- echo "Usage: agent-daemon.sh"
-
- echo "command options:"
- echo " start Start server in daemon mode"
- echo " stop Stop server running in daemon mode"
- echo " restart Restart server in daemon mode"
- echo " -log <LOG_FILE> Where to redirect stdout/stderr (defaults to $DEFAULT_LOG_FILE)"
- echo " -h Display this help and exit"
- shift
- exit 0
- ;;
-esac
-
diff --git a/agent/src/main/resources/distribution/bin/agent.sh b/agent/src/main/resources/distribution/bin/agent.sh
deleted file mode 100644
index 6ba0db7..0000000
--- a/agent/src/main/resources/distribution/bin/agent.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-. `dirname $0`/setenv.sh
-# Capture user's working dir before changing directory
-CWD="$PWD"
-cd ${AIRAVATA_HOME}/bin
-LOGO_FILE="logo.txt"
-
-JAVA_OPTS="-Dspring.config.location=${AIRAVATA_HOME}/conf/ -Dairavata.home=${AIRAVATA_HOME} -Dlog4j.configurationFile=file:${AIRAVATA_HOME}/conf/log4j2.xml"
-AIRAVATA_COMMAND=""
-EXTRA_ARGS=""
-SERVERS=""
-IS_SUBSET=false
-SUBSET=""
-DEFAULT_LOG_FILE="${AIRAVATA_HOME}/logs/airavata.out"
-LOG_FILE=$DEFAULT_LOG_FILE
-
-# parse command arguments
-for var in "$@"
-do
- case ${var} in
- -xdebug)
- AIRAVATA_COMMAND="${AIRAVATA_COMMAND}"
- JAVA_OPTS="$JAVA_OPTS -Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,server=y,address=*:8000"
- shift
- ;;
- -log)
- shift
- LOG_FILE="$1"
- shift
- # If relative path, expand to absolute path using the user's $CWD
- if [ -z "`echo "$LOG_FILE" | egrep "^/"`" ]; then
- LOG_FILE="${CWD}/${LOG_FILE}"
- fi
- ;;
- -h)
- echo "Usage: agent.sh"
-
- echo "command options:"
- echo " -xdebug Start MFT Agent under JPDA debugger"
- echo " -h Display this help and exit"
- shift
- exit 0
- ;;
- *)
- EXTRA_ARGS="${EXTRA_ARGS} ${var}"
- shift
- ;;
- esac
-done
-
-java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
- org.apache.airavata.mft.agent.MFTAgent ${AIRAVATA_COMMAND} $*
-
diff --git a/agent/src/main/resources/distribution/bin/setenv.sh b/agent/src/main/resources/distribution/bin/setenv.sh
deleted file mode 100755
index 9e894e1..0000000
--- a/agent/src/main/resources/distribution/bin/setenv.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# resolve links - $0 may be a softlink
-PRG="$0"
-
-while [ -h "$PRG" ]; do
- ls=`ls -ld "$PRG"`
- link=`expr "$ls" : '.*-> \(.*\)$'`
- if expr "$link" : '.*/.*' > /dev/null; then
- PRG="$link"
- else
- PRG=`dirname "$PRG"`/"$link"
- fi
-done
-
-PRGDIR=`dirname "$PRG"`
-
-# Only set AIRAVATA_HOME if not already set
-[ -z "$AIRAVATA_HOME" ] && AIRAVATA_HOME=`cd "$PRGDIR/.." ; pwd`
-
-AIRAVATA_CLASSPATH=""
-
-for f in "$AIRAVATA_HOME"/lib/*.jar
-do
- AIRAVATA_CLASSPATH="$AIRAVATA_CLASSPATH":$f
-done
-
-export AIRAVATA_HOME
-export AIRAVATA_CLASSPATH
diff --git a/agent/stub/pom.xml b/agent/stub/pom.xml
new file mode 100644
index 0000000..6c3420b
--- /dev/null
+++ b/agent/stub/pom.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>mft-agent</artifactId>
+ <groupId>org.apache.airavata</groupId>
+ <version>0.01-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>mft-agent-stub</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>mft-resource-service-stub</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>mft-secret-service-stub</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <extensions>
+ <extension>
+ <groupId>kr.motd.maven</groupId>
+ <artifactId>os-maven-plugin</artifactId>
+ <version>${os.maven.plugin}</version>
+ </extension>
+ </extensions>
+ <plugins>
+ <plugin>
+ <groupId>org.xolstice.maven.plugins</groupId>
+ <artifactId>protobuf-maven-plugin</artifactId>
+ <version>${protobuf.maven.plugin}</version>
+ <configuration>
+ <protocArtifact>com.google.protobuf:protoc:3.0.2:exe:${os.detected.classifier}</protocArtifact>
+ <pluginId>grpc-java</pluginId>
+ <pluginArtifact>io.grpc:protoc-gen-grpc-java:1.0.1:exe:${os.detected.classifier}</pluginArtifact>
+ <additionalProtoPathElements>
+ <additionalProtoPathElement>../../common/mft-common-proto/src/main/proto</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/resource-service/stub/src/main/proto/azure</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/resource-service/stub/src/main/proto/box</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/resource-service/stub/src/main/proto/dropbox</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/resource-service/stub/src/main/proto/ftp</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/resource-service/stub/src/main/proto/gcs</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/resource-service/stub/src/main/proto/local</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/resource-service/stub/src/main/proto/odata</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/resource-service/stub/src/main/proto/s3</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/resource-service/stub/src/main/proto/scp</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/resource-service/stub/src/main/proto/swift</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/secret-service/stub/src/main/proto/azure</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/secret-service/stub/src/main/proto/box</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/secret-service/stub/src/main/proto/dropbox</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/secret-service/stub/src/main/proto/ftp</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/secret-service/stub/src/main/proto/gcs</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/secret-service/stub/src/main/proto/odata</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/secret-service/stub/src/main/proto/s3</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/secret-service/stub/src/main/proto/scp</additionalProtoPathElement>
+ <additionalProtoPathElement>../../services/secret-service/stub/src/main/proto/swift</additionalProtoPathElement>
+ </additionalProtoPathElements>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>compile</goal>
+ <goal>compile-custom</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <properties>
+ <maven.compiler.source>11</maven.compiler.source>
+ <maven.compiler.target>11</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+</project>
\ No newline at end of file
diff --git a/agent/stub/src/main/proto/MFTAgentStubs.proto b/agent/stub/src/main/proto/MFTAgentStubs.proto
new file mode 100644
index 0000000..0ea914a
--- /dev/null
+++ b/agent/stub/src/main/proto/MFTAgentStubs.proto
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = "proto3";
+
+option java_multiple_files = true;
+package org.apache.airavata.mft.agent.stub;
+
+import "AzureStorage.proto";
+import "BoxStorage.proto";
+import "DropboxStorage.proto";
+import "FTPStorage.proto";
+import "GCSStorage.proto";
+import "LocalStorage.proto";
+import "ODataStorage.proto";
+import "S3Storage.proto";
+import "SCPStorage.proto";
+import "SwiftStorage.proto";
+
+import "AzureCredential.proto";
+import "BoxCredential.proto";
+import "DropboxCredential.proto";
+import "FTPCredential.proto";
+import "GCSCredential.proto";
+import "ODataCredential.proto";
+import "S3Credential.proto";
+import "SCPCredential.proto";
+import "SwiftCredential.proto";
+
+message StorageWrapper {
+ oneof storage {
+ org.apache.airavata.mft.resource.stubs.azure.storage.AzureStorage azure = 1;
+ org.apache.airavata.mft.resource.stubs.box.storage.BoxStorage box = 2;
+ org.apache.airavata.mft.resource.stubs.dropbox.storage.DropboxStorage dropbox = 3;
+ org.apache.airavata.mft.resource.stubs.ftp.storage.FTPStorage ftp = 4;
+ org.apache.airavata.mft.resource.stubs.gcs.storage.GCSStorage gcs = 5;
+ org.apache.airavata.mft.resource.stubs.local.storage.LocalStorage local = 6;
+ org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorage odata = 7;
+ org.apache.airavata.mft.resource.stubs.s3.storage.S3Storage s3 = 8;
+ org.apache.airavata.mft.resource.stubs.scp.storage.SCPStorage scp = 9;
+ org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage swift = 10;
+ }
+}
+
+message SecretWrapper {
+ oneof secret {
+ org.apache.airavata.mft.credential.stubs.azure.AzureSecret azure = 1;
+ org.apache.airavata.mft.credential.stubs.box.BoxSecret box = 2;
+ org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecret dropbox = 3;
+ org.apache.airavata.mft.credential.stubs.ftp.FTPSecret ftp = 4;
+ org.apache.airavata.mft.credential.stubs.gcs.GCSSecret gcs = 5;
+ org.apache.airavata.mft.credential.stubs.odata.ODataSecret odata = 6;
+ org.apache.airavata.mft.credential.stubs.s3.S3Secret s3 = 7;
+ org.apache.airavata.mft.credential.stubs.scp.SCPSecret scp = 8;
+ org.apache.airavata.mft.credential.stubs.swift.SwiftSecret swift = 9;
+ }
+}
+
+message FileResourceMetadataGetRequest {
+ string resourcePath = 1;
+ string storageId = 2;
+ string resourceToken = 3;
+ string mftAuthorizationToken = 4;
+}
+
+message AgentTransferRequest {
+ StorageWrapper sourceStorage = 1;
+ SecretWrapper sourceSecret = 2;
+ string sourcePath = 3;
+ StorageWrapper destinationStorage = 4;
+ SecretWrapper destinationSecret = 5;
+ string destinationPath = 6;
+}
+
+message FileMetadata {
+ string friendlyName = 1;
+ int64 resourceSize = 2;
+ int64 createdTime = 3;
+ int64 updateTime = 4;
+ string md5sum = 5;
+ string resourcePath = 6;
+}
+
+message DirectoryMetadata {
+ string friendlyName = 1;
+ int64 createdTime = 2;
+ int64 updateTime = 3;
+ string resourcePath = 4;
+ repeated DirectoryMetadata directories = 5;
+ repeated FileMetadata files = 6;
+}
+
+enum MetadataFetchError {
+ NOT_FOUND = 0;
+ NO_PERMISSION = 1;
+}
+
+message ResourceMetadata {
+ oneof metadata {
+ FileMetadata file = 1;
+ DirectoryMetadata directory = 2;
+ MetadataFetchError error = 3;
+ }
+}
+
+message GetResourceMetadataRequest {
+ string resourcePath = 1;
+ StorageWrapper storage = 2;
+ SecretWrapper secret = 3;
+}
+
diff --git a/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java b/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java
index 1078f55..4cc5473 100644
--- a/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java
+++ b/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java
@@ -27,11 +27,9 @@ import org.apache.airavata.mft.admin.models.AgentInfo;
import org.apache.airavata.mft.admin.models.TransferState;
import org.apache.airavata.mft.admin.models.rpc.SyncRPCRequest;
import org.apache.airavata.mft.admin.models.rpc.SyncRPCResponse;
+import org.apache.airavata.mft.agent.stub.DirectoryMetadata;
+import org.apache.airavata.mft.agent.stub.FileMetadata;
import org.apache.airavata.mft.api.service.*;
-import org.apache.airavata.mft.core.DirectoryResourceMetadata;
-import org.apache.airavata.mft.core.FileResourceMetadata;
-import org.apache.airavata.mft.core.MetadataCollectorResolver;
-import org.apache.airavata.mft.core.api.MetadataCollector;
import org.apache.commons.lang3.tuple.Pair;
import org.dozer.DozerBeanMapper;
import org.lognet.springboot.grpc.GRpcService;
@@ -278,7 +276,7 @@ public class MFTApiHandler extends MFTTransferServiceGrpc.MFTTransferServiceImpl
switch (rpcResponse.getResponseStatus()) {
case SUCCESS:
- FileResourceMetadata fileResourceMetadata = jsonMapper.readValue(rpcResponse.getResponseAsStr(), FileResourceMetadata.class);
+ FileMetadata fileResourceMetadata = jsonMapper.readValue(rpcResponse.getResponseAsStr(), FileMetadata.class);
FileMetadataResponse.Builder responseBuilder = FileMetadataResponse.newBuilder();
dozerBeanMapper.map(fileResourceMetadata, responseBuilder);
responseObserver.onNext(responseBuilder.build());
@@ -328,18 +326,18 @@ public class MFTApiHandler extends MFTTransferServiceGrpc.MFTTransferServiceImpl
switch (rpcResponse.getResponseStatus()) {
case SUCCESS:
- DirectoryResourceMetadata dirResourceMetadata = jsonMapper.readValue(rpcResponse.getResponseAsStr(), DirectoryResourceMetadata.class);
+ DirectoryMetadata dirResourceMetadata = jsonMapper.readValue(rpcResponse.getResponseAsStr(), DirectoryMetadata.class);
DirectoryMetadataResponse.Builder responseBuilder = DirectoryMetadataResponse.newBuilder();
dozerBeanMapper.map(dirResourceMetadata, responseBuilder);
// As dozer mapper can't map collections in protobuf, do it manually for directories and files
- for (DirectoryResourceMetadata dm : dirResourceMetadata.getDirectories()) {
+ for (DirectoryMetadata dm : dirResourceMetadata.getDirectoriesList()) {
DirectoryMetadataResponse.Builder db = DirectoryMetadataResponse.newBuilder();
dozerBeanMapper.map(dm, db);
responseBuilder.addDirectories(db);
}
- for (FileResourceMetadata fm : dirResourceMetadata.getFiles()) {
+ for (FileMetadata fm : dirResourceMetadata.getFilesList()) {
FileMetadataResponse.Builder fb = FileMetadataResponse.newBuilder();
dozerBeanMapper.map(fm, fb);
responseBuilder.addFiles(fb);
diff --git a/common/common-clients/pom.xml b/common/common-clients/pom.xml
index 19fb766..b56fd67 100644
--- a/common/common-clients/pom.xml
+++ b/common/common-clients/pom.xml
@@ -53,5 +53,21 @@
<artifactId>mft-api-stub</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>mft-agent-stub</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>mft-secret-service-client</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>mft-resource-service-client</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
</dependencies>
</project>
\ No newline at end of file
diff --git a/common/common-clients/src/main/java/org/apache/airavata/mft/admin/ControllerRequestBuilder.java b/common/common-clients/src/main/java/org/apache/airavata/mft/admin/ControllerRequestBuilder.java
new file mode 100644
index 0000000..a2ba9ef
--- /dev/null
+++ b/common/common-clients/src/main/java/org/apache/airavata/mft/admin/ControllerRequestBuilder.java
@@ -0,0 +1,255 @@
+package org.apache.airavata.mft.admin;
+
+import org.apache.airavata.mft.agent.stub.AgentTransferRequest;
+import org.apache.airavata.mft.agent.stub.SecretWrapper;
+import org.apache.airavata.mft.agent.stub.StorageWrapper;
+import org.apache.airavata.mft.api.service.TransferApiRequest;
+import org.apache.airavata.mft.credential.stubs.azure.AzureSecret;
+import org.apache.airavata.mft.credential.stubs.azure.AzureSecretGetRequest;
+import org.apache.airavata.mft.credential.stubs.box.BoxSecret;
+import org.apache.airavata.mft.credential.stubs.box.BoxSecretGetRequest;
+import org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecret;
+import org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecretGetRequest;
+import org.apache.airavata.mft.credential.stubs.ftp.FTPSecret;
+import org.apache.airavata.mft.credential.stubs.ftp.FTPSecretGetRequest;
+import org.apache.airavata.mft.credential.stubs.gcs.GCSSecret;
+import org.apache.airavata.mft.credential.stubs.gcs.GCSSecretGetRequest;
+import org.apache.airavata.mft.credential.stubs.odata.ODataSecret;
+import org.apache.airavata.mft.credential.stubs.odata.ODataSecretGetRequest;
+import org.apache.airavata.mft.credential.stubs.s3.S3Secret;
+import org.apache.airavata.mft.credential.stubs.s3.S3SecretGetRequest;
+import org.apache.airavata.mft.credential.stubs.scp.SCPSecret;
+import org.apache.airavata.mft.credential.stubs.scp.SCPSecretGetRequest;
+import org.apache.airavata.mft.credential.stubs.swift.SwiftSecret;
+import org.apache.airavata.mft.credential.stubs.swift.SwiftSecretGetRequest;
+import org.apache.airavata.mft.resource.client.StorageServiceClient;
+import org.apache.airavata.mft.resource.client.StorageServiceClientBuilder;
+import org.apache.airavata.mft.resource.stubs.azure.storage.AzureStorage;
+import org.apache.airavata.mft.resource.stubs.azure.storage.AzureStorageGetRequest;
+import org.apache.airavata.mft.resource.stubs.box.storage.BoxStorage;
+import org.apache.airavata.mft.resource.stubs.box.storage.BoxStorageGetRequest;
+import org.apache.airavata.mft.resource.stubs.dropbox.storage.DropboxStorage;
+import org.apache.airavata.mft.resource.stubs.dropbox.storage.DropboxStorageGetRequest;
+import org.apache.airavata.mft.resource.stubs.ftp.storage.FTPStorage;
+import org.apache.airavata.mft.resource.stubs.ftp.storage.FTPStorageGetRequest;
+import org.apache.airavata.mft.resource.stubs.gcs.storage.GCSStorage;
+import org.apache.airavata.mft.resource.stubs.gcs.storage.GCSStorageGetRequest;
+import org.apache.airavata.mft.resource.stubs.local.storage.LocalStorage;
+import org.apache.airavata.mft.resource.stubs.local.storage.LocalStorageGetRequest;
+import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorage;
+import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageGetRequest;
+import org.apache.airavata.mft.resource.stubs.s3.storage.S3Storage;
+import org.apache.airavata.mft.resource.stubs.s3.storage.S3StorageGetRequest;
+import org.apache.airavata.mft.resource.stubs.scp.storage.SCPStorage;
+import org.apache.airavata.mft.resource.stubs.scp.storage.SCPStorageGetRequest;
+import org.apache.airavata.mft.resource.stubs.storage.common.StorageTypeResolveRequest;
+import org.apache.airavata.mft.resource.stubs.storage.common.StorageTypeResolveResponse;
+import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage;
+import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorageGetRequest;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class ControllerRequestBuilder {
+
+ private final Map<String, StorageWrapper> storageCache = new ConcurrentHashMap<>();
+ private final Map<String, SecretWrapper> secretCache = new ConcurrentHashMap<>();
+
+ @Autowired
+ private StorageServiceClient storageServiceClient;
+
+ @Autowired
+ private SecretServiceClient secretServiceClient;
+
+ public AgentTransferRequest createAgentTransferRequest(TransferApiRequest transferRequest) {
+ AgentTransferRequest.Builder agentTransferBuilder = AgentTransferRequest.newBuilder();
+ agentTransferBuilder.setSourcePath(transferRequest.getSourcePath());
+ agentTransferBuilder.setDestinationPath(transferRequest.getDestinationPath());
+ Pair<StorageWrapper, SecretWrapper> sourceCred = createCredentials(transferRequest.getSourceStorageId(),
+ transferRequest.getSourceToken());
+
+ agentTransferBuilder.setSourceStorage(sourceCred.getLeft());
+ agentTransferBuilder.setSourceSecret(sourceCred.getRight());
+
+ Pair<StorageWrapper, SecretWrapper> destCred = createCredentials(transferRequest.getDestinationStorageId(),
+ transferRequest.getDestinationToken());
+
+ agentTransferBuilder.setDestinationStorage(destCred.getLeft());
+ agentTransferBuilder.setDestinationSecret(destCred.getRight());
+
+ return agentTransferBuilder.build();
+ }
+ public Pair<StorageWrapper, SecretWrapper> createCredentials(String storageId, String secretId) {
+
+ if (secretCache.containsKey(secretId) && storageCache.containsKey(storageId)) {
+ return Pair.of(storageCache.get(storageId),secretCache.get(secretId));
+ }
+
+ StorageTypeResolveResponse resolve = storageServiceClient.common()
+ .resolveStorageType(StorageTypeResolveRequest.newBuilder().setStorageId(storageId).buildPartial());
+
+ StorageWrapper.Builder storageBuilder = StorageWrapper.newBuilder();
+ SecretWrapper.Builder secretBuilder = SecretWrapper.newBuilder();
+
+ switch (resolve.getStorageType()) {
+ case "S3":
+ S3Storage s3Storage = storageCache.containsKey(storageId)? storageCache.get(storageId).getS3():
+ storageServiceClient
+ .s3()
+ .getS3Storage(S3StorageGetRequest.newBuilder().setStorageId(storageId).build());
+ storageBuilder.setS3(s3Storage);
+
+ S3Secret s3Secret = storageCache.containsKey(secretId)? secretCache.get(secretId).getS3():
+ secretServiceClient
+ .s3()
+ .getS3Secret(S3SecretGetRequest.newBuilder().setSecretId(secretId).build());
+
+ secretBuilder.setS3(s3Secret);
+ break;
+
+ case "SCP":
+ SCPStorage scpStorage = storageCache.containsKey(storageId)? storageCache.get(storageId).getScp():
+ storageServiceClient
+ .scp()
+ .getSCPStorage(SCPStorageGetRequest.newBuilder().setStorageId(storageId).build());
+ storageBuilder.setScp(scpStorage);
+
+ SCPSecret scpSecret = storageCache.containsKey(secretId)? secretCache.get(secretId).getScp():
+ secretServiceClient
+ .scp()
+ .getSCPSecret(SCPSecretGetRequest.newBuilder().setSecretId(secretId).build());
+
+ secretBuilder.setScp(scpSecret);
+ break;
+ case "LOCAL":
+ LocalStorage localStorage = storageCache.containsKey(storageId)? storageCache.get(storageId).getLocal():
+ storageServiceClient
+ .local()
+ .getLocalStorage(LocalStorageGetRequest.newBuilder().setStorageId(storageId).build());
+ storageBuilder.setLocal(localStorage);
+ break;
+ case "FTP":
+ FTPStorage ftpStorage = storageCache.containsKey(storageId)? storageCache.get(storageId).getFtp():
+ storageServiceClient
+ .ftp()
+ .getFTPStorage(FTPStorageGetRequest.newBuilder().setStorageId(storageId).build());
+ storageBuilder.setFtp(ftpStorage);
+
+ FTPSecret ftpSecret = storageCache.containsKey(secretId)? secretCache.get(secretId).getFtp():
+ secretServiceClient
+ .ftp()
+ .getFTPSecret(FTPSecretGetRequest.newBuilder().setSecretId(secretId).build());
+
+ secretBuilder.setFtp(ftpSecret);
+ break;
+ case "BOX":
+ BoxStorage boxStorage = storageCache.containsKey(storageId)? storageCache.get(storageId).getBox():
+ storageServiceClient
+ .box()
+ .getBoxStorage(BoxStorageGetRequest.newBuilder().setStorageId(storageId).build());
+ storageBuilder.setBox(boxStorage);
+
+ BoxSecret boxSecret = storageCache.containsKey(secretId)? secretCache.get(secretId).getBox():
+ secretServiceClient
+ .box()
+ .getBoxSecret(BoxSecretGetRequest.newBuilder().setSecretId(secretId).build());
+
+ secretBuilder.setBox(boxSecret);
+ break;
+ case "DROPBOX":
+ DropboxStorage dropboxStorage = storageCache.containsKey(storageId)? storageCache.get(storageId).getDropbox():
+ storageServiceClient
+ .dropbox()
+ .getDropboxStorage(DropboxStorageGetRequest.newBuilder().setStorageId(storageId).build());
+ storageBuilder.setDropbox(dropboxStorage);
+
+ DropboxSecret dropboxSecret = storageCache.containsKey(secretId)? secretCache.get(secretId).getDropbox():
+ secretServiceClient
+ .dropbox()
+ .getDropboxSecret(DropboxSecretGetRequest.newBuilder().setSecretId(secretId).build());
+
+ secretBuilder.setDropbox(dropboxSecret);
+ break;
+
+ case "GCS":
+ GCSStorage gcsStorage = storageCache.containsKey(storageId)? storageCache.get(storageId).getGcs():
+ storageServiceClient
+ .gcs()
+ .getGCSStorage(GCSStorageGetRequest.newBuilder().setStorageId(storageId).build());
+ storageBuilder.setGcs(gcsStorage);
+
+ GCSSecret gcsSecret = storageCache.containsKey(secretId)? secretCache.get(secretId).getGcs():
+ secretServiceClient
+ .gcs()
+ .getGCSSecret(GCSSecretGetRequest.newBuilder().setSecretId(secretId).build());
+
+ secretBuilder.setGcs(gcsSecret);
+ break;
+
+ case "AZURE":
+ AzureStorage azureStorage = storageCache.containsKey(storageId)? storageCache.get(storageId).getAzure():
+ storageServiceClient
+ .azure()
+ .getAzureStorage(AzureStorageGetRequest.newBuilder().setStorageId(storageId).build());
+ storageBuilder.setAzure(azureStorage);
+
+ AzureSecret azureSecret = storageCache.containsKey(secretId)? secretCache.get(secretId).getAzure():
+ secretServiceClient
+ .azure()
+ .getAzureSecret(AzureSecretGetRequest.newBuilder().setSecretId(secretId).build());
+
+ secretBuilder.setAzure(azureSecret);
+ break;
+
+ case "SWIFT":
+ SwiftStorage swiftStorage = storageCache.containsKey(storageId)? storageCache.get(storageId).getSwift():
+ storageServiceClient
+ .swift()
+ .getSwiftStorage(SwiftStorageGetRequest.newBuilder().setStorageId(storageId).build());
+ storageBuilder.setSwift(swiftStorage);
+
+ SwiftSecret swiftSecret = storageCache.containsKey(secretId)? secretCache.get(secretId).getSwift():
+ secretServiceClient
+ .swift()
+ .getSwiftSecret(SwiftSecretGetRequest.newBuilder().setSecretId(secretId).build());
+
+ secretBuilder.setSwift(swiftSecret);
+ break;
+
+ case "ODATA":
+ ODataStorage odataStorage = storageCache.containsKey(storageId)? storageCache.get(storageId).getOdata():
+ storageServiceClient
+ .odata()
+ .getODataStorage(ODataStorageGetRequest.newBuilder().setStorageId(storageId).build());
+ storageBuilder.setOdata(odataStorage);
+
+ ODataSecret odataSecret = storageCache.containsKey(secretId)? secretCache.get(secretId).getOdata():
+ secretServiceClient
+ .odata()
+ .getODataSecret(ODataSecretGetRequest.newBuilder().setSecretId(secretId).build());
+
+ secretBuilder.setOdata(odataSecret);
+ break;
+
+ }
+
+ if (!storageCache.containsKey(storageId)) {
+ storageCache.put(storageId, storageBuilder.build());
+ }
+
+ if (!secretCache.containsKey(secretId)) {
+ secretCache.put(secretId, secretBuilder.build());
+ }
+
+ return Pair.of(storageBuilder.build(), secretBuilder.build());
+ }
+
+ public SecretWrapper createSecretWrapper(String secretId) {
+ SecretWrapper.Builder builder = SecretWrapper.newBuilder();
+
+ return builder.build();
+ }
+}
diff --git a/common/common-clients/src/main/java/org/apache/airavata/mft/admin/MFTConsulClient.java b/common/common-clients/src/main/java/org/apache/airavata/mft/admin/MFTConsulClient.java
index 8fdeaaf..97a00e8 100644
--- a/common/common-clients/src/main/java/org/apache/airavata/mft/admin/MFTConsulClient.java
+++ b/common/common-clients/src/main/java/org/apache/airavata/mft/admin/MFTConsulClient.java
@@ -27,10 +27,12 @@ import com.orbitz.consul.ConsulException;
import com.orbitz.consul.KeyValueClient;
import com.orbitz.consul.SessionClient;
import com.orbitz.consul.model.kv.Value;
+import com.orbitz.consul.option.PutOptions;
import org.apache.airavata.mft.admin.models.AgentInfo;
import org.apache.airavata.mft.admin.models.TransferState;
import org.apache.airavata.mft.admin.models.rpc.SyncRPCRequest;
import org.apache.airavata.mft.admin.models.rpc.SyncRPCResponse;
+import org.apache.airavata.mft.agent.stub.AgentTransferRequest;
import org.apache.airavata.mft.api.service.TransferApiRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -86,11 +88,11 @@ public class MFTConsulClient {
public String submitTransfer(TransferApiRequest transferRequest) throws MFTConsulClientException {
try {
- String asStr = JsonFormat.printer().print(transferRequest);
String transferId = UUID.randomUUID().toString();
- kvClient.putValue(CONTROLLER_TRANSFER_MESSAGE_PATH + transferId, asStr);
+ kvClient.putValue(CONTROLLER_TRANSFER_MESSAGE_PATH + transferId, transferRequest.toByteArray(),
+ 0L, PutOptions.BLANK);
return transferId;
- } catch (InvalidProtocolBufferException e) {
+ } catch (Exception e) {
throw new MFTConsulClientException("Error in serializing transfer request", e);
}
}
@@ -102,7 +104,8 @@ public class MFTConsulClient {
* @param transferRequest Target transfer request
* @throws MFTConsulClientException If {@link TransferApiRequest} can not be delivered to consul store
*/
- public void commandTransferToAgent(String agentId, String transferId, TransferApiRequest transferRequest) throws MFTConsulClientException {
+ public void commandTransferToAgent(String agentId, String transferId, AgentTransferRequest transferRequest)
+ throws MFTConsulClientException {
try {
submitTransferStateToProcess(transferId, "controller", new TransferState()
.setState("INITIALIZING")
@@ -110,11 +113,12 @@ public class MFTConsulClient {
.setUpdateTimeMils(System.currentTimeMillis())
.setPublisher("controller")
.setDescription("Initializing the transfer"));
- String asString = JsonFormat.printer().print(transferRequest);
- kvClient.putValue(AGENTS_TRANSFER_REQUEST_MESSAGE_PATH + agentId + "/" + transferId, asString);
+ byte[] transferReqBytes = transferRequest.toByteArray();
+ kvClient.putValue(AGENTS_TRANSFER_REQUEST_MESSAGE_PATH + agentId + "/" + transferId, transferReqBytes,
+ 0L, PutOptions.BLANK);
- } catch (InvalidProtocolBufferException e) {
- throw new MFTConsulClientException("Error in serializing transfer request", e);
+ } catch (Exception e) {
+ throw new MFTConsulClientException("Error in submitting transfer command to Agent through consul", e);
}
}
diff --git a/controller/pom.xml b/controller/pom.xml
index a922c84..59b1a9f 100644
--- a/controller/pom.xml
+++ b/controller/pom.xml
@@ -33,10 +33,20 @@
<artifactId>mft-controller</artifactId>
<dependencies>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>mft-resource-service-client</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>mft-secret-service-client</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>mft-common-clients</artifactId>
- <version>0.01-SNAPSHOT</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.orbitz.consul</groupId>
diff --git a/controller/src/main/java/org/apache/airavata/mft/controller/AppConfig.java b/controller/src/main/java/org/apache/airavata/mft/controller/AppConfig.java
index 203b50b..eed6dcc 100644
--- a/controller/src/main/java/org/apache/airavata/mft/controller/AppConfig.java
+++ b/controller/src/main/java/org/apache/airavata/mft/controller/AppConfig.java
@@ -17,7 +17,12 @@
package org.apache.airavata.mft.controller;
+import org.apache.airavata.mft.admin.ControllerRequestBuilder;
import org.apache.airavata.mft.admin.MFTConsulClient;
+import org.apache.airavata.mft.resource.client.StorageServiceClient;
+import org.apache.airavata.mft.resource.client.StorageServiceClientBuilder;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -30,6 +35,32 @@ public class AppConfig {
@org.springframework.beans.factory.annotation.Value("${consul.port}")
Integer consulPort;
+ @org.springframework.beans.factory.annotation.Value("${resource.service.host}")
+ private String resourceServiceHost;
+
+ @org.springframework.beans.factory.annotation.Value("${resource.service.port}")
+ private int resourceServicePort;
+
+ @org.springframework.beans.factory.annotation.Value("${secret.service.host}")
+ private String secretServiceHost;
+
+ @org.springframework.beans.factory.annotation.Value("${secret.service.port}")
+ private int secretServicePort;
+ @Bean
+ public StorageServiceClient storageServiceClient() {
+ return StorageServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort);
+ }
+
+ @Bean
+ public SecretServiceClient secretServiceClient() {
+ return SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort);
+ }
+
+ @Bean
+ public ControllerRequestBuilder controllerRequestBuilder() {
+ return new ControllerRequestBuilder();
+ }
+
@Bean
public MFTConsulClient mftConsulClient() {
return new MFTConsulClient(consulHost, consulPort);
diff --git a/controller/src/main/java/org/apache/airavata/mft/controller/MFTController.java b/controller/src/main/java/org/apache/airavata/mft/controller/MFTController.java
index e674993..97bdd72 100644
--- a/controller/src/main/java/org/apache/airavata/mft/controller/MFTController.java
+++ b/controller/src/main/java/org/apache/airavata/mft/controller/MFTController.java
@@ -23,10 +23,12 @@ import com.google.protobuf.util.JsonFormat;
import com.orbitz.consul.cache.ConsulCache;
import com.orbitz.consul.cache.KVCache;
import com.orbitz.consul.model.kv.Value;
+import org.apache.airavata.mft.admin.ControllerRequestBuilder;
import org.apache.airavata.mft.admin.MFTConsulClient;
import org.apache.airavata.mft.admin.MFTConsulClientException;
import org.apache.airavata.mft.admin.models.AgentInfo;
import org.apache.airavata.mft.admin.models.TransferState;
+import org.apache.airavata.mft.agent.stub.AgentTransferRequest;
import org.apache.airavata.mft.api.service.TransferApiRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,6 +66,9 @@ public class MFTController implements CommandLineRunner {
private KVCache liveAgentCache;
private ScheduledExecutorService pendingMonitor;
+ @Autowired
+ private ControllerRequestBuilder controllerRequestBuilder;
+
@Autowired
private MFTConsulClient mftConsulClient;
@@ -100,15 +105,14 @@ public class MFTController implements CommandLineRunner {
// Due to bug in consul https://github.com/hashicorp/consul/issues/571
ConsulCache.Listener<String, Value> messageCacheListener = newValues -> newValues.forEach((key, value) -> {
String transferId = key.substring(key.lastIndexOf("/") + 1);
- Optional<String> decodedValue = value.getValueAsString();
+ Optional<byte[]> decodedValue = value.getValueAsBytes();
decodedValue.ifPresent(v -> {
logger.info("Received transfer request : {} with id {}", v, transferId);
-
TransferApiRequest transferRequest;
try {
TransferApiRequest.Builder builder = TransferApiRequest.newBuilder();
- JsonFormat.parser().merge(value.getValueAsString().get(), builder);
+ builder.mergeFrom(v);
transferRequest = builder.build();
} catch (IOException e) {
logger.error("Failed to parse the transfer request {}", v, e);
@@ -188,9 +192,9 @@ public class MFTController implements CommandLineRunner {
if (!scheduledSession.equals(sessionId)) {
logger.info("Old transfer session found. Re scheduling to agent {}", agentId);
- TransferApiRequest.Builder builder = TransferApiRequest.newBuilder();
- JsonFormat.parser().merge(v.getValueAsString().get(), builder);
- mftConsulClient.commandTransferToAgent(agentId, scheduledTransfer, builder.build());
+ AgentTransferRequest transferRequest = AgentTransferRequest
+ .newBuilder().mergeFrom(v.getValueAsBytes().get()).build();
+ mftConsulClient.commandTransferToAgent(agentId, scheduledTransfer, transferRequest);
// Delete the key as it is already processed
mftConsulClient.getKvClient().deleteKey(v.getKey());
@@ -308,7 +312,10 @@ public class MFTController implements CommandLineRunner {
if (agent.isPresent()) {
logger.info("Found agent {} to initiate the transfer {}", agent, transferId);
- mftConsulClient.commandTransferToAgent(agent.get(), transferId, transferRequest);
+
+ AgentTransferRequest agentTransferRequest = controllerRequestBuilder.createAgentTransferRequest(transferRequest);
+
+ mftConsulClient.commandTransferToAgent(agent.get(), transferId, agentTransferRequest);
markAsProcessed(transferId, transferRequest);
mftConsulClient.getKvClient().deleteKey(value.getKey());
logger.info("Marked transfer {} as processed", transferId);
diff --git a/controller/src/main/resources/application.properties b/controller/src/main/resources/application.properties
index 8ad1c58..8bb6485 100644
--- a/controller/src/main/resources/application.properties
+++ b/controller/src/main/resources/application.properties
@@ -15,4 +15,8 @@
# limitations under the License.
#
consul.host=localhost
-consul.port=8500
\ No newline at end of file
+consul.port=8500
+resource.service.host=localhost
+resource.service.port=7002
+secret.service.host=localhost
+secret.service.port=7003
\ No newline at end of file
diff --git a/core/pom.xml b/core/pom.xml
index b81b61d..dd9757c 100755
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -44,5 +44,10 @@
<artifactId>mft-common-proto</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>mft-agent-stub</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/core/src/main/java/org/apache/airavata/mft/core/FileResourceMetadata.java b/core/src/main/java/org/apache/airavata/mft/core/FileResourceMetadata.java
index 7033d02..0070da1 100644
--- a/core/src/main/java/org/apache/airavata/mft/core/FileResourceMetadata.java
+++ b/core/src/main/java/org/apache/airavata/mft/core/FileResourceMetadata.java
@@ -18,7 +18,6 @@
package org.apache.airavata.mft.core;
public class FileResourceMetadata {
-
private String friendlyName;
private long resourceSize;
private long createdTime;
diff --git a/core/src/main/java/org/apache/airavata/mft/core/api/ConnectorConfig.java b/core/src/main/java/org/apache/airavata/mft/core/api/ConnectorConfig.java
index effd5cc..faf8e61 100644
--- a/core/src/main/java/org/apache/airavata/mft/core/api/ConnectorConfig.java
+++ b/core/src/main/java/org/apache/airavata/mft/core/api/ConnectorConfig.java
@@ -1,59 +1,38 @@
package org.apache.airavata.mft.core.api;
-import org.apache.airavata.mft.common.AuthToken;
-import org.apache.airavata.mft.core.FileResourceMetadata;
+import org.apache.airavata.mft.agent.stub.ResourceMetadata;
+import org.apache.airavata.mft.agent.stub.SecretWrapper;
+import org.apache.airavata.mft.agent.stub.StorageWrapper;
public class ConnectorConfig {
- private String resourceServiceHost;
- private int resourceServicePort;
- private String secretServiceHost;
- private int secretServicePort;
- private String storageId;
- private String credentialToken;
- private AuthToken authToken;
private String transferId;
- private FileResourceMetadata metadata;
+ private StorageWrapper storage;
+ private SecretWrapper secret;
private String resourcePath;
- private String storageType;
+ private ResourceMetadata metadata;
- public String getResourceServiceHost() {
- return resourceServiceHost;
- }
-
- public void setResourceServiceHost(String resourceServiceHost) {
- this.resourceServiceHost = resourceServiceHost;
- }
-
- public int getResourceServicePort() {
- return resourceServicePort;
- }
-
- public void setResourceServicePort(int resourceServicePort) {
- this.resourceServicePort = resourceServicePort;
- }
-
- public String getSecretServiceHost() {
- return secretServiceHost;
+ public String getTransferId() {
+ return transferId;
}
- public void setSecretServiceHost(String secretServiceHost) {
- this.secretServiceHost = secretServiceHost;
+ public void setTransferId(String transferId) {
+ this.transferId = transferId;
}
- public int getSecretServicePort() {
- return secretServicePort;
+ public StorageWrapper getStorage() {
+ return storage;
}
- public void setSecretServicePort(int secretServicePort) {
- this.secretServicePort = secretServicePort;
+ public void setStorage(StorageWrapper storage) {
+ this.storage = storage;
}
- public String getStorageId() {
- return storageId;
+ public SecretWrapper getSecret() {
+ return secret;
}
- public void setStorageId(String storageId) {
- this.storageId = storageId;
+ public void setSecret(SecretWrapper secret) {
+ this.secret = secret;
}
public String getResourcePath() {
@@ -64,58 +43,21 @@ public class ConnectorConfig {
this.resourcePath = resourcePath;
}
- public String getCredentialToken() {
- return credentialToken;
- }
-
- public void setCredentialToken(String credentialToken) {
- this.credentialToken = credentialToken;
- }
-
- public AuthToken getAuthToken() {
- return authToken;
- }
-
- public void setAuthToken(AuthToken authToken) {
- this.authToken = authToken;
- }
-
- public String getTransferId() {
- return transferId;
- }
-
- public void setTransferId(String transferId) {
- this.transferId = transferId;
- }
-
- public FileResourceMetadata getMetadata() {
+ public ResourceMetadata getMetadata() {
return metadata;
}
- public void setMetadata(FileResourceMetadata metadata) {
+ public void setMetadata(ResourceMetadata metadata) {
this.metadata = metadata;
}
- public String getStorageType() {
- return storageType;
- }
-
- public void setStorageType(String storageType) {
- this.storageType = storageType;
- }
public static final class ConnectorConfigBuilder {
- private String resourceServiceHost;
- private int resourceServicePort;
- private String secretServiceHost;
- private int secretServicePort;
- private String stroageId;
- private String credentialToken;
- private AuthToken authToken;
private String transferId;
+ private StorageWrapper storage;
+ private SecretWrapper secret;
private String resourcePath;
- private FileResourceMetadata metadata;
- private String storageType;
+ private ResourceMetadata metadata;
private ConnectorConfigBuilder() {
}
@@ -124,28 +66,18 @@ public class ConnectorConfig {
return new ConnectorConfigBuilder();
}
- public ConnectorConfigBuilder withResourceServiceHost(String resourceServiceHost) {
- this.resourceServiceHost = resourceServiceHost;
- return this;
- }
-
- public ConnectorConfigBuilder withResourceServicePort(int resourceServicePort) {
- this.resourceServicePort = resourceServicePort;
- return this;
- }
-
- public ConnectorConfigBuilder withSecretServiceHost(String secretServiceHost) {
- this.secretServiceHost = secretServiceHost;
+ public ConnectorConfigBuilder withTransferId(String transferId) {
+ this.transferId = transferId;
return this;
}
- public ConnectorConfigBuilder withSecretServicePort(int secretServicePort) {
- this.secretServicePort = secretServicePort;
+ public ConnectorConfigBuilder withStorage(StorageWrapper storage) {
+ this.storage = storage;
return this;
}
- public ConnectorConfigBuilder withStorageId(String storageId) {
- this.stroageId = storageId;
+ public ConnectorConfigBuilder withSecret(SecretWrapper secret) {
+ this.secret = secret;
return this;
}
@@ -154,44 +86,18 @@ public class ConnectorConfig {
return this;
}
- public ConnectorConfigBuilder withStorageType(String storageType) {
- this.storageType = storageType;
- return this;
- }
-
- public ConnectorConfigBuilder withCredentialToken(String credentialToken) {
- this.credentialToken = credentialToken;
- return this;
- }
-
- public ConnectorConfigBuilder withAuthToken(AuthToken authToken) {
- this.authToken = authToken;
- return this;
- }
-
- public ConnectorConfigBuilder withTransferId(String transferId) {
- this.transferId = transferId;
- return this;
- }
-
- public ConnectorConfigBuilder withMetadata(FileResourceMetadata metadata) {
+ public ConnectorConfigBuilder withMetadata(ResourceMetadata metadata) {
this.metadata = metadata;
return this;
}
public ConnectorConfig build() {
ConnectorConfig connectorConfig = new ConnectorConfig();
- connectorConfig.setResourceServiceHost(resourceServiceHost);
- connectorConfig.setResourceServicePort(resourceServicePort);
- connectorConfig.setSecretServiceHost(secretServiceHost);
- connectorConfig.setSecretServicePort(secretServicePort);
- connectorConfig.setStorageId(stroageId);
- connectorConfig.setStorageType(storageType);
- connectorConfig.setCredentialToken(credentialToken);
- connectorConfig.setAuthToken(authToken);
connectorConfig.setTransferId(transferId);
- connectorConfig.setMetadata(metadata);
+ connectorConfig.setStorage(storage);
+ connectorConfig.setSecret(secret);
connectorConfig.setResourcePath(resourcePath);
+ connectorConfig.setMetadata(metadata);
return connectorConfig;
}
}
diff --git a/core/src/main/java/org/apache/airavata/mft/core/api/MetadataCollector.java b/core/src/main/java/org/apache/airavata/mft/core/api/MetadataCollector.java
index 3f2b592..92811fa 100644
--- a/core/src/main/java/org/apache/airavata/mft/core/api/MetadataCollector.java
+++ b/core/src/main/java/org/apache/airavata/mft/core/api/MetadataCollector.java
@@ -17,7 +17,9 @@
package org.apache.airavata.mft.core.api;
-import org.apache.airavata.mft.common.AuthToken;
+import org.apache.airavata.mft.agent.stub.ResourceMetadata;
+import org.apache.airavata.mft.agent.stub.SecretWrapper;
+import org.apache.airavata.mft.agent.stub.StorageWrapper;
import org.apache.airavata.mft.core.DirectoryResourceMetadata;
import org.apache.airavata.mft.core.FileResourceMetadata;
@@ -26,50 +28,27 @@ public interface MetadataCollector {
/**
* Initializes the {@link MetadataCollector}
*
- * @param resourceServiceHost hostname of the resource service
- * @param resourceServicePort port of the resource service
- * @param secretServiceHost hostname of the secret service
- * @param secretServicePort port of the secret service
+ * @param storage
+ * @param secret
*/
- public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort);
+ public void init(StorageWrapper storage, SecretWrapper secret);
/**
* Fetches a metadata of given File Resource
*
- *
- * @param authZToken
* @param resourcePath path of the resource
- * @param storageId Storage id
- * @param credentialToken credential token for the resource
* @return an object of {@link FileResourceMetadata}
* @throws Exception if the resource id is not a File Resource type or the resource can't be fetched from the resource service
*/
- public FileResourceMetadata getFileResourceMetadata(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception;
-
- /**
- * Fetches a metadata of given Directory Resource
- *
- *
- * @param authZToken
- * @param resourcePath path of the resource
- * @param storageId Storage id
- * @param credentialToken credential token for the resource
- * @return an object of {@link DirectoryResourceMetadata}
- * @throws Exception if the resource id is not a Directory Resource type or the resource can't be fetched from the resource service
- */
- public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception;
+ public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception;
/**
* Check whether the resource is available in the actual storage
*
- *
- * @param authZToken
- * @param resourcePath path of the resource
- * @param storageId Storage id
- * @param credentialToken credential token for the resource
+ * @param resourcePath path of the resource
* @return true of the resource is available false otherwise
* @throws Exception if the resource details can not be fetched from the resource service
*/
- public Boolean isAvailable(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception;
+ public Boolean isAvailable(String resourcePath) throws Exception;
}
diff --git a/python-cli/README.md b/python-cli/README.md
new file mode 100644
index 0000000..0d8ae5c
--- /dev/null
+++ b/python-cli/README.md
@@ -0,0 +1,6 @@
+python3.10 -m venv venv
+source venv/bin/activate
+pip install poetry
+cd mft-cli
+poetry shell
+poetry install
\ No newline at end of file
diff --git a/python-cli/mft_cli/mft_cli/main.py b/python-cli/mft_cli/mft_cli/main.py
new file mode 100644
index 0000000..3c8e134
--- /dev/null
+++ b/python-cli/mft_cli/mft_cli/main.py
@@ -0,0 +1,24 @@
+import typer
+
+app = typer.Typer()
+
+@app.callback()
+def callback():
+ """
+ Awesome Portal Gun
+ """
+
+@app.command()
+def shoot():
+ """
+ Shoot the portal gun
+ """
+ typer.echo("Shooting portal gun")
+
+
+@app.command()
+def load():
+ """
+ Load the portal gun
+ """
+ typer.echo("Loading portal gun")
\ No newline at end of file
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java
deleted file mode 100644
index 380e25c..0000000
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java
+++ /dev/null
@@ -1,626 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
- package org.apache.airavata.mft.resource.server.backend.file;
-
-import org.apache.airavata.mft.resource.server.backend.ResourceBackend;
-import org.apache.airavata.mft.resource.stubs.azure.storage.*;
-import org.apache.airavata.mft.resource.stubs.box.storage.*;
-import org.apache.airavata.mft.resource.stubs.common.*;
-import org.apache.airavata.mft.resource.stubs.dropbox.storage.*;
-import org.apache.airavata.mft.resource.stubs.ftp.storage.*;
-import org.apache.airavata.mft.resource.stubs.gcs.storage.*;
-import org.apache.airavata.mft.resource.stubs.local.storage.*;
-import org.apache.airavata.mft.resource.stubs.odata.storage.*;
-import org.apache.airavata.mft.resource.stubs.s3.storage.*;
-import org.apache.airavata.mft.resource.stubs.scp.storage.*;
-import org.apache.airavata.mft.resource.stubs.storage.common.StorageTypeResolveRequest;
-import org.apache.airavata.mft.resource.stubs.storage.common.StorageTypeResolveResponse;
-import org.apache.airavata.mft.resource.stubs.swift.storage.*;
-import org.apache.airavata.mft.storage.stubs.storagesecret.*;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-@SuppressWarnings("unchecked")
-public class FileBasedResourceBackend implements ResourceBackend {
-
- private static final Logger logger = LoggerFactory.getLogger(FileBasedResourceBackend.class);
-
- @org.springframework.beans.factory.annotation.Value("${file.backend.resource.file}")
- private String resourceFile;
-
- @org.springframework.beans.factory.annotation.Value("${file.backend.storage.file}")
- private String storageFile;
-
- @Override
- public void init() {
- logger.info("Initializing file based resource backend");
- }
-
- @Override
- public void destroy() {
- logger.info("Destroying file based resource backend");
- }
-
- @Override
- public Optional<GenericResource> getGenericResource(GenericResourceGetRequest request) throws Exception {
- InputStream inputStream = FileBasedResourceBackend.class.getClassLoader().getResourceAsStream("resources.json");
-
- JSONParser jsonParser = new JSONParser();
-
- try (InputStreamReader reader = new InputStreamReader(inputStream)) {
-
- Object obj = jsonParser.parse(reader);
-
- JSONArray resourceList = (JSONArray) obj;
-
- return resourceList.stream()
- .filter(resource -> request.getResourceId().equals(((JSONObject) resource).get("resourceId").toString()))
- .findFirst().map(resource -> {
- JSONObject r = (JSONObject) resource;
-
- GenericResource.Builder resourceBuilder = GenericResource.newBuilder();
- String resourcePath = r.get("resourcePath").toString();
-
- resourceBuilder.setResourceId(request.getResourceId());
-
- switch (r.get("resourceMode").toString()) {
- case "FILE":
- FileResource fileResource = FileResource.newBuilder().setResourcePath(resourcePath).build();
- resourceBuilder.setFile(fileResource);
- break;
- case "DIRECTORY":
- DirectoryResource directoryResource = DirectoryResource.newBuilder().setResourcePath(resourcePath).build();
- resourceBuilder.setDirectory(directoryResource);
- break;
- }
-
- String storageId = r.get("storageId").toString();
- String type = r.get("type").toString();
-
- try {
- switch (type) {
- case "SCP":
- resourceBuilder.setScpStorage(getSCPStorage(SCPStorageGetRequest.newBuilder()
- .setStorageId(storageId).build()).orElseThrow(() -> new Exception("Storage not found")));
- break;
- case "S3":
- resourceBuilder.setS3Storage(getS3Storage(S3StorageGetRequest.newBuilder()
- .setStorageId(storageId).build()).orElseThrow(() -> new Exception("Storage not found")));
- break;
- case "LOCAL":
- resourceBuilder.setLocalStorage(getLocalStorage(LocalStorageGetRequest.newBuilder()
- .setStorageId(storageId).build()).orElseThrow(() -> new Exception("Storage not found")));
- break;
- case "DROPBOX":
- resourceBuilder.setDropboxStorage(getDropboxStorage(DropboxStorageGetRequest.newBuilder()
- .setStorageId(storageId).build()).orElseThrow(() -> new Exception("Storage not found")));
- break;
- case "FTP":
- resourceBuilder.setFtpStorage(getFTPStorage(FTPStorageGetRequest.newBuilder()
- .setStorageId(storageId).build()).orElseThrow(() -> new Exception("Storage not found")));
- break;
- case "BOX":
- resourceBuilder.setBoxStorage(getBoxStorage(BoxStorageGetRequest.newBuilder()
- .setStorageId(storageId).build()).orElseThrow(() -> new Exception("Storage not found")));
- break;
- case "GCS":
- resourceBuilder.setGcsStorage(getGCSStorage(GCSStorageGetRequest.newBuilder()
- .setStorageId(storageId).build()).orElseThrow(() -> new Exception("Storage not found")));
- break;
-
- }
- } catch (Exception e) {
- logger.error("Failed to fetch storage with id {} and type {}", storageId, type);
- return null;
- }
- return resourceBuilder.build();
-
- });
- }
- }
-
- @Override
- public GenericResource createGenericResource(GenericResourceCreateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateGenericResource(GenericResourceUpdateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteGenericResource(GenericResourceDeleteRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<StorageSecret> getStorageSecret(StorageSecretGetRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public StorageSecret createStorageSecret(StorageSecretCreateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateStorageSecret(StorageSecretUpdateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteStorageSecret(StorageSecretDeleteRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<StorageSecret> searchStorageSecret(StorageSecretSearchRequest request) throws Exception {
- return Optional.empty();
- }
-
- @Override
- public SCPStorageListResponse listSCPStorage(SCPStorageListRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<SCPStorage> getSCPStorage(SCPStorageGetRequest request) throws Exception {
- InputStream inputStream = FileBasedResourceBackend.class.getClassLoader().getResourceAsStream(storageFile);
-
- JSONParser jsonParser = new JSONParser();
-
- try (InputStreamReader reader = new InputStreamReader(inputStream)) {
-
- Object obj = jsonParser.parse(reader);
-
- JSONArray resourceList = (JSONArray) obj;
-
- List<SCPStorage> scpStorages = (List<SCPStorage>) resourceList.stream()
- .filter(resource -> "SCP".equals(((JSONObject) resource).get("type").toString()))
- .map(st -> {
- JSONObject s = (JSONObject) st;
-
- SCPStorage storage = SCPStorage.newBuilder()
- .setStorageId(s.get("storageId").toString())
- .setHost(s.get("host").toString())
- .setPort(Integer.parseInt(s.get("port").toString())).build();
-
- return storage;
-
- }).collect(Collectors.toList());
-
- return scpStorages.stream().filter(s -> request.getStorageId().equals(s.getStorageId())).findFirst();
- }
- }
-
- @Override
- public SCPStorage createSCPStorage(SCPStorageCreateRequest request) {
- throw new UnsupportedOperationException("Operation is not supported in backend");
-
- }
-
- @Override
- public boolean updateSCPStorage(SCPStorageUpdateRequest request) {
- throw new UnsupportedOperationException("Operation is not supported in backend");
-
- }
-
- @Override
- public boolean deleteSCPStorage(SCPStorageDeleteRequest request) {
- throw new UnsupportedOperationException("Operation is not supported in backend");
-
- }
-
- @Override
- public LocalStorageListResponse listLocalStorage(LocalStorageListRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<LocalStorage> getLocalStorage(LocalStorageGetRequest request) throws Exception {
- JSONParser jsonParser = new JSONParser();
- InputStream inputStream = FileBasedResourceBackend.class.getClassLoader().getResourceAsStream(storageFile);
-
- try (InputStreamReader reader = new InputStreamReader(inputStream)) {
- Object obj = jsonParser.parse(reader);
-
- JSONArray storageList = (JSONArray) obj;
-
- List<LocalStorage> localStorages = (List<LocalStorage>) storageList.stream()
- .filter(storage -> "LOCAL".equals(((JSONObject) storage).get("type").toString()))
- .map( storage -> {
- JSONObject s = (JSONObject) storage;
-
- LocalStorage st = LocalStorage.newBuilder()
- .setStorageId(s.get("storageId").toString())
- .setAgentId(s.get("agentId").toString())
- .build();
-
- return st;
- }).collect(Collectors.toList());
- return localStorages.stream().filter(s -> request.getStorageId().equals(s.getStorageId())).findFirst();
- }
- }
-
- @Override
- public LocalStorage createLocalStorage(LocalStorageCreateRequest request) throws Exception {
- return null;
- }
-
- @Override
- public boolean updateLocalStorage(LocalStorageUpdateRequest request) throws Exception {
- return false;
- }
-
- @Override
- public boolean deleteLocalStorage(LocalStorageDeleteRequest request) throws Exception {
- return false;
- }
-
- @Override
- public S3StorageListResponse listS3Storage(S3StorageListRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<S3Storage> getS3Storage(S3StorageGetRequest request) throws Exception {
- JSONParser jsonParser = new JSONParser();
- InputStream inputStream = FileBasedResourceBackend.class.getClassLoader().getResourceAsStream(storageFile);
-
- try (InputStreamReader reader = new InputStreamReader(inputStream)) {
- Object obj = jsonParser.parse(reader);
-
- JSONArray storagesList = (JSONArray) obj;
-
- List<S3Storage> s3Storages = (List<S3Storage>) storagesList.stream()
- .filter(storage -> "S3".equals(((JSONObject) storage).get("type").toString()))
- .map(storage -> {
- JSONObject s = (JSONObject) storage;
-
- S3Storage st = S3Storage.newBuilder()
- .setStorageId(s.get("storageId").toString())
- .setRegion(s.get("region").toString())
- .setRegion(s.get("bucketName").toString())
- .build();
-
- return st;
- }).collect(Collectors.toList());
- return s3Storages.stream().filter(s -> request.getStorageId().equals(s.getStorageId())).findFirst();
- }
- }
-
- @Override
- public S3Storage createS3Storage(S3StorageCreateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateS3Storage(S3StorageUpdateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteS3Storage(S3StorageDeleteRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public BoxStorageListResponse listBoxStorage(BoxStorageListRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<BoxStorage> getBoxStorage(BoxStorageGetRequest request) throws Exception {
- JSONParser jsonParser = new JSONParser();
- InputStream inputStream = FileBasedResourceBackend.class.getClassLoader().getResourceAsStream(storageFile);
-
- try (InputStreamReader reader = new InputStreamReader(inputStream)) {
- Object obj = jsonParser.parse(reader);
-
- JSONArray storageList = (JSONArray) obj;
-
- System.out.println("All resources ");
- List<BoxStorage> boxStorages = (List<BoxStorage>) storageList.stream()
- .filter(storage -> "BOX".equals(((JSONObject) storage).get("type").toString()))
- .map(storage -> {
- JSONObject s = (JSONObject) storage;
-
- BoxStorage st = BoxStorage.newBuilder()
- .setStorageId(s.get("storageId").toString())
- .build();
-
- return st;
- }).collect(Collectors.toList());
- return boxStorages.stream().filter(s -> request.getStorageId().equals(s.getStorageId())).findFirst();
- }
- }
-
- @Override
- public BoxStorage createBoxStorage(BoxStorageCreateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateBoxStorage(BoxStorageUpdateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteBoxStorage(BoxStorageDeleteRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public AzureStorageListResponse listAzureStorage(AzureStorageListRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<AzureStorage> getAzureStorage(AzureStorageGetRequest request) throws Exception {
- JSONParser jsonParser = new JSONParser();
- InputStream inputStream = FileBasedResourceBackend.class.getClassLoader().getResourceAsStream(storageFile);
-
- try (InputStreamReader reader = new InputStreamReader(inputStream)) {
- Object obj = jsonParser.parse(reader);
-
- JSONArray storageList = (JSONArray) obj;
-
- List<AzureStorage> azureStorages = (List<AzureStorage>) storageList.stream()
- .filter(storage -> "AZURE".equals(((JSONObject) storage).get("type").toString()))
- .map(storage -> {
- JSONObject s = (JSONObject) storage;
-
- AzureStorage st = AzureStorage.newBuilder()
- .setStorageId(s.get("storageId").toString())
- .setContainer(s.get("container").toString())
- .build();
-
- return st;
- }).collect(Collectors.toList());
- return azureStorages.stream().filter(s -> request.getStorageId().equals(s.getStorageId())).findFirst();
- }
- }
-
- @Override
- public AzureStorage createAzureStorage(AzureStorageCreateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateAzureStorage(AzureStorageUpdateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteAzureStorage(AzureStorageDeleteRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public GCSStorageListResponse listGCSStorage(GCSStorageListRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<GCSStorage> getGCSStorage(GCSStorageGetRequest request) throws Exception {
- JSONParser jsonParser = new JSONParser();
- InputStream inputStream = FileBasedResourceBackend.class.getClassLoader().getResourceAsStream(storageFile);
-
- try (InputStreamReader reader = new InputStreamReader(inputStream)) {
- Object obj = jsonParser.parse(reader);
-
- JSONArray storageList = (JSONArray) obj;
-
- List<GCSStorage> gcsStorages = (List<GCSStorage>) storageList.stream()
- .filter(storage -> "GCS".equals(((JSONObject) storage).get("type").toString()))
- .map(storage -> {
- JSONObject s = (JSONObject) storage;
-
- GCSStorage st = GCSStorage.newBuilder()
- .setStorageId(s.get("storageId").toString())
- .setBucketName(s.get("bucketName").toString())
- .build();
-
- return st;
- }).collect(Collectors.toList());
- return gcsStorages.stream().filter(s -> request.getStorageId().equals(s.getStorageId())).findFirst();
- }
- }
-
- @Override
- public GCSStorage createGCSStorage(GCSStorageCreateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateGCSStorage(GCSStorageUpdateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteGCSStorage(GCSStorageDeleteRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public DropboxStorageListResponse listDropboxStorage(DropboxStorageListRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<DropboxStorage> getDropboxStorage(DropboxStorageGetRequest request) throws Exception {
- JSONParser jsonParser = new JSONParser();
- InputStream inputStream = FileBasedResourceBackend.class.getClassLoader().getResourceAsStream(resourceFile);
-
- try (InputStreamReader reader = new InputStreamReader(inputStream)) {
- Object obj = jsonParser.parse(reader);
-
- JSONArray storageList = (JSONArray) obj;
-
- List<DropboxStorage> dropboxStorages = (List<DropboxStorage>) storageList.stream()
- .filter(resource -> "DROPBOX".equals(((JSONObject) resource).get("type").toString()))
- .map(resource -> {
- JSONObject r = (JSONObject) resource;
- String resourcePath = r.get("resourcePath").toString();
- resourcePath = resourcePath.startsWith("/") ? resourcePath : "/" + resourcePath;
-
- DropboxStorage storage = DropboxStorage.newBuilder()
- .setStorageId(((JSONObject)r.get("dropboxStorage")).get("storageId").toString())
- .build();
-
- return storage;
- }).collect(Collectors.toList());
- return dropboxStorages.stream().filter(s -> request.getStorageId().equals(s.getStorageId())).findFirst();
- }
- }
-
- @Override
- public DropboxStorage createDropboxStorage(DropboxStorageCreateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateDropboxStorage(DropboxStorageUpdateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteDropboxStorage(DropboxStorageDeleteRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public FTPStorageListResponse listFTPStorage(FTPStorageListRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<FTPStorage> getFTPStorage(FTPStorageGetRequest request) throws Exception {
- InputStream inputStream = FileBasedResourceBackend.class.getClassLoader().getResourceAsStream(storageFile);
-
- JSONParser jsonParser = new JSONParser();
-
- if (inputStream == null) {
- throw new IOException("resources file not found");
- }
-
- try (InputStreamReader reader = new InputStreamReader(inputStream)) {
-
- Object obj = jsonParser.parse(reader);
-
- JSONArray storageList = (JSONArray) obj;
-
- List<FTPStorage> ftpStorages = (List<FTPStorage>) storageList.stream()
- .filter(storage -> "FTP".equals(((JSONObject) storage).get("type").toString()))
- .map(storage -> {
- JSONObject s = (JSONObject) storage;
-
- FTPStorage st = FTPStorage.newBuilder()
- .setStorageId(s.get("storageId").toString())
- .setHost(s.get("host").toString())
- .setPort(Integer.parseInt(s.get("port").toString())).build();
-
- return st;
-
- }).collect(Collectors.toList());
-
- return ftpStorages.stream().filter(s -> request.getStorageId().equals(s.getStorageId())).findFirst();
- }
- }
-
- @Override
- public FTPStorage createFTPStorage(FTPStorageCreateRequest request) {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateFTPStorage(FTPStorageUpdateRequest request) {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteFTPStorage(FTPStorageDeleteRequest request) {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public SwiftStorageListResponse listSwiftStorage(SwiftStorageListRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<SwiftStorage> getSwiftStorage(SwiftStorageGetRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public SwiftStorage createSwiftStorage(SwiftStorageCreateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateSwiftStorage(SwiftStorageUpdateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteSwiftStorage(SwiftStorageDeleteRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public ODataStorageListResponse listODataStorage(ODataStorageListRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<ODataStorage> getODataStorage(ODataStorageGetRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public ODataStorage createODataStorage(ODataStorageCreateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateODataStorage(ODataStorageUpdateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteODataStorage(ODataStorageDeleteRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public StorageTypeResolveResponse resolveStorageType(StorageTypeResolveRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-}
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/file/FileBasedSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/file/FileBasedSecretBackend.java
deleted file mode 100644
index 501d95e..0000000
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/file/FileBasedSecretBackend.java
+++ /dev/null
@@ -1,386 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.airavata.mft.secret.server.backend.file;
-
-import org.apache.airavata.mft.credential.stubs.azure.*;
-import org.apache.airavata.mft.credential.stubs.box.*;
-import org.apache.airavata.mft.credential.stubs.dropbox.*;
-import org.apache.airavata.mft.credential.stubs.ftp.*;
-import org.apache.airavata.mft.credential.stubs.gcs.*;
-import org.apache.airavata.mft.credential.stubs.odata.*;
-import org.apache.airavata.mft.credential.stubs.s3.*;
-import org.apache.airavata.mft.credential.stubs.scp.*;
-import org.apache.airavata.mft.credential.stubs.swift.*;
-import org.apache.airavata.mft.secret.server.backend.SecretBackend;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-@SuppressWarnings("unchecked")
-public class FileBasedSecretBackend implements SecretBackend {
-
- private static final Logger logger = LoggerFactory.getLogger(FileBasedSecretBackend.class);
-
- @org.springframework.beans.factory.annotation.Value("${file.backend.secret.file}")
- private String secretFile;
-
- @Override
- public void init() {
- logger.info("Initializing file based secret backend");
- }
-
- @Override
- public void destroy() {
- logger.info("Destroying file based secret backend");
- }
-
- @Override
- public Optional<SCPSecret> getSCPSecret(SCPSecretGetRequest request) throws Exception {
- JSONParser jsonParser = new JSONParser();
- InputStream inputStream = FileBasedSecretBackend.class.getClassLoader().getResourceAsStream(secretFile);
-
- try (InputStreamReader reader = new InputStreamReader(inputStream)) {
- Object obj = jsonParser.parse(reader);
-
- JSONArray resourceList = (JSONArray) obj;
-
- List<SCPSecret> scpSecrets = (List<SCPSecret>) resourceList.stream()
- .filter(resource -> "SCP".equals(((JSONObject) resource).get("type").toString()))
- .map(resource -> {
- JSONObject r = (JSONObject) resource;
-
- SCPSecret scpSecret = SCPSecret.newBuilder()
- .setSecretId(r.get("secretId").toString())
- .setPublicKey(r.get("publicKey").toString())
- .setPassphrase(r.get("passphrase").toString())
- .setUser(r.get("user").toString())
- .setPrivateKey(r.get("privateKey").toString()).build();
-
- return scpSecret;
- }).collect(Collectors.toList());
- return scpSecrets.stream().filter(r -> request.getSecretId().equals(r.getSecretId())).findFirst();
- }
- }
-
- @Override
- public SCPSecret createSCPSecret(SCPSecretCreateRequest request) {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateSCPSecret(SCPSecretUpdateRequest request) {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteSCPSecret(SCPSecretDeleteRequest request) {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<S3Secret> getS3Secret(S3SecretGetRequest request) throws Exception {
- JSONParser jsonParser = new JSONParser();
- InputStream inputStream = FileBasedSecretBackend.class.getClassLoader().getResourceAsStream(secretFile);
-
- try (InputStreamReader reader = new InputStreamReader(inputStream)) {
- Object obj = jsonParser.parse(reader);
-
- JSONArray resourceList = (JSONArray) obj;
-
- List<S3Secret> s3Secrets = (List<S3Secret>) resourceList.stream()
- .filter(resource -> "S3".equals(((JSONObject) resource).get("type").toString()))
- .map(resource -> {
- JSONObject r = (JSONObject) resource;
-
- S3Secret s3Secret = S3Secret.newBuilder()
- .setSecretId(r.get("secretId").toString())
- .setAccessKey(r.get("accessKey").toString())
- .setSecretKey(r.get("secretKey").toString()).build();
-
- return s3Secret;
- }).collect(Collectors.toList());
- return s3Secrets.stream().filter(r -> request.getSecretId().equals(r.getSecretId())).findFirst();
- }
- }
-
- @Override
- public S3Secret createS3Secret(S3SecretCreateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateS3Secret(S3SecretUpdateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteS3Secret(S3SecretDeleteRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<BoxSecret> getBoxSecret(BoxSecretGetRequest request) throws Exception {
- JSONParser jsonParser = new JSONParser();
- InputStream inputStream = FileBasedSecretBackend.class.getClassLoader().getResourceAsStream(secretFile);
-
- try (InputStreamReader reader = new InputStreamReader(inputStream)) {
- Object obj = jsonParser.parse(reader);
-
- JSONArray resourceList = (JSONArray) obj;
-
- List<BoxSecret> boxSecrets = (List<BoxSecret>) resourceList.stream()
- .filter(resource -> "BOX".equals(((JSONObject) resource).get("type").toString()))
- .map(resource -> {
- JSONObject r = (JSONObject) resource;
-
- BoxSecret boxSecret = BoxSecret.newBuilder()
- .setSecretId(r.get("secretId").toString())
- .setAccessToken(r.get("accessToken").toString())
- .build();
-
- return boxSecret;
- }).collect(Collectors.toList());
- return boxSecrets.stream().filter(r -> request.getSecretId().equals(r.getSecretId())).findFirst();
- }
- }
-
- @Override
- public BoxSecret createBoxSecret(BoxSecretCreateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateBoxSecret(BoxSecretUpdateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteBoxSecret(BoxSecretDeleteRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<AzureSecret> getAzureSecret(AzureSecretGetRequest request) throws Exception {
- JSONParser jsonParser = new JSONParser();
- InputStream inputStream = FileBasedSecretBackend.class.getClassLoader().getResourceAsStream(secretFile);
-
- try (InputStreamReader reader = new InputStreamReader(inputStream)) {
- Object obj = jsonParser.parse(reader);
-
- JSONArray resourceList = (JSONArray) obj;
-
- List<AzureSecret> azureSecrets = (List<AzureSecret>) resourceList.stream()
- .filter(resource -> "AZURE".equals(((JSONObject) resource).get("type").toString()))
- .map(resource -> {
- JSONObject r = (JSONObject) resource;
-
- AzureSecret azureSecret = AzureSecret.newBuilder()
- .setSecretId(r.get("secretId").toString())
- .setConnectionString(r.get("connectionString").toString()).build();
-
- return azureSecret;
- }).collect(Collectors.toList());
- return azureSecrets.stream().filter(r -> request.getSecretId().equals(r.getSecretId())).findFirst();
- }
- }
-
- @Override
- public AzureSecret createAzureSecret(AzureSecretCreateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateAzureSecret(AzureSecretUpdateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteAzureSecret(AzureSecretDeleteRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<GCSSecret> getGCSSecret(GCSSecretGetRequest request) throws Exception {
- JSONParser jsonParser = new JSONParser();
- InputStream inputStream = FileBasedSecretBackend.class.getClassLoader().getResourceAsStream(secretFile);
-
- try (InputStreamReader reader = new InputStreamReader(inputStream)) {
- Object obj = jsonParser.parse(reader);
- JSONArray resourceList = (JSONArray) obj;
-
- List<GCSSecret> gcsSecrets = (List<GCSSecret>) resourceList.stream()
- .filter(resource -> "GCS".equals(((JSONObject) resource).get("type").toString()))
- .map(resource -> {
- JSONObject r = (JSONObject) resource;
-
- GCSSecret gcsSecret = GCSSecret.newBuilder()
- .setSecretId(r.get("secretId").toString())
- .setCredentialsJson(r.get("credentialsJson").toString()).build();
- return gcsSecret;
-
- }).collect(Collectors.toList());
- return gcsSecrets.stream().filter(r -> request.getSecretId().equals(r.getSecretId())).findFirst();
- }
- }
-
- @Override
- public GCSSecret createGCSSecret(GCSSecretCreateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateGCSSecret(GCSSecretUpdateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteGCSSecret(GCSSecretDeleteRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<DropboxSecret> getDropboxSecret(DropboxSecretGetRequest request) throws Exception {
- JSONParser jsonParser = new JSONParser();
- InputStream inputStream = FileBasedSecretBackend.class.getClassLoader().getResourceAsStream(secretFile);
-
- try (InputStreamReader reader = new InputStreamReader(inputStream)) {
- Object obj = jsonParser.parse(reader);
- JSONArray resourceList = (JSONArray) obj;
-
- List<DropboxSecret> dbxSecrets = (List<DropboxSecret>) resourceList.stream()
- .filter(resource -> "DROPBOX".equals(((JSONObject) resource).get("type").toString()))
- .map(resource -> {
- JSONObject r = (JSONObject) resource;
-
- DropboxSecret dbxSecret = DropboxSecret.newBuilder()
- .setSecretId(r.get("secretId").toString())
- .setAccessToken(r.get("accessToken").toString()).build();
- return dbxSecret;
-
- }).collect(Collectors.toList());
- return dbxSecrets.stream().filter(r -> request.getSecretId().equals(r.getSecretId())).findFirst();
- }
- }
-
- @Override
- public DropboxSecret createDropboxSecret(DropboxSecretCreateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateDropboxSecret(DropboxSecretUpdateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteDropboxSecret(DropboxSecretDeleteRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- public Optional<FTPSecret> getFTPSecret(FTPSecretGetRequest request) throws Exception {
- JSONParser jsonParser = new JSONParser();
- InputStream inputStream = FileBasedSecretBackend.class.getClassLoader().getResourceAsStream(secretFile);
-
- if (inputStream == null) {
- throw new IOException("secrets file not found");
- }
-
- try (InputStreamReader reader = new InputStreamReader(inputStream)) {
- Object obj = jsonParser.parse(reader);
- JSONArray resourceList = (JSONArray) obj;
-
- List<FTPSecret> ftpSecrets = (List<FTPSecret>) resourceList.stream()
- .filter(resource -> "FTP".equals(((JSONObject) resource).get("type").toString()))
- .map(resource -> {
- JSONObject r = (JSONObject) resource;
-
- return FTPSecret.newBuilder()
- .setSecretId(r.get("secretId").toString())
- .setUserId(r.get("userId").toString())
- .setPassword(r.get("password").toString()).build();
-
- }).collect(Collectors.toList());
- return ftpSecrets.stream().filter(r -> request.getSecretId().equals(r.getSecretId())).findFirst();
- }
- }
-
- @Override
- public FTPSecret createFTPSecret(FTPSecretCreateRequest request) {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateFTPSecret(FTPSecretUpdateRequest request) {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteFTPSecret(FTPSecretDeleteRequest request) {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<SwiftSecret> getSwiftSecret(SwiftSecretGetRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public SwiftSecret createSwiftSecret(SwiftSecretCreateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateSwiftSecret(SwiftSecretUpdateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteSwiftSecret(SwiftSecretDeleteRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public Optional<ODataSecret> getODataSecret(ODataSecretGetRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public ODataSecret createODataSecret(ODataSecretCreateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean updateODataSecret(ODataSecretUpdateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
- @Override
- public boolean deleteODataSecret(ODataSecretDeleteRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
- }
-
-}
diff --git a/services/secret-service/stub/src/main/proto/gcs/GCSCredential.proto b/services/secret-service/stub/src/main/proto/gcs/GCSCredential.proto
index f4ab974..0d7d2a0 100644
--- a/services/secret-service/stub/src/main/proto/gcs/GCSCredential.proto
+++ b/services/secret-service/stub/src/main/proto/gcs/GCSCredential.proto
@@ -22,9 +22,12 @@ package org.apache.airavata.mft.credential.stubs.gcs;
import "CredCommon.proto";
+// This represents GCS Service account credentials. https://cloud.google.com/iam/docs/service-accounts
message GCSSecret {
string secretId = 1;
- string credentialsJson = 2;
+ string projectId = 2;
+ string privateKey = 3;
+ string clientEmail = 4;
}
message GCSSecretGetRequest {
@@ -39,8 +42,10 @@ message GCSSecretCreateRequest {
message GCSSecretUpdateRequest {
string secretId = 1;
- string credentialsJson = 2;
- org.apache.airavata.mft.common.AuthToken authzToken = 3;
+ string projectId = 2;
+ string privateKey = 3;
+ string clientEmail = 4;
+ org.apache.airavata.mft.common.AuthToken authzToken = 5;
}
message GCSSecretUpdateResponse {
diff --git a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java
index d037f52..79f8715 100644
--- a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java
+++ b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java
@@ -17,45 +17,29 @@
package org.apache.airavata.mft.transport.azure;
+import com.azure.core.http.rest.PagedIterable;
import com.azure.storage.blob.BlobClient;
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.blob.models.BlobItem;
+import com.azure.storage.blob.models.BlobItemProperties;
import com.azure.storage.blob.models.BlobProperties;
-import org.apache.airavata.mft.common.AuthToken;
-import org.apache.airavata.mft.core.DirectoryResourceMetadata;
-import org.apache.airavata.mft.core.FileResourceMetadata;
-import org.apache.airavata.mft.core.ResourceTypes;
+import org.apache.airavata.mft.agent.stub.*;
import org.apache.airavata.mft.core.api.MetadataCollector;
import org.apache.airavata.mft.credential.stubs.azure.AzureSecret;
-import org.apache.airavata.mft.credential.stubs.azure.AzureSecretGetRequest;
-import org.apache.airavata.mft.resource.client.ResourceServiceClient;
-import org.apache.airavata.mft.resource.client.ResourceServiceClientBuilder;
-import org.apache.airavata.mft.resource.client.StorageServiceClient;
-import org.apache.airavata.mft.resource.client.StorageServiceClientBuilder;
import org.apache.airavata.mft.resource.stubs.azure.storage.AzureStorage;
-import org.apache.airavata.mft.resource.stubs.azure.storage.AzureStorageGetRequest;
-import org.apache.airavata.mft.resource.stubs.common.FileResource;
-import org.apache.airavata.mft.resource.stubs.common.GenericResource;
-import org.apache.airavata.mft.resource.stubs.common.GenericResourceGetRequest;
-import org.apache.airavata.mft.resource.stubs.s3.storage.S3StorageGetRequest;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
public class AzureMetadataCollector implements MetadataCollector {
- private String resourceServiceHost;
- private int resourceServicePort;
- private String secretServiceHost;
- private int secretServicePort;
boolean initialized = false;
+ private AzureStorage azureStorage;
+ private AzureSecret azureSecret;
@Override
- public void init( String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) {
- this.resourceServiceHost = resourceServiceHost;
- this.resourceServicePort = resourceServicePort;
- this.secretServiceHost = secretServiceHost;
- this.secretServicePort = secretServicePort;
+ public void init(StorageWrapper storage, SecretWrapper secret) {
+ this.azureStorage = storage.getAzure();
+ this.azureSecret = secret.getAzure();
this.initialized = true;
}
@@ -66,84 +50,79 @@ public class AzureMetadataCollector implements MetadataCollector {
}
@Override
- public FileResourceMetadata getFileResourceMetadata(AuthToken authZToken, String resourcePath,
- String storageId, String credentialToken) throws Exception {
+ public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception {
checkInitialized();
- if (!isAvailable(authZToken, resourcePath, storageId, credentialToken)) {
- throw new Exception("Azure blob can not find for resource path " + resourcePath);
- }
-
- AzureStorage azureStorage;
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(resourceServiceHost, resourceServicePort)) {
-
- azureStorage = storageServiceClient.azure()
- .getAzureStorage(AzureStorageGetRequest.newBuilder().setStorageId(storageId).build());
- }
-
- AzureSecret azureSecret;
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- secretServiceHost, secretServicePort)) {
- azureSecret = secretClient.azure().getAzureSecret(AzureSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
-
+ // Azure does not have a concept called hierarchical containers. So we assume that there are no containers inside
+ // the given container
+ ResourceMetadata.Builder metadataBuilder = ResourceMetadata.newBuilder();
+ if (!isAvailable(resourcePath)) {
+ metadataBuilder.setError(MetadataFetchError.NOT_FOUND);
+ return metadataBuilder.build();
}
BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(azureSecret.getConnectionString()).buildClient();
- BlobClient blobClient = blobServiceClient.getBlobContainerClient(azureStorage.getContainer())
- .getBlobClient(resourcePath);
-
- BlobProperties properties = blobClient.getBlockBlobClient().getProperties();
- FileResourceMetadata metadata = new FileResourceMetadata();
- metadata.setResourceSize(properties.getBlobSize());
- metadata.setCreatedTime(properties.getCreationTime().toEpochSecond());
- metadata.setUpdateTime(properties.getCreationTime().toEpochSecond());
-
- byte[] contentMd5 = properties.getContentMd5();
- StringBuilder md5sb = new StringBuilder();
- for (byte aByte : contentMd5) {
- md5sb.append(Integer.toString((aByte & 0xff) + 0x100, 16).substring(1));
+ BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(azureStorage.getContainer());
+
+ if (resourcePath.isEmpty()) { // List the container
+ PagedIterable<BlobItem> blobItems = blobContainerClient.listBlobs();
+ DirectoryMetadata.Builder directoryBuilder = DirectoryMetadata.newBuilder();
+ blobItems.forEach(blobItem -> {
+ FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
+ BlobItemProperties properties = blobItem.getProperties();
+
+ fileBuilder.setResourceSize(properties.getContentLength());
+ fileBuilder.setCreatedTime(properties.getCreationTime().toEpochSecond());
+ fileBuilder.setUpdateTime(properties.getCreationTime().toEpochSecond());
+ fileBuilder.setFriendlyName(blobItem.getName());
+ fileBuilder.setResourcePath(blobItem.getName());
+ byte[] contentMd5 = properties.getContentMd5();
+ StringBuilder md5sb = new StringBuilder();
+ for (byte aByte : contentMd5) {
+ md5sb.append(Integer.toString((aByte & 0xff) + 0x100, 16).substring(1));
+ }
+ fileBuilder.setMd5Sum(md5sb.toString());
+ directoryBuilder.addFiles(fileBuilder);
+ });
+ metadataBuilder.setDirectory(directoryBuilder);
+
+ } else { // If resource is a file
+
+ BlobClient blobClient = blobContainerClient.getBlobClient(resourcePath);
+ FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
+ BlobProperties properties = blobClient.getProperties();
+
+ fileBuilder.setResourceSize(properties.getBlobSize());
+ fileBuilder.setCreatedTime(properties.getCreationTime().toEpochSecond());
+ fileBuilder.setUpdateTime(properties.getCreationTime().toEpochSecond());
+ fileBuilder.setFriendlyName(blobClient.getBlobName());
+ fileBuilder.setResourcePath(resourcePath);
+ byte[] contentMd5 = properties.getContentMd5();
+ StringBuilder md5sb = new StringBuilder();
+ for (byte aByte : contentMd5) {
+ md5sb.append(Integer.toString((aByte & 0xff) + 0x100, 16).substring(1));
+ }
+ fileBuilder.setMd5Sum(md5sb.toString());
+ metadataBuilder.setFile(fileBuilder);
}
- metadata.setMd5sum(md5sb.toString());
-
- return metadata;
- }
-
- @Override
- public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception {
- throw new UnsupportedOperationException("Method not implemented");
+ return metadataBuilder.build();
}
-
@Override
- public Boolean isAvailable(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception {
+ public Boolean isAvailable(String resourcePath) throws Exception {
checkInitialized();
- AzureStorage storage;
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(resourceServiceHost, resourceServicePort)) {
-
- storage = storageServiceClient.azure()
- .getAzureStorage(AzureStorageGetRequest.newBuilder().setStorageId(storageId).build());
- }
-
- AzureSecret azureSecret;
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- secretServiceHost, secretServicePort)) {
- azureSecret = secretClient.azure().getAzureSecret(AzureSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
-
- }
-
BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(azureSecret.getConnectionString()).buildClient();
- BlobContainerClient containerClient = blobServiceClient.getBlobContainerClient(storage.getContainer());
+ BlobContainerClient containerClient = blobServiceClient.getBlobContainerClient(azureStorage.getContainer());
boolean containerExists = containerClient.exists();
if (!containerExists) {
return false;
}
+ if (resourcePath.isEmpty()) {
+ return true;
+ }
return containerClient.getBlobClient(resourcePath).exists();
}
-
-
}
diff --git a/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxMetadataCollector.java b/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxMetadataCollector.java
index cde5542..48e6cb8 100644
--- a/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxMetadataCollector.java
+++ b/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxMetadataCollector.java
@@ -18,31 +18,26 @@
package org.apache.airavata.mft.transport.box;
-import com.box.sdk.BoxAPIConnection;
-import com.box.sdk.BoxFile;
-import org.apache.airavata.mft.common.AuthToken;
-import org.apache.airavata.mft.core.DirectoryResourceMetadata;
-import org.apache.airavata.mft.core.FileResourceMetadata;
+import com.box.sdk.*;
+import org.apache.airavata.mft.agent.stub.*;
import org.apache.airavata.mft.core.api.MetadataCollector;
import org.apache.airavata.mft.credential.stubs.box.BoxSecret;
-import org.apache.airavata.mft.credential.stubs.box.BoxSecretGetRequest;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
+import org.apache.airavata.mft.resource.stubs.box.storage.BoxStorage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class BoxMetadataCollector implements MetadataCollector {
- private String resourceServiceHost;
- private int resourceServicePort;
- private String secretServiceHost;
- private int secretServicePort;
+ private static final Logger logger = LoggerFactory.getLogger(BoxMetadataCollector.class);
+
boolean initialized = false;
+ private BoxStorage boxStorage;
+ private BoxSecret boxSecret;
@Override
- public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) {
- this.resourceServiceHost = resourceServiceHost;
- this.resourceServicePort = resourceServicePort;
- this.secretServiceHost = secretServiceHost;
- this.secretServicePort = secretServicePort;
+ public void init(StorageWrapper storage, SecretWrapper secret) {
+ this.boxStorage = storage.getBox();
+ this.boxSecret = secret.getBox();
this.initialized = true;
}
@@ -53,53 +48,90 @@ public class BoxMetadataCollector implements MetadataCollector {
}
@Override
- public FileResourceMetadata getFileResourceMetadata(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception {
+ public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception {
checkInitialized();
- BoxSecret boxSecret;
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- secretServiceHost, secretServicePort)) {
- boxSecret = secretClient.box().getBoxSecret(BoxSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
-
- }
-
BoxAPIConnection api = new BoxAPIConnection(boxSecret.getAccessToken());
- BoxFile boxFile = new BoxFile(api, resourcePath);
- BoxFile.Info boxFileInfo = boxFile.getInfo();
- FileResourceMetadata metadata = new FileResourceMetadata();
- metadata.setResourceSize(boxFileInfo.getSize());
-
- // TODO
- // metadata.setMd5sum(boxFileInfo.getSha1());
-
- metadata.setUpdateTime(boxFileInfo.getModifiedAt().getTime());
- metadata.setCreatedTime(boxFileInfo.getCreatedAt().getTime());
-
- return metadata;
- }
+ BoxFile boxFile = null;
+ BoxFile.Info boxFileInfo = null;
+ BoxFolder boxFolder = null;
+ BoxFolder.Info boxFolderInfo;
+ boolean isFile = true;
+
+ ResourceMetadata.Builder resourceBuilder = ResourceMetadata.newBuilder();
+
+ try { // There is no API to derive the type of entry. So, manually checking the file type is used
+ boxFile = new BoxFile(api, resourcePath);
+ boxFileInfo = boxFile.getInfo();
+ } catch (BoxAPIException e) {
+ try {
+ boxFolder = new BoxFolder(api, resourcePath);
+ boxFolderInfo = boxFolder.getInfo();
+ isFile = false;
+ } catch (BoxAPIException ex) {
+ logger.error("Failed to fetch information of box item {}", resourcePath, ex);
+ resourceBuilder.setError(MetadataFetchError.NOT_FOUND);
+ return resourceBuilder.build();
+ }
+ }
- @Override
- public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception {
- throw new UnsupportedOperationException("Method not implemented");
+ if (isFile) {
+ FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
+ fileBuilder.setResourcePath(resourcePath);
+ fileBuilder.setFriendlyName(boxFileInfo.getName());
+ fileBuilder.setResourceSize(boxFileInfo.getSize());
+ fileBuilder.setCreatedTime(boxFileInfo.getCreatedAt().getTime());
+ fileBuilder.setUpdateTime(boxFileInfo.getModifiedAt().getTime());
+ resourceBuilder.setFile(fileBuilder);
+ } else {
+ DirectoryMetadata.Builder dirBuilder = DirectoryMetadata.newBuilder();
+
+ Iterable<BoxItem.Info> itemsInFolder = boxFolder.getChildren("metadata.global.properties");
+ for (BoxItem.Info itemInfo : itemsInFolder) {
+ if (itemInfo instanceof BoxFile.Info) {
+ BoxFile.Info fileInfo = (BoxFile.Info) itemInfo;
+ FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
+ fileBuilder.setResourcePath(fileInfo.getID());
+ fileBuilder.setFriendlyName(fileInfo.getName());
+ fileBuilder.setResourceSize(fileInfo.getSize());
+ fileBuilder.setCreatedTime(fileInfo.getCreatedAt().getTime());
+ fileBuilder.setUpdateTime(fileInfo.getModifiedAt().getTime());
+ dirBuilder.addFiles(fileBuilder);
+ } else if (itemInfo instanceof BoxFolder.Info) {
+ BoxFolder.Info folderInfo = (BoxFolder.Info) itemInfo;
+ DirectoryMetadata.Builder subDirBuilder = DirectoryMetadata.newBuilder();
+ subDirBuilder.setFriendlyName(folderInfo.getName());
+ subDirBuilder.setResourcePath(folderInfo.getID());
+ subDirBuilder.setCreatedTime(folderInfo.getCreatedAt().getTime());
+ subDirBuilder.setUpdateTime(folderInfo.getModifiedAt().getTime());
+ dirBuilder.addDirectories(subDirBuilder);
+ }
+ }
+ resourceBuilder.setDirectory(dirBuilder);
+ }
+ return resourceBuilder.build();
}
@Override
- public Boolean isAvailable(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception {
+ public Boolean isAvailable(String resourcePath) throws Exception {
checkInitialized();
- BoxSecret boxSecret;
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- secretServiceHost, secretServicePort)) {
- boxSecret = secretClient.box().getBoxSecret(BoxSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
-
- }
-
BoxAPIConnection api = new BoxAPIConnection(boxSecret.getAccessToken());
- BoxFile boxFile = new BoxFile(api, resourcePath);
- // TODO Fix this. Need to figur out how to check if the file exists in box
+ try {
+ BoxFile boxFile = new BoxFile(api, resourcePath);
+ boxFile.getInfo();
+ } catch (BoxAPIException e) {
+ try {
+ BoxFolder boxFolder = new BoxFolder(api, resourcePath);
+ boxFolder.getInfo();
+ } catch (BoxAPIException ex) {
+ logger.warn("Failed to fetch information of box item {}", resourcePath, ex);
+ return false;
+ }
+ }
return true;
}
}
diff --git a/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxMetadataCollector.java b/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxMetadataCollector.java
index fe81f7a..1a767e8 100644
--- a/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxMetadataCollector.java
+++ b/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxMetadataCollector.java
@@ -19,30 +19,27 @@ package org.apache.airavata.mft.transport.dropbox;
import com.dropbox.core.DbxRequestConfig;
import com.dropbox.core.v2.DbxClientV2;
-import com.dropbox.core.v2.files.FileMetadata;
-import org.apache.airavata.mft.common.AuthToken;
-import org.apache.airavata.mft.core.DirectoryResourceMetadata;
-import org.apache.airavata.mft.core.FileResourceMetadata;
+import com.dropbox.core.v2.files.FolderMetadata;
+import com.dropbox.core.v2.files.ListFolderResult;
+import com.dropbox.core.v2.files.Metadata;
+import org.apache.airavata.mft.agent.stub.*;
import org.apache.airavata.mft.core.api.MetadataCollector;
import org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecret;
-import org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecretGetRequest;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
+import org.apache.airavata.mft.resource.stubs.dropbox.storage.DropboxStorage;
+
+import java.util.List;
public class DropboxMetadataCollector implements MetadataCollector {
- private String resourceServiceHost;
- private int resourceServicePort;
- private String secretServiceHost;
- private int secretServicePort;
+ private DropboxStorage dropboxStorage;
+ private DropboxSecret dropboxSecret;
+
boolean initialized = false;
@Override
- public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) {
- this.resourceServiceHost = resourceServiceHost;
- this.resourceServicePort = resourceServicePort;
- this.secretServiceHost = secretServiceHost;
- this.secretServicePort = secretServicePort;
+ public void init(StorageWrapper storage, SecretWrapper secret) {
+ this.dropboxStorage = storage.getDropbox();
+ this.dropboxSecret = secret.getDropbox();
this.initialized = true;
}
@@ -53,46 +50,92 @@ public class DropboxMetadataCollector implements MetadataCollector {
}
@Override
- public FileResourceMetadata getFileResourceMetadata(AuthToken authZToken, String resourcePath, String storageId,
- String credentialToken) throws Exception {
+ public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception {
checkInitialized();
- DropboxSecret dropboxSecret;
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- secretServiceHost, secretServicePort)) {
- dropboxSecret = secretClient.dropbox().getDropboxSecret(DropboxSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
-
- }
-
DbxRequestConfig config = DbxRequestConfig.newBuilder("mftdropbox/v1").build();
DbxClientV2 dbxClientV2 = new DbxClientV2(config, dropboxSecret.getAccessToken());
- FileResourceMetadata metadata = new FileResourceMetadata();
- FileMetadata fileMetadata = (FileMetadata) dbxClientV2.files().getMetadata(resourcePath);
- metadata.setResourceSize(fileMetadata.getSize());
- metadata.setMd5sum(null);
- metadata.setUpdateTime(fileMetadata.getServerModified().getTime());
- metadata.setCreatedTime(fileMetadata.getClientModified().getTime());
- return metadata;
- }
+ ResourceMetadata.Builder resourceBuilder = ResourceMetadata.newBuilder();
+
+ if (resourcePath.isEmpty()) { // List root folder
+ ListFolderResult listFolderResult = dbxClientV2.files().listFolder(resourcePath);
+ List<Metadata> entryMetadatas = listFolderResult.getEntries();
+ DirectoryMetadata.Builder directoryBuilder = DirectoryMetadata.newBuilder();
+ directoryBuilder.setResourcePath("");
+ directoryBuilder.setFriendlyName("");
+
+ entryMetadatas.forEach(entryMetadata -> {
+ if (entryMetadata instanceof com.dropbox.core.v2.files.FileMetadata) {
+ com.dropbox.core.v2.files.FileMetadata fileMetadata = (com.dropbox.core.v2.files.FileMetadata) entryMetadata;
+ FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
+ fileBuilder.setResourceSize(fileMetadata.getSize());
+ fileBuilder.setUpdateTime(fileMetadata.getServerModified().getTime());
+ fileBuilder.setCreatedTime(fileMetadata.getClientModified().getTime());
+ fileBuilder.setFriendlyName(fileMetadata.getName());
+ fileBuilder.setResourcePath(resourcePath);
+ directoryBuilder.addFiles(fileBuilder);
+ } else if (entryMetadata instanceof FolderMetadata) {
+ FolderMetadata folderMetadata = (FolderMetadata) entryMetadata;
+ DirectoryMetadata.Builder subDirBuilder = DirectoryMetadata.newBuilder();
+ subDirBuilder.setFriendlyName(folderMetadata.getName());
+ subDirBuilder.setResourcePath(folderMetadata.getName());
+ directoryBuilder.addDirectories(subDirBuilder);
+ }
+ });
+
+ resourceBuilder.setDirectory(directoryBuilder);
+
+ } else {
+ Metadata entryMetadata = dbxClientV2.files().getMetadata(resourcePath);
+ if (entryMetadata instanceof com.dropbox.core.v2.files.FileMetadata) {
+ com.dropbox.core.v2.files.FileMetadata fileMetadata = (com.dropbox.core.v2.files.FileMetadata) entryMetadata;
+ FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
+ fileBuilder.setResourceSize(fileMetadata.getSize());
+ fileBuilder.setUpdateTime(fileMetadata.getServerModified().getTime());
+ fileBuilder.setCreatedTime(fileMetadata.getClientModified().getTime());
+ fileBuilder.setFriendlyName(fileMetadata.getName());
+ fileBuilder.setResourcePath(resourcePath);
+ resourceBuilder.setFile(fileBuilder);
+ } else if (entryMetadata instanceof FolderMetadata) {
+
+ ListFolderResult listFolderResult = dbxClientV2.files().listFolder(resourcePath);
+ List<Metadata> entryMetadatas = listFolderResult.getEntries();
+ DirectoryMetadata.Builder directoryBuilder = DirectoryMetadata.newBuilder();
+ directoryBuilder.setResourcePath(resourcePath);
+ directoryBuilder.setFriendlyName(entryMetadata.getName());
+
+ entryMetadatas.forEach(em -> {
+ if (em instanceof com.dropbox.core.v2.files.FileMetadata) {
+ com.dropbox.core.v2.files.FileMetadata fileMetadata = (com.dropbox.core.v2.files.FileMetadata) em;
+ FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
+ fileBuilder.setResourceSize(fileMetadata.getSize());
+ fileBuilder.setUpdateTime(fileMetadata.getServerModified().getTime());
+ fileBuilder.setCreatedTime(fileMetadata.getClientModified().getTime());
+ fileBuilder.setFriendlyName(fileMetadata.getName());
+ fileBuilder.setResourcePath(resourcePath + "/" + fileMetadata.getName());
+ directoryBuilder.addFiles(fileBuilder);
+ } else if (em instanceof FolderMetadata) {
+ FolderMetadata folderMetadata = (FolderMetadata) em;
+ DirectoryMetadata.Builder subDirBuilder = DirectoryMetadata.newBuilder();
+ subDirBuilder.setFriendlyName(folderMetadata.getName());
+ subDirBuilder.setResourcePath(resourcePath + "/" + folderMetadata.getName());
+ directoryBuilder.addDirectories(subDirBuilder);
+ }
+ resourceBuilder.setDirectory(directoryBuilder);
+ });
+ } else {
+ resourceBuilder.setError(MetadataFetchError.NOT_FOUND);
+ }
+ }
- @Override
- public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception {
- throw new UnsupportedOperationException("Method not implemented");
+ return resourceBuilder.build();
}
-
@Override
- public Boolean isAvailable(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception {
+ public Boolean isAvailable(String resourcePath) throws Exception {
checkInitialized();
- DropboxSecret dropboxSecret;
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- secretServiceHost, secretServicePort)) {
- dropboxSecret = secretClient.dropbox().getDropboxSecret(DropboxSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
-
- }
-
DbxRequestConfig config = DbxRequestConfig.newBuilder("mftdropbox/v1").build();
DbxClientV2 dbxClientV2 = new DbxClientV2(config, dropboxSecret.getAccessToken());
diff --git a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java
index ce1050c..2e2ba33 100644
--- a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java
+++ b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java
@@ -17,42 +17,27 @@
package org.apache.airavata.mft.transport.ftp;
-import org.apache.airavata.mft.common.AuthToken;
-import org.apache.airavata.mft.core.DirectoryResourceMetadata;
-import org.apache.airavata.mft.core.FileResourceMetadata;
+import org.apache.airavata.mft.agent.stub.*;
import org.apache.airavata.mft.core.api.MetadataCollector;
import org.apache.airavata.mft.credential.stubs.ftp.FTPSecret;
-import org.apache.airavata.mft.credential.stubs.ftp.FTPSecretGetRequest;
-import org.apache.airavata.mft.resource.client.StorageServiceClient;
-import org.apache.airavata.mft.resource.client.StorageServiceClientBuilder;
import org.apache.airavata.mft.resource.stubs.ftp.storage.FTPStorage;
-import org.apache.airavata.mft.resource.stubs.ftp.storage.FTPStorageGetRequest;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;
-import org.apache.commons.net.ftp.FTPReply;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.InputStream;
+import java.util.Arrays;
public class FTPMetadataCollector implements MetadataCollector {
private static final Logger logger = LoggerFactory.getLogger(FTPMetadataCollector.class);
-
- private String resourceServiceHost;
- private int resourceServicePort;
- private String secretServiceHost;
- private int secretServicePort;
private boolean initialized = false;
-
+ private FTPStorage ftpStorage;
+ private FTPSecret ftpSecret;
@Override
- public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) {
- this.resourceServiceHost = resourceServiceHost;
- this.resourceServicePort = resourceServicePort;
- this.secretServiceHost = secretServiceHost;
- this.secretServicePort = secretServicePort;
+ public void init(StorageWrapper storage, SecretWrapper secret) {
+ this.ftpStorage = storage.getFtp();
+ this.ftpSecret = secret.getFtp();
this.initialized = true;
}
@@ -63,88 +48,72 @@ public class FTPMetadataCollector implements MetadataCollector {
}
@Override
- public FileResourceMetadata getFileResourceMetadata(AuthToken authZToken, String resourcePath,
- String storageId, String credentialToken) throws Exception {
+ public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception {
checkInitialized();
-
- FTPStorage ftpStorage;
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(resourceServiceHost, resourceServicePort)) {
-
- ftpStorage = storageServiceClient.ftp()
- .getFTPStorage(FTPStorageGetRequest.newBuilder().setStorageId(storageId).build());
- }
-
- FTPSecret ftpSecret;
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- secretServiceHost, secretServicePort)) {
- ftpSecret = secretClient.ftp().getFTPSecret(FTPSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
- }
-
- FileResourceMetadata resourceMetadata = new FileResourceMetadata();
+ ResourceMetadata.Builder resourceBuilder = ResourceMetadata.newBuilder();
FTPClient ftpClient = null;
try {
ftpClient = FTPTransportUtil.getFTPClient(ftpStorage, ftpSecret);
logger.info("Fetching metadata for resource {} in {}", resourcePath, ftpStorage.getHost());
FTPFile ftpFile = ftpClient.mlistFile(resourcePath);
+ if (ftpFile == null) {
+ resourceBuilder.setError(MetadataFetchError.NOT_FOUND);
+ return resourceBuilder.build();
+ }
- if (ftpFile != null) {
- resourceMetadata.setResourceSize(ftpFile.getSize());
- resourceMetadata.setUpdateTime(ftpFile.getTimestamp().getTimeInMillis());
- if (ftpClient.hasFeature("MD5") && FTPReply.isPositiveCompletion(ftpClient.sendCommand("MD5 " + resourcePath))) {
- String[] replies = ftpClient.getReplyStrings();
- resourceMetadata.setMd5sum(replies[0]);
- } else {
- logger.warn("MD5 fetch error out {}", ftpClient.getReplyString());
- }
+ if (ftpFile.isDirectory()) {
+
+ DirectoryMetadata.Builder dirBuilder = DirectoryMetadata.newBuilder();
+ FTPFile[] ftpFiles = ftpClient.listFiles(resourcePath);
+ Arrays.stream(ftpFiles).forEach(child -> {
+ if (child.isFile()) {
+ FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
+ fileBuilder.setResourcePath(resourcePath + "/" + child.getName());
+ fileBuilder.setFriendlyName(child.getName());
+ fileBuilder.setCreatedTime(child.getTimestamp().getTimeInMillis());
+ fileBuilder.setUpdateTime(child.getTimestamp().getTimeInMillis());
+ fileBuilder.setResourceSize(child.getSize());
+ dirBuilder.addFiles(fileBuilder);
+ } else if (child.isFile()) {
+ DirectoryMetadata.Builder childDirBuilder = DirectoryMetadata.newBuilder();
+ childDirBuilder.setResourcePath(resourcePath + "/" + child.getName());
+ childDirBuilder.setFriendlyName(child.getName());
+ childDirBuilder.setCreatedTime(child.getTimestamp().getTimeInMillis());
+ childDirBuilder.setUpdateTime(child.getTimestamp().getTimeInMillis());
+ dirBuilder.addDirectories(childDirBuilder);
+ }
+ });
+ resourceBuilder.setDirectory(dirBuilder);
+ } else if (ftpFile.isFile()) {
+ FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
+ fileBuilder.setResourcePath(resourcePath);
+ fileBuilder.setFriendlyName(ftpFile.getName());
+ fileBuilder.setCreatedTime(ftpFile.getTimestamp().getTimeInMillis());
+ fileBuilder.setUpdateTime(ftpFile.getTimestamp().getTimeInMillis());
+ fileBuilder.setResourceSize(ftpFile.getSize());
+ resourceBuilder.setFile(fileBuilder);
}
- } catch (Exception e) {
- logger.warn("Failed to fetch md5 for FTP resource path {}", resourcePath, e);
+
+ return resourceBuilder.build();
+
} finally {
FTPTransportUtil.disconnectFTP(ftpClient);
}
-
- return resourceMetadata;
- }
-
- @Override
- public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthToken authZToken, String resourcePath,
- String storageId, String credentialToken) throws Exception {
- throw new UnsupportedOperationException("Method not implemented");
}
@Override
- public Boolean isAvailable(AuthToken authZToken, String resourcePath, String storageId, String credentialToken)
- throws Exception{
+ public Boolean isAvailable(String resourcePath) throws Exception{
checkInitialized();
- FTPStorage ftpStorage;
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(resourceServiceHost, resourceServicePort)) {
-
- ftpStorage = storageServiceClient.ftp()
- .getFTPStorage(FTPStorageGetRequest.newBuilder().setStorageId(storageId).build());
- }
-
- FTPSecret ftpSecret;
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- secretServiceHost, secretServicePort)) {
- ftpSecret = secretClient.ftp().getFTPSecret(FTPSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
- }
-
FTPClient ftpClient = null;
try {
ftpClient = FTPTransportUtil.getFTPClient(ftpStorage, ftpSecret);
- InputStream inputStream = ftpClient.retrieveFileStream(resourcePath);
-
- return !(inputStream == null || ftpClient.getReplyCode() == 550);
- } catch (Exception e) {
- logger.error("FTP client initialization failed ", e);
- return false;
+ FTPFile ftpFile = ftpClient.mlistFile(resourcePath);
+ return ftpFile != null;
} finally {
FTPTransportUtil.disconnectFTP(ftpClient);
}
diff --git a/transport/gcp-transport/pom.xml b/transport/gcp-transport/pom.xml
index cf7b9c7..4cdc8ad 100644
--- a/transport/gcp-transport/pom.xml
+++ b/transport/gcp-transport/pom.xml
@@ -32,16 +32,34 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>mft-gcp-transport</artifactId>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>libraries-bom</artifactId>
+ <version>26.1.5</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<dependencies>
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>mft-core</artifactId>
<version>0.01-SNAPSHOT</version>
</dependency>
+
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-storage</artifactId>
+ </dependency>
<dependency>
- <groupId>com.google.apis</groupId>
- <artifactId>google-api-services-storage</artifactId>
- <version>v1-rev171-1.25.0</version>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-core</artifactId>
+ <version>1.12.152</version>
</dependency>
</dependencies>
</project>
diff --git a/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSIncomingChunkedConnector.java b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSIncomingChunkedConnector.java
new file mode 100644
index 0000000..6726c5f
--- /dev/null
+++ b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSIncomingChunkedConnector.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.transport.gcp;
+
+import com.google.auth.oauth2.ServiceAccountCredentials;
+import com.google.cloud.ReadChannel;
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageOptions;
+import com.google.common.io.ByteStreams;
+import org.apache.airavata.mft.core.api.ConnectorConfig;
+import org.apache.airavata.mft.core.api.IncomingChunkedConnector;
+import org.apache.airavata.mft.credential.stubs.gcs.GCSSecret;
+import org.apache.airavata.mft.resource.stubs.gcs.storage.GCSStorage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.security.PrivateKey;
+
+public class GCSIncomingChunkedConnector implements IncomingChunkedConnector {
+
+ private static final Logger logger = LoggerFactory.getLogger(GCSIncomingChunkedConnector.class);
+
+ private Blob blob;
+ private ConnectorConfig connectorConfig;
+
+ @Override
+ public void init(ConnectorConfig connectorConfig) throws Exception {
+ this.connectorConfig = connectorConfig;
+ GCSSecret gcsSecret = connectorConfig.getSecret().getGcs();
+
+ GCSStorage gcsStorage = connectorConfig.getStorage().getGcs();
+
+ PrivateKey privKey = GCSUtil.getPrivateKey(gcsSecret.getPrivateKey());
+
+ try (Storage storage = StorageOptions.newBuilder().setCredentials(ServiceAccountCredentials.newBuilder()
+ .setProjectId(gcsSecret.getProjectId())
+ .setPrivateKey(privKey)
+ .setClientEmail(gcsSecret.getClientEmail())
+ .build()).build().getService()) {
+
+ blob = storage.get(gcsStorage.getBucketName(),
+ connectorConfig.getResourcePath(), Storage.BlobGetOption.fields(Storage.BlobField.values()));
+ }
+ }
+
+ @Override
+ public void complete() throws Exception {
+ logger.info("File {} successfully received", connectorConfig.getResourcePath());
+
+ }
+
+ @Override
+ public void failed() throws Exception {
+ logger.error("Failed while receiving file {}", connectorConfig.getResourcePath());
+ }
+
+ @Override
+ public void downloadChunk(int chunkId, long startByte, long endByte, String downloadFile) throws Exception {
+ try (ReadChannel from = blob.reader();
+ FileChannel to = FileChannel.open(Paths.get(downloadFile), StandardOpenOption.WRITE)) {
+ from.seek(startByte);
+ from.limit(endByte);
+ ByteStreams.copy(from, to);
+ }
+ }
+
+ @Override
+ public InputStream downloadChunk(int chunkId, long startByte, long endByte) throws Exception {
+ try (ReadChannel from = blob.reader()) {
+ from.seek(startByte);
+ from.limit(endByte);
+ return Channels.newInputStream(from);
+ }
+ }
+}
diff --git a/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSIncomingStreamingConnector.java b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSIncomingStreamingConnector.java
new file mode 100644
index 0000000..5646c68
--- /dev/null
+++ b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSIncomingStreamingConnector.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.transport.gcp;
+
+import com.google.auth.oauth2.ServiceAccountCredentials;
+import com.google.cloud.ReadChannel;
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageOptions;
+import org.apache.airavata.mft.core.api.ConnectorConfig;
+import org.apache.airavata.mft.core.api.IncomingStreamingConnector;
+import org.apache.airavata.mft.credential.stubs.gcs.GCSSecret;
+import org.apache.airavata.mft.resource.stubs.gcs.storage.GCSStorage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.security.PrivateKey;
+
+public class GCSIncomingStreamingConnector implements IncomingStreamingConnector {
+
+ private static final Logger logger = LoggerFactory.getLogger(GCSIncomingStreamingConnector.class);
+
+ private Blob blob;
+ private ConnectorConfig connectorConfig;
+ @Override
+ public void init(ConnectorConfig connectorConfig) throws Exception {
+
+ this.connectorConfig = connectorConfig;
+ GCSSecret gcsSecret = connectorConfig.getSecret().getGcs();
+
+ GCSStorage gcsStorage = connectorConfig.getStorage().getGcs();
+
+ PrivateKey privKey = GCSUtil.getPrivateKey(gcsSecret.getPrivateKey());
+
+ try (Storage storage = StorageOptions.newBuilder().setCredentials(ServiceAccountCredentials.newBuilder()
+ .setProjectId(gcsSecret.getProjectId())
+ .setPrivateKey(privKey)
+ .setClientEmail(gcsSecret.getClientEmail())
+ .build()).build().getService()) {
+
+ blob = storage.get(gcsStorage.getBucketName(),
+ connectorConfig.getResourcePath(), Storage.BlobGetOption.fields(Storage.BlobField.values()));
+ }
+
+ }
+
+ @Override
+ public void complete() throws Exception {
+ logger.info("File {} successfully received", connectorConfig.getResourcePath());
+ }
+
+ @Override
+ public void failed() throws Exception {
+ logger.error("Failed while receiving file {}", connectorConfig.getResourcePath());
+ }
+
+ @Override
+ public InputStream fetchInputStream() throws Exception {
+ ReadChannel reader = blob.reader();
+ return Channels.newInputStream(reader);
+ }
+}
diff --git a/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSMetadataCollector.java b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSMetadataCollector.java
index 1f0e056..df32a6f 100644
--- a/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSMetadataCollector.java
+++ b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSMetadataCollector.java
@@ -17,47 +17,30 @@
package org.apache.airavata.mft.transport.gcp;
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
-import com.google.api.client.http.HttpTransport;
-import com.google.api.client.json.JsonFactory;
-import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.services.storage.Storage;
-import com.google.api.services.storage.StorageScopes;
-import com.google.api.services.storage.model.StorageObject;
-import org.apache.airavata.mft.common.AuthToken;
-import org.apache.airavata.mft.core.DirectoryResourceMetadata;
-import org.apache.airavata.mft.core.FileResourceMetadata;
+import com.google.api.gax.paging.Page;
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.Storage;
+import com.google.auth.oauth2.ServiceAccountCredentials;
+import com.google.cloud.storage.StorageOptions;
+import org.apache.airavata.mft.agent.stub.*;
import org.apache.airavata.mft.core.api.MetadataCollector;
import org.apache.airavata.mft.credential.stubs.gcs.GCSSecret;
-import org.apache.airavata.mft.credential.stubs.gcs.GCSSecretGetRequest;
-import org.apache.airavata.mft.resource.client.StorageServiceClient;
-import org.apache.airavata.mft.resource.client.StorageServiceClientBuilder;
import org.apache.airavata.mft.resource.stubs.gcs.storage.GCSStorage;
-import org.apache.airavata.mft.resource.stubs.gcs.storage.GCSStorageGetRequest;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
-import java.io.ByteArrayInputStream;
-import java.math.BigInteger;
-import java.nio.charset.StandardCharsets;
-import java.util.Base64;
-import java.util.Collection;
+import java.security.PrivateKey;
+import java.time.temporal.ChronoField;
public class GCSMetadataCollector implements MetadataCollector {
boolean initialized = false;
- private String resourceServiceHost;
- private int resourceServicePort;
- private String secretServiceHost;
- private int secretServicePort;
+
+ private GCSStorage gcsStorage;
+ private GCSSecret gcsSecret;
@Override
- public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) {
- this.resourceServiceHost = resourceServiceHost;
- this.resourceServicePort = resourceServicePort;
- this.secretServiceHost = secretServiceHost;
- this.secretServicePort = secretServicePort;
+ public void init(StorageWrapper storage, SecretWrapper secret) {
+ this.gcsStorage = storage.getGcs();
+ this.gcsSecret = secret.getGcs();
this.initialized = true;
}
@@ -68,79 +51,99 @@ public class GCSMetadataCollector implements MetadataCollector {
}
@Override
- public FileResourceMetadata getFileResourceMetadata(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception {
+ public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception {
checkInitialized();
- GCSStorage gcsStorage;
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(resourceServiceHost, resourceServicePort)) {
-
- gcsStorage = storageServiceClient.gcs()
- .getGCSStorage(GCSStorageGetRequest.newBuilder().setStorageId(storageId).build());
- }
-
- GCSSecret gcsSecret;
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- secretServiceHost, secretServicePort)) {
- gcsSecret = secretClient.gcs().getGCSSecret(GCSSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
- }
-
- HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
- JsonFactory jsonFactory = new JacksonFactory();
- String jsonString = gcsSecret.getCredentialsJson();
- GoogleCredential credential = GoogleCredential.fromStream(new ByteArrayInputStream(jsonString.getBytes(StandardCharsets.UTF_8)), transport, jsonFactory);
- if (credential.createScopedRequired()) {
- Collection<String> scopes = StorageScopes.all();
- credential = credential.createScoped(scopes);
+ PrivateKey privKey = GCSUtil.getPrivateKey(gcsSecret.getPrivateKey());
+
+ try (Storage storage = StorageOptions.newBuilder().setCredentials(ServiceAccountCredentials.newBuilder()
+ .setProjectId(gcsSecret.getProjectId())
+ .setPrivateKey(privKey)
+ .setClientEmail(gcsSecret.getClientEmail())
+ .build()).build().getService()) {
+
+ Blob blob = storage.get(gcsStorage.getBucketName(),
+ resourcePath, Storage.BlobGetOption.fields(Storage.BlobField.values()));
+
+
+ ResourceMetadata.Builder resourceBuilder = ResourceMetadata.newBuilder();
+ if (blob != null) {
+ FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
+ fileBuilder.setFriendlyName(blob.getName());
+ fileBuilder.setResourcePath(resourcePath);
+ fileBuilder.setCreatedTime(blob.getCreateTimeOffsetDateTime().getLong(ChronoField.INSTANT_SECONDS));
+ fileBuilder.setUpdateTime(blob.getUpdateTimeOffsetDateTime().getLong(ChronoField.INSTANT_SECONDS));
+ fileBuilder.setResourceSize(blob.getSize());
+ fileBuilder.setMd5Sum(blob.getMd5());
+ resourceBuilder.setFile(fileBuilder);
+ } else {
+ final String dirPath = resourcePath.endsWith("/") ? resourcePath : resourcePath + "/";
+
+ DirectoryMetadata.Builder dirBuilder = DirectoryMetadata.newBuilder();
+
+ try {
+
+ Page<Blob> blobs = storage.list(gcsStorage.getBucketName(),
+ Storage.BlobListOption.currentDirectory(),
+ Storage.BlobListOption.prefix(dirPath));
+ Iterable<Blob> blobIter = blobs.iterateAll();
+ blobIter.forEach(b -> {
+ if (b.isDirectory()) {
+ DirectoryMetadata.Builder subDirBuilder = DirectoryMetadata.newBuilder();
+ subDirBuilder.setCreatedTime(b.getCreateTimeOffsetDateTime().getLong(ChronoField.INSTANT_SECONDS));
+ subDirBuilder.setUpdateTime(b.getUpdateTimeOffsetDateTime().getLong(ChronoField.INSTANT_SECONDS));
+ dirBuilder.addDirectories(subDirBuilder);
+ } else {
+ FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
+ fileBuilder.setFriendlyName(b.getName());
+ fileBuilder.setResourcePath(dirPath + b.getName());
+ fileBuilder.setCreatedTime(b.getCreateTimeOffsetDateTime().getLong(ChronoField.INSTANT_SECONDS));
+ fileBuilder.setUpdateTime(b.getUpdateTimeOffsetDateTime().getLong(ChronoField.INSTANT_SECONDS));
+ fileBuilder.setResourceSize(b.getSize());
+ fileBuilder.setMd5Sum(b.getMd5());
+ dirBuilder.addFiles(fileBuilder);
+ }
+ });
+
+
+ resourceBuilder.setDirectory(dirBuilder);
+ } catch (Exception e) {
+ resourceBuilder.setError(MetadataFetchError.NOT_FOUND);
+ }
+ }
+ return resourceBuilder.build();
}
-
- Storage storage = new Storage.Builder(transport, jsonFactory, credential).build();
-
- FileResourceMetadata metadata = new FileResourceMetadata();
- StorageObject gcsMetadata = storage.objects().get(gcsStorage.getBucketName(), resourcePath).execute();
-
- metadata.setResourceSize(gcsMetadata.getSize().longValue());
- String md5Sum = String.format("%032x", new BigInteger(1, Base64.getDecoder().decode(gcsMetadata.getMd5Hash())));
- metadata.setMd5sum(md5Sum);
- metadata.setUpdateTime(gcsMetadata.getTimeStorageClassUpdated().getValue());
- metadata.setCreatedTime(gcsMetadata.getTimeCreated().getValue());
- return metadata;
}
@Override
- public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthToken authZToken, String resourcePath,
- String storageId, String credentialToken) throws Exception {
- throw new UnsupportedOperationException("Method not implemented");
- }
-
- @Override
- public Boolean isAvailable(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception {
+ public Boolean isAvailable(String resourcePath) throws Exception {
checkInitialized();
-
- GCSStorage gcsStorage;
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(resourceServiceHost, resourceServicePort)) {
-
- gcsStorage = storageServiceClient.gcs()
- .getGCSStorage(GCSStorageGetRequest.newBuilder().setStorageId(storageId).build());
+ PrivateKey privKey = GCSUtil.getPrivateKey(gcsSecret.getPrivateKey());
+
+ try (Storage storage = StorageOptions.newBuilder().setCredentials(ServiceAccountCredentials.newBuilder()
+ .setProjectId(gcsSecret.getProjectId())
+ .setPrivateKey(privKey)
+ .setClientEmail(gcsSecret.getClientEmail())
+ .build()).build().getService()) {
+
+ Blob blob = storage.get(gcsStorage.getBucketName(),
+ resourcePath, Storage.BlobGetOption.fields(Storage.BlobField.values()));
+
+ if (blob != null) {
+ return true;
+ } else {
+ final String dirPath = resourcePath.endsWith("/") ? resourcePath : resourcePath + "/";
+ try {
+
+ Page<Blob> blobs = storage.list(gcsStorage.getBucketName(),
+ Storage.BlobListOption.currentDirectory(),
+ Storage.BlobListOption.prefix(dirPath));
+
+ return true;
+ } catch (Exception e) {
+ return false;
+ }
+ }
}
-
- GCSSecret gcsSecret;
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- secretServiceHost, secretServicePort)) {
- gcsSecret = secretClient.gcs().getGCSSecret(GCSSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
- }
-
- HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
- JsonFactory jsonFactory = new JacksonFactory();
- String jsonString = gcsSecret.getCredentialsJson();
- GoogleCredential credential = GoogleCredential.fromStream(new ByteArrayInputStream(jsonString.getBytes(StandardCharsets.UTF_8)), transport, jsonFactory);
- if (credential.createScopedRequired()) {
- Collection<String> scopes = StorageScopes.all();
- credential = credential.createScoped(scopes);
- }
-
- Storage storage = new Storage.Builder(transport, jsonFactory, credential).build();
- return !storage.objects().get(gcsStorage.getBucketName(), resourcePath).execute().isEmpty();
}
}
diff --git a/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSOutgoingStreamingConnector.java b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSOutgoingStreamingConnector.java
new file mode 100644
index 0000000..ffd600a
--- /dev/null
+++ b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSOutgoingStreamingConnector.java
@@ -0,0 +1,78 @@
+package org.apache.airavata.mft.transport.gcp;
+
+import com.google.auth.oauth2.ServiceAccountCredentials;
+import com.google.cloud.WriteChannel;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageOptions;
+import org.apache.airavata.mft.core.api.ConnectorConfig;
+import org.apache.airavata.mft.core.api.OutgoingStreamingConnector;
+import org.apache.airavata.mft.credential.stubs.gcs.GCSSecret;
+import org.apache.airavata.mft.resource.stubs.gcs.storage.GCSStorage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.OutputStream;
+import java.nio.channels.Channel;
+import java.nio.channels.Channels;
+import java.security.PrivateKey;
+
+public class GCSOutgoingStreamingConnector implements OutgoingStreamingConnector {
+
+ private static final Logger logger = LoggerFactory.getLogger(GCSOutgoingStreamingConnector.class);
+
+ private Storage storage;
+ private BlobInfo blobInfo;
+ private ConnectorConfig connectorConfig;
+ @Override
+ public void init(ConnectorConfig connectorConfig) throws Exception {
+
+ this.connectorConfig = connectorConfig;
+ GCSSecret gcsSecret = connectorConfig.getSecret().getGcs();
+
+ GCSStorage gcsStorage = connectorConfig.getStorage().getGcs();
+
+ PrivateKey privKey = GCSUtil.getPrivateKey(gcsSecret.getPrivateKey());
+
+ storage = StorageOptions.newBuilder().setCredentials(ServiceAccountCredentials.newBuilder()
+ .setProjectId(gcsSecret.getProjectId())
+ .setPrivateKey(privKey)
+ .setClientEmail(gcsSecret.getClientEmail())
+ .build()).build().getService();
+
+ BlobId blobId = BlobId.of(gcsStorage.getBucketName(), connectorConfig.getResourcePath());
+ blobInfo = BlobInfo.newBuilder(blobId).build();
+
+ }
+
+ @Override
+ public void complete() throws Exception {
+ if (storage != null) {
+ try {
+ this.storage.close();
+ } catch (Exception e) {
+ // Ignore
+ }
+ }
+ logger.info("File {} successfully sent", connectorConfig.getResourcePath());
+ }
+
+ @Override
+ public void failed() throws Exception {
+ if (storage != null) {
+ try {
+ this.storage.close();
+ } catch (Exception e) {
+ // Ignore
+ }
+ }
+ logger.info("File {} failed to send", connectorConfig.getResourcePath());
+ }
+
+ @Override
+ public OutputStream fetchOutputStream() throws Exception {
+ WriteChannel writer = storage.writer(blobInfo);
+ return Channels.newOutputStream(writer);
+ }
+}
diff --git a/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSReceiver.java b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSReceiver.java
deleted file mode 100644
index 15ade66..0000000
--- a/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSReceiver.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.airavata.mft.transport.gcp;
-
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
-import com.google.api.client.http.HttpTransport;
-import com.google.api.client.json.JsonFactory;
-import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.services.storage.Storage;
-import com.google.api.services.storage.StorageScopes;
-import org.apache.airavata.mft.common.AuthToken;
-import org.apache.airavata.mft.core.ConnectorContext;
-import org.apache.airavata.mft.core.api.Connector;
-import org.apache.airavata.mft.credential.stubs.gcs.GCSSecret;
-import org.apache.airavata.mft.credential.stubs.gcs.GCSSecretGetRequest;
-import org.apache.airavata.mft.resource.client.ResourceServiceClient;
-import org.apache.airavata.mft.resource.client.ResourceServiceClientBuilder;
-import org.apache.airavata.mft.resource.stubs.common.GenericResource;
-import org.apache.airavata.mft.resource.stubs.common.GenericResourceGetRequest;
-import org.apache.airavata.mft.resource.stubs.gcs.storage.GCSStorage;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.Collection;
-
-
-public class GCSReceiver implements Connector {
-
- private static final Logger logger = LoggerFactory.getLogger(GCSReceiver.class);
-
- private Storage storage;
-
- private String resourceServiceHost;
- private int resourceServicePort;
- private String secretServiceHost;
- private int secretServicePort;
-
- @Override
- public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception {
-
- this.resourceServiceHost = resourceServiceHost;
- this.resourceServicePort = resourceServicePort;
- this.secretServiceHost = secretServiceHost;
- this.secretServicePort = secretServicePort;
- }
-
- @Override
- public void destroy() {
-
- }
-
- @Override
- public void startStream(AuthToken authToken, String resourceId, String credentialToken, ConnectorContext context) throws Exception {
- logger.info("Starting GCS Receiver stream for transfer {}", context.getTransferId());
-
- ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort);
- GenericResource resource = resourceClient.get().getGenericResource(GenericResourceGetRequest.newBuilder()
- .setResourceId(resourceId).build());
-
- if (resource.getStorageCase() != GenericResource.StorageCase.GCSSTORAGE) {
- logger.error("Invalid storage type {} specified for resource {}", resource.getStorageCase(), resourceId);
- throw new Exception("Invalid storage type specified for resource " + resourceId);
- }
-
- GCSStorage gcsStorage = resource.getGcsStorage();
-
- SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort);
- GCSSecret gcsSecret = secretClient.gcs().getGCSSecret(GCSSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
-
- HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
- JsonFactory jsonFactory = new JacksonFactory();
- String jsonString = gcsSecret.getCredentialsJson();
- GoogleCredential credential = GoogleCredential.fromStream(new ByteArrayInputStream(jsonString.getBytes(StandardCharsets.UTF_8)));
- if (credential.createScopedRequired()) {
- Collection<String> scopes = StorageScopes.all();
- credential = credential.createScoped(scopes);
- }
- storage = new Storage.Builder(transport, jsonFactory, credential).build();
-
- InputStream inputStream = storage.objects().get(gcsStorage.getBucketName(),
- resource.getFile().getResourcePath()).executeMediaAsInputStream();
-
- OutputStream os = context.getStreamBuffer().getOutputStream();
- int read;
- long bytes = 0;
- long fileSize = context.getMetadata().getResourceSize();
- byte[] buf = new byte[1024];
- while (true) {
- int bufSize = 0;
-
- if (buf.length < fileSize) {
- bufSize = buf.length;
- } else {
- bufSize = (int) fileSize;
- }
- bufSize = inputStream.read(buf, 0, bufSize);
-
- if (bufSize < 0) {
- break;
- }
-
- os.write(buf, 0, bufSize);
- os.flush();
-
- fileSize -= bufSize;
- if (fileSize == 0L)
- break;
- }
-
- os.close();
-
- logger.info("Completed GCS Receiver stream for transfer {}", context.getTransferId());
-
- }
-
- @Override
- public void startStream(AuthToken authToken, String resourceId, String childResourcePath, String credentialToken,
- ConnectorContext context) throws Exception {
- throw new UnsupportedOperationException();
- }
-}
diff --git a/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSSender.java b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSSender.java
deleted file mode 100644
index 7e91fb2..0000000
--- a/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSSender.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.airavata.mft.transport.gcp;
-
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
-import com.google.api.client.http.HttpTransport;
-import com.google.api.client.http.InputStreamContent;
-import com.google.api.client.json.JsonFactory;
-import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.services.storage.Storage;
-import com.google.api.services.storage.Storage.Objects.Insert;
-import com.google.api.services.storage.StorageScopes;
-import com.google.api.services.storage.model.ObjectAccessControl;
-import com.google.api.services.storage.model.StorageObject;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import org.apache.airavata.mft.common.AuthToken;
-import org.apache.airavata.mft.core.ConnectorContext;
-import org.apache.airavata.mft.core.api.Connector;
-import org.apache.airavata.mft.credential.stubs.gcs.GCSSecret;
-import org.apache.airavata.mft.credential.stubs.gcs.GCSSecretGetRequest;
-import org.apache.airavata.mft.resource.client.ResourceServiceClient;
-import org.apache.airavata.mft.resource.client.ResourceServiceClientBuilder;
-import org.apache.airavata.mft.resource.stubs.common.GenericResource;
-import org.apache.airavata.mft.resource.stubs.common.GenericResourceGetRequest;
-import org.apache.airavata.mft.resource.stubs.gcs.storage.GCSStorage;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayInputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.Collection;
-
-
-public class GCSSender implements Connector {
-
- private static final Logger logger = LoggerFactory.getLogger(GCSSender.class);
-
- private Storage storage;
- private JsonObject jsonObject;
-
- private String resourceServiceHost;
- private int resourceServicePort;
- private String secretServiceHost;
- private int secretServicePort;
-
- @Override
- public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception {
-
- this.resourceServiceHost = resourceServiceHost;
- this.resourceServicePort = resourceServicePort;
- this.secretServiceHost = secretServiceHost;
- this.secretServicePort = secretServicePort;
- }
-
-
- @Override
- public void destroy() {
-
- }
-
- @Override
- public void startStream(AuthToken authToken, String resourceId, String credentialToken, ConnectorContext context) throws Exception {
- logger.info("Starting GCS Sender stream for transfer {}", context.getTransferId());
- logger.debug("Content length for transfer {} {}", context.getTransferId(), context.getMetadata().getResourceSize());
-
- ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort);
- GenericResource resource = resourceClient.get().getGenericResource(GenericResourceGetRequest.newBuilder()
- .setResourceId(resourceId).build());
-
- if (resource.getStorageCase() != GenericResource.StorageCase.GCSSTORAGE) {
- logger.error("Invalid storage type {} specified for resource {}", resource.getStorageCase(), resourceId);
- throw new Exception("Invalid storage type specified for resource " + resourceId);
- }
-
- GCSStorage gcsStorage = resource.getGcsStorage();
-
- SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort);
- GCSSecret gcsSecret = secretClient.gcs().getGCSSecret(GCSSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
-
- HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
- JsonFactory jsonFactory = new JacksonFactory();
- String jsonString = gcsSecret.getCredentialsJson();
- jsonObject = new JsonParser().parse(jsonString).getAsJsonObject();
- GoogleCredential credential = GoogleCredential.fromStream(new ByteArrayInputStream(jsonString.getBytes(StandardCharsets.UTF_8)));
- if (credential.createScopedRequired()) {
- Collection<String> scopes = StorageScopes.all();
- credential = credential.createScoped(scopes);
- }
-
- storage = new Storage.Builder(transport, jsonFactory, credential).build();
-
- InputStreamContent contentStream = new InputStreamContent(
- null, context.getStreamBuffer().getInputStream());
- String entityUser = jsonObject.get("client_email").getAsString();
- StorageObject objectMetadata = new StorageObject()
- // Set the destination object name
- .setName(resource.getFile().getResourcePath())
- // Set the access control list to publicly read-only
- .setAcl(Arrays.asList(new ObjectAccessControl().setEntity("user-" + entityUser).setRole("OWNER")));
-
- Insert insertRequest = storage.objects().insert(gcsStorage.getBucketName(), objectMetadata, contentStream);
-
- insertRequest.execute();
-
- logger.info("Completed GCS Sender stream for transfer {}", context.getTransferId());
- }
-
- @Override
- public void startStream(AuthToken authToken, String resourceId, String childResourcePath, String credentialToken,
- ConnectorContext context) throws Exception {
- throw new UnsupportedOperationException();
- }
-
-
-}
diff --git a/controller/src/main/java/org/apache/airavata/mft/controller/AppConfig.java b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSUtil.java
similarity index 55%
copy from controller/src/main/java/org/apache/airavata/mft/controller/AppConfig.java
copy to transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSUtil.java
index 203b50b..a5770d2 100644
--- a/controller/src/main/java/org/apache/airavata/mft/controller/AppConfig.java
+++ b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSUtil.java
@@ -15,23 +15,19 @@
* limitations under the License.
*/
-package org.apache.airavata.mft.controller;
+package org.apache.airavata.mft.transport.gcp;
-import org.apache.airavata.mft.admin.MFTConsulClient;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
+import java.security.KeyFactory;
+import java.security.PrivateKey;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.util.Base64;
-@Configuration
-public class AppConfig {
+public class GCSUtil {
+ static PrivateKey getPrivateKey(String privateKeyContent) throws Exception {
+ privateKeyContent = privateKeyContent.replaceAll("\\n", "").replace("-----BEGIN PRIVATE KEY-----", "").replace("-----END PRIVATE KEY-----", "");
+ KeyFactory kf = KeyFactory.getInstance("RSA");
- @org.springframework.beans.factory.annotation.Value("${consul.host}")
- String consulHost;
-
- @org.springframework.beans.factory.annotation.Value("${consul.port}")
- Integer consulPort;
-
- @Bean
- public MFTConsulClient mftConsulClient() {
- return new MFTConsulClient(consulHost, consulPort);
+ PKCS8EncodedKeySpec keySpecPKCS8 = new PKCS8EncodedKeySpec(Base64.getDecoder().decode(privateKeyContent));
+ return kf.generatePrivate(keySpecPKCS8);
}
}
diff --git a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalMetadataCollector.java b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalMetadataCollector.java
index 23aa05c..e6a6f1b 100644
--- a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalMetadataCollector.java
+++ b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalMetadataCollector.java
@@ -17,17 +17,15 @@
package org.apache.airavata.mft.transport.local;
-import org.apache.airavata.mft.common.AuthToken;
-import org.apache.airavata.mft.core.DirectoryResourceMetadata;
-import org.apache.airavata.mft.core.FileResourceMetadata;
+import org.apache.airavata.mft.agent.stub.*;
import org.apache.airavata.mft.core.api.MetadataCollector;
+import org.apache.airavata.mft.resource.stubs.local.storage.LocalStorage;
import java.io.File;
-import java.io.FileInputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
-import java.security.MessageDigest;
+import java.util.stream.Stream;
public class LocalMetadataCollector implements MetadataCollector {
@@ -37,12 +35,10 @@ public class LocalMetadataCollector implements MetadataCollector {
private int secretServicePort;
boolean initialized = false;
+ private LocalStorage localStorage;
@Override
- public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) {
- this.resourceServiceHost = resourceServiceHost;
- this.resourceServicePort = resourceServicePort;
- this.secretServiceHost = secretServiceHost;
- this.secretServicePort = secretServicePort;
+ public void init(StorageWrapper storage, SecretWrapper secret) {
+ this.localStorage = storage.getLocal();
this.initialized = true;
}
@@ -52,45 +48,72 @@ public class LocalMetadataCollector implements MetadataCollector {
}
}
+ private FileMetadata.Builder getFileBuilderFromPath(File file) throws Exception {
+
+ BasicFileAttributes basicFileAttributes = Files.readAttributes(Path.of(file.getPath()), BasicFileAttributes.class);
+ FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
+ fileBuilder.setCreatedTime(basicFileAttributes.creationTime().toMillis());
+ fileBuilder.setUpdateTime(basicFileAttributes.lastModifiedTime().toMillis());
+ fileBuilder.setResourceSize(basicFileAttributes.size());
+
+ fileBuilder.setResourcePath(file.getPath());
+ fileBuilder.setFriendlyName(file.getName());
+
+ return fileBuilder;
+ }
@Override
- public FileResourceMetadata getFileResourceMetadata(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception {
+ public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception {
+
+ checkInitialized();
+ ResourceMetadata.Builder resouceBuilder = ResourceMetadata.newBuilder();
File resourceFile = new File(resourcePath);
if (resourceFile.exists()) {
- BasicFileAttributes basicFileAttributes = Files.readAttributes(Path.of(resourcePath), BasicFileAttributes.class);
- FileResourceMetadata metadata = new FileResourceMetadata();
- metadata.setCreatedTime(basicFileAttributes.creationTime().toMillis());
- metadata.setUpdateTime(basicFileAttributes.lastModifiedTime().toMillis());
- metadata.setResourceSize(basicFileAttributes.size());
-
- MessageDigest digest = MessageDigest.getInstance("MD5");
- FileInputStream fis = new FileInputStream(resourcePath);
- byte[] byteArray = new byte[1024];
- int bytesCount = 0;
- while ((bytesCount = fis.read(byteArray)) != -1) {
- digest.update(byteArray, 0, bytesCount);
- };
- fis.close();
- byte[] bytes = digest.digest();
- StringBuilder sb = new StringBuilder();
- for (byte aByte : bytes) {
- sb.append(Integer.toString((aByte & 0xff) + 0x100, 16).substring(1));
- }
- metadata.setMd5sum(sb.toString());
+ if (resourceFile.isFile()) {
+ FileMetadata.Builder fileBuilder = getFileBuilderFromPath(resourceFile);
+ resouceBuilder.setFile(fileBuilder);
- return metadata;
+ } else if (resourceFile.isDirectory()) {
+ DirectoryMetadata.Builder dirBuilder = DirectoryMetadata.newBuilder();
+ BasicFileAttributes basicFileAttributes = Files.readAttributes(Path.of(resourcePath), BasicFileAttributes.class);
+ dirBuilder.setCreatedTime(basicFileAttributes.creationTime().toMillis());
+ dirBuilder.setUpdateTime(basicFileAttributes.lastModifiedTime().toMillis());
+
+ Stream<Path> fileList = Files.list(resourceFile.toPath());
+ fileList.forEach(p -> {
+ try {
+ if (p.toFile().isFile()) {
+
+ FileMetadata.Builder fileBuilder = getFileBuilderFromPath(resourceFile);
+ dirBuilder.addFiles(fileBuilder);
+ } else if (p.toFile().isDirectory()) {
+ DirectoryMetadata.Builder subDirBuilder = DirectoryMetadata.newBuilder();
+ BasicFileAttributes bfa = Files.readAttributes(p, BasicFileAttributes.class);
+ subDirBuilder.setCreatedTime(bfa.creationTime().toMillis());
+ subDirBuilder.setUpdateTime(bfa.lastModifiedTime().toMillis());
+ dirBuilder.addDirectories(subDirBuilder);
+ }
+ } catch (Exception e) {
+ // Ignore
+ }
+ });
+
+ resouceBuilder.setDirectory(dirBuilder);
+ } else {
+ resouceBuilder.setError(MetadataFetchError.UNRECOGNIZED);
+ }
} else {
- throw new Exception("Resource with path " + resourcePath + " does not exist");
+ resouceBuilder.setError(MetadataFetchError.NOT_FOUND);
}
+
+ return resouceBuilder.build();
}
@Override
- public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception {
- throw new UnsupportedOperationException("Method not implemented"); }
+ public Boolean isAvailable(String resourcePath) throws Exception {
- @Override
- public Boolean isAvailable(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception {
+ checkInitialized();
return new File(resourcePath).exists();
}
}
diff --git a/transport/odata-transport/src/main/java/org/apache/airavata/mft/transport/odata/ODataIncomingConnector.java b/transport/odata-transport/src/main/java/org/apache/airavata/mft/transport/odata/ODataIncomingConnector.java
index 55e6fb2..118edf2 100644
--- a/transport/odata-transport/src/main/java/org/apache/airavata/mft/transport/odata/ODataIncomingConnector.java
+++ b/transport/odata-transport/src/main/java/org/apache/airavata/mft/transport/odata/ODataIncomingConnector.java
@@ -57,29 +57,17 @@ public class ODataIncomingConnector implements IncomingStreamingConnector {
@Override
public void init(ConnectorConfig cc) throws Exception {
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(cc.getResourceServiceHost(), cc.getResourceServicePort())) {
-
- odataStorage = storageServiceClient.odata()
- .getODataStorage(ODataStorageGetRequest.newBuilder().setStorageId(cc.getStorageId()).build());
- }
this.resourcePath = cc.getResourcePath();
+ odataStorage = cc.getStorage().getOdata();
+ ODataSecret oDataSecret = cc.getSecret().getOdata();
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- cc.getSecretServiceHost(), cc.getSecretServicePort())) {
-
- ODataSecret oDataSecret = secretClient.odata().getODataSecret(ODataSecretGetRequest.newBuilder()
- .setAuthzToken(cc.getAuthToken())
- .setSecretId(cc.getCredentialToken()).build());
+ CredentialsProvider provider = new BasicCredentialsProvider();
+ UsernamePasswordCredentials credentials
+ = new UsernamePasswordCredentials(oDataSecret.getUserName(), oDataSecret.getPassword());
+ provider.setCredentials(AuthScope.ANY, credentials);
- CredentialsProvider provider = new BasicCredentialsProvider();
- UsernamePasswordCredentials credentials
- = new UsernamePasswordCredentials(oDataSecret.getUserName(), oDataSecret.getPassword());
- provider.setCredentials(AuthScope.ANY, credentials);
-
- client = HttpClientBuilder.create().setDefaultCredentialsProvider(provider).build();
- }
+ client = HttpClientBuilder.create().setDefaultCredentialsProvider(provider).build();
}
@Override
diff --git a/transport/odata-transport/src/main/java/org/apache/airavata/mft/transport/odata/ODataMetadataCollector.java b/transport/odata-transport/src/main/java/org/apache/airavata/mft/transport/odata/ODataMetadataCollector.java
index a0deb54..818df45 100644
--- a/transport/odata-transport/src/main/java/org/apache/airavata/mft/transport/odata/ODataMetadataCollector.java
+++ b/transport/odata-transport/src/main/java/org/apache/airavata/mft/transport/odata/ODataMetadataCollector.java
@@ -17,18 +17,10 @@
package org.apache.airavata.mft.transport.odata;
-import org.apache.airavata.mft.common.AuthToken;
-import org.apache.airavata.mft.core.DirectoryResourceMetadata;
-import org.apache.airavata.mft.core.FileResourceMetadata;
+import org.apache.airavata.mft.agent.stub.*;
import org.apache.airavata.mft.core.api.MetadataCollector;
import org.apache.airavata.mft.credential.stubs.odata.ODataSecret;
-import org.apache.airavata.mft.credential.stubs.odata.ODataSecretGetRequest;
-import org.apache.airavata.mft.resource.client.StorageServiceClient;
-import org.apache.airavata.mft.resource.client.StorageServiceClientBuilder;
import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorage;
-import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageGetRequest;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
import org.apache.http.HttpEntity;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
@@ -55,18 +47,12 @@ import java.util.Optional;
public class ODataMetadataCollector implements MetadataCollector {
private static final Logger logger = LoggerFactory.getLogger(ODataMetadataCollector.class);
-
- private String resourceServiceHost;
- private int resourceServicePort;
- private String secretServiceHost;
- private int secretServicePort;
-
+ private ODataStorage oDataStorage;
+ private ODataSecret oDataSecret;
@Override
- public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) {
- this.resourceServiceHost = resourceServiceHost;
- this.resourceServicePort = resourceServicePort;
- this.secretServiceHost = secretServiceHost;
- this.secretServicePort = secretServicePort;
+ public void init(StorageWrapper storage, SecretWrapper secret) {
+ oDataStorage = storage.getOdata();
+ oDataSecret = secret.getOdata();
}
private CloseableHttpClient getHttpClient(ODataSecret oDataSecret) {
@@ -79,45 +65,27 @@ public class ODataMetadataCollector implements MetadataCollector {
}
@Override
- public FileResourceMetadata getFileResourceMetadata(AuthToken authZToken, String resourcePath, String storageId,
- String credentialToken) throws Exception {
- return findFileResourceMetadata(authZToken, resourcePath, storageId, credentialToken)
- .orElseThrow(() -> new Exception("Could not find a file resource entry for resource path " + resourcePath));
- }
-
-
- @Override
- public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthToken authZToken, String resourcePath,
- String storageId, String credentialToken) throws Exception {
- throw new UnsupportedOperationException("OData does not have directory structures");
+ public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception {
+ ResourceMetadata.Builder resourceBuilder = ResourceMetadata.newBuilder();
+ Optional<FileMetadata> fileResourceMetadata = findFileResourceMetadata(resourcePath);
+ if (fileResourceMetadata.isPresent()) {
+ resourceBuilder.setFile(fileResourceMetadata.get());
+ } else {
+ resourceBuilder.setError(MetadataFetchError.NOT_FOUND);
+ }
+ return resourceBuilder.build();
}
@Override
- public Boolean isAvailable(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception {
- return findFileResourceMetadata(authZToken, resourcePath, storageId, credentialToken).isPresent();
+ public Boolean isAvailable(String resourcePath) throws Exception {
+ return findFileResourceMetadata(resourcePath).isPresent();
}
- private Optional<FileResourceMetadata> findFileResourceMetadata(AuthToken authZToken, String resourcePath,
- String storageId, String credentialToken) throws Exception {
-
- ODataStorage odataStorage;
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(resourceServiceHost, resourceServicePort)) {
-
- odataStorage = storageServiceClient.odata()
- .getODataStorage(ODataStorageGetRequest.newBuilder().setStorageId(storageId).build());
- }
-
- ODataSecret oDataSecret;
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- secretServiceHost, secretServicePort)) {
- oDataSecret = secretClient.odata().getODataSecret(
- ODataSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
- }
+ private Optional<FileMetadata> findFileResourceMetadata(String resourcePath) throws Exception {
try (CloseableHttpClient httpClient = getHttpClient(oDataSecret)) {
- HttpGet httpGet = new HttpGet(odataStorage.getBaseUrl() +
+ HttpGet httpGet = new HttpGet(oDataStorage.getBaseUrl() +
"/Products('" + resourcePath +"')");
try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
@@ -133,7 +101,7 @@ public class ODataMetadataCollector implements MetadataCollector {
}
}
- private Optional<FileResourceMetadata> parseXML(String xmlBody) {
+ private Optional<FileMetadata> parseXML(String xmlBody) {
try {
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
@@ -147,7 +115,7 @@ public class ODataMetadataCollector implements MetadataCollector {
if (properties.getLength() == 1) {
- FileResourceMetadata.Builder builder = FileResourceMetadata.Builder.newBuilder();
+ FileMetadata.Builder builder = FileMetadata.newBuilder();
Node propertyNode = properties.item(0);
NodeList childNodes = propertyNode.getChildNodes();
@@ -155,26 +123,26 @@ public class ODataMetadataCollector implements MetadataCollector {
Node item = childNodes.item(i);
switch (item.getNodeName()) {
case "d:ContentLength":
- builder.withResourceSize(Long.parseLong(item.getTextContent()));
+ builder.setResourceSize(Long.parseLong(item.getTextContent()));
break;
case "d:CreationDate":
- builder.withCreatedTime(Instant.parse(item.getTextContent() + "Z").toEpochMilli());
+ builder.setCreatedTime(Instant.parse(item.getTextContent() + "Z").toEpochMilli());
break;
case "d:ModificationDate":
- builder.withUpdateTime(Instant.parse(item.getTextContent() + "Z").toEpochMilli());
+ builder.setUpdateTime(Instant.parse(item.getTextContent() + "Z").toEpochMilli());
break;
case "d:Name":
- builder.withFriendlyName(item.getTextContent());
+ builder.setFriendlyName(item.getTextContent());
break;
case "d:Id":
- builder.withResourcePath(item.getTextContent());
+ builder.setResourcePath(item.getTextContent());
break;
case "d:Checksum":
NodeList checksumNodes = item.getChildNodes();
for (int j = 0; j < checksumNodes.getLength(); j++) {
Node item1 = checksumNodes.item(j);
if (item1.getNodeName().equals("d:Value")) {
- builder.withMd5sum(item1.getTextContent());
+ builder.setMd5Sum(item1.getTextContent());
}
}
break;
diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3IncomingConnector.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3IncomingConnector.java
index 0d92054..3cc9e46 100644
--- a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3IncomingConnector.java
+++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3IncomingConnector.java
@@ -43,39 +43,26 @@ public class S3IncomingConnector implements IncomingChunkedConnector, IncomingSt
@Override
public void init(ConnectorConfig cc) throws Exception {
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(cc.getResourceServiceHost(), cc.getResourceServicePort())) {
+ s3Storage = cc.getStorage().getS3();
+ this.resourcePath = cc.getResourcePath();
- s3Storage = storageServiceClient.s3()
- .getS3Storage(S3StorageGetRequest.newBuilder().setStorageId(cc.getStorageId()).build());
- }
+ S3Secret s3Secret = cc.getSecret().getS3();
- this.resourcePath = cc.getResourcePath();
+ AWSCredentials awsCreds;
- S3Secret s3Secret;
-
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- cc.getSecretServiceHost(), cc.getSecretServicePort())) {
-
- s3Secret = secretClient.s3().getS3Secret(S3SecretGetRequest.newBuilder()
- .setAuthzToken(cc.getAuthToken())
- .setSecretId(cc.getCredentialToken()).build());
-
- AWSCredentials awsCreds;
- if (s3Secret.getSessionToken() == null || s3Secret.getSessionToken().equals("")) {
- awsCreds = new BasicAWSCredentials(s3Secret.getAccessKey(), s3Secret.getSecretKey());
- } else {
- awsCreds = new BasicSessionCredentials(s3Secret.getAccessKey(),
- s3Secret.getSecretKey(),
- s3Secret.getSessionToken());
- }
-
- s3Client = AmazonS3ClientBuilder.standard()
- .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
- s3Storage.getEndpoint(), s3Storage.getRegion()))
- .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
- .build();
+ if (s3Secret.getSessionToken() == null || s3Secret.getSessionToken().equals("")) {
+ awsCreds = new BasicAWSCredentials(s3Secret.getAccessKey(), s3Secret.getSecretKey());
+ } else {
+ awsCreds = new BasicSessionCredentials(s3Secret.getAccessKey(),
+ s3Secret.getSecretKey(),
+ s3Secret.getSessionToken());
}
+
+ s3Client = AmazonS3ClientBuilder.standard()
+ .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
+ s3Storage.getEndpoint(), s3Storage.getRegion()))
+ .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
+ .build();
}
diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3MetadataCollector.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3MetadataCollector.java
index 71c625d..5f2d9b9 100644
--- a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3MetadataCollector.java
+++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3MetadataCollector.java
@@ -24,34 +24,27 @@ import com.amazonaws.auth.BasicSessionCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
-import org.apache.airavata.mft.common.AuthToken;
-import org.apache.airavata.mft.core.DirectoryResourceMetadata;
-import org.apache.airavata.mft.core.FileResourceMetadata;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import org.apache.airavata.mft.agent.stub.*;
import org.apache.airavata.mft.core.api.MetadataCollector;
import org.apache.airavata.mft.credential.stubs.s3.S3Secret;
-import org.apache.airavata.mft.credential.stubs.s3.S3SecretGetRequest;
-import org.apache.airavata.mft.resource.client.StorageServiceClient;
-import org.apache.airavata.mft.resource.client.StorageServiceClientBuilder;
import org.apache.airavata.mft.resource.stubs.s3.storage.S3Storage;
-import org.apache.airavata.mft.resource.stubs.s3.storage.S3StorageGetRequest;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
+
+import java.io.File;
+import java.util.List;
public class S3MetadataCollector implements MetadataCollector {
- private String resourceServiceHost;
- private int resourceServicePort;
- private String secretServiceHost;
- private int secretServicePort;
boolean initialized = false;
+ private S3Storage s3Storage;
+ private S3Secret s3Secret;
@Override
- public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) {
- this.resourceServiceHost = resourceServiceHost;
- this.resourceServicePort = resourceServicePort;
- this.secretServiceHost = secretServiceHost;
- this.secretServicePort = secretServicePort;
+ public void init(StorageWrapper storage, SecretWrapper secret) {
+ this.s3Storage = storage.getS3();
+ this.s3Secret = secret.getS3();
this.initialized = true;
}
@@ -62,24 +55,10 @@ public class S3MetadataCollector implements MetadataCollector {
}
@Override
- public FileResourceMetadata getFileResourceMetadata(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception {
+ public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception {
checkInitialized();
- S3Storage s3Storage;
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(resourceServiceHost, resourceServicePort)) {
-
- s3Storage = storageServiceClient.s3()
- .getS3Storage(S3StorageGetRequest.newBuilder().setStorageId(storageId).build());
- }
-
- S3Secret s3Secret;
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- secretServiceHost, secretServicePort)) {
- s3Secret = secretClient.s3().getS3Secret(S3SecretGetRequest.newBuilder().setSecretId(credentialToken).build());
- }
-
AWSCredentials awsCreds;
if (s3Secret.getSessionToken() == null || s3Secret.getSessionToken().equals("")) {
awsCreds = new BasicAWSCredentials(s3Secret.getAccessKey(), s3Secret.getSecretKey());
@@ -96,39 +75,53 @@ public class S3MetadataCollector implements MetadataCollector {
.withCredentials(new AWSStaticCredentialsProvider(awsCreds))
.build();
- FileResourceMetadata metadata = new FileResourceMetadata();
- ObjectMetadata s3Metadata = s3Client.getObjectMetadata(s3Storage.getBucketName(), resourcePath);
- metadata.setResourceSize(s3Metadata.getContentLength());
- metadata.setMd5sum(s3Metadata.getETag());
- metadata.setUpdateTime(s3Metadata.getLastModified().getTime());
- metadata.setCreatedTime(s3Metadata.getLastModified().getTime());
- return metadata;
- }
-
- @Override
- public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthToken authZToken, String resourcePath, String storageId,
- String credentialToken) throws Exception {
- throw new UnsupportedOperationException("Method not implemented"); }
+ ResourceMetadata.Builder resourceBuilder = ResourceMetadata.newBuilder();
+
+ String key = s3Client.getObject(s3Storage.getBucketName(), resourcePath).getKey();
+
+ if (key.endsWith("/")) { // Folder
+
+ ObjectListing objectListing = s3Client.listObjects(s3Storage.getBucketName(), key);
+ List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
+ DirectoryMetadata.Builder dirBuilder = DirectoryMetadata.newBuilder();
+ for (S3ObjectSummary summary: objectSummaries) {
+ if (summary.getKey().endsWith("/")) {
+ DirectoryMetadata.Builder subDirBuilder = DirectoryMetadata.newBuilder();
+ subDirBuilder.setCreatedTime(summary.getLastModified().getTime());
+ subDirBuilder.setUpdateTime(summary.getLastModified().getTime());
+ subDirBuilder.setResourcePath(summary.getKey());
+ subDirBuilder.setFriendlyName(new File(summary.getKey()).getName());
+ dirBuilder.addDirectories(subDirBuilder);
+ } else {
+ FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
+ fileBuilder.setUpdateTime(summary.getLastModified().getTime());
+ fileBuilder.setCreatedTime(summary.getLastModified().getTime());
+ fileBuilder.setResourcePath(summary.getKey());
+ fileBuilder.setFriendlyName(new File(summary.getKey()).getName());
+ fileBuilder.setResourceSize(summary.getSize());
+ dirBuilder.addFiles(fileBuilder);
+ }
+ }
+ resourceBuilder.setDirectory(dirBuilder);
+ } else { // File
+ FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
+ ObjectMetadata fileMetadata = s3Client.getObjectMetadata(s3Storage.getBucketName(), resourcePath);
+ fileBuilder.setResourceSize(fileMetadata.getContentLength());
+ fileBuilder.setResourcePath(resourcePath);
+ fileBuilder.setMd5Sum(fileMetadata.getContentMD5());
+ fileBuilder.setFriendlyName(new File(resourcePath).getName());
+ fileBuilder.setCreatedTime(fileMetadata.getLastModified().getTime());
+ fileBuilder.setUpdateTime(fileMetadata.getLastModified().getTime());
+ resourceBuilder.setFile(fileBuilder);
+ }
+ return resourceBuilder.build();
+ }
@Override
- public Boolean isAvailable(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception {
+ public Boolean isAvailable(String resourcePath) throws Exception {
checkInitialized();
-
- S3Storage s3Storage;
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(resourceServiceHost, resourceServicePort)) {
-
- s3Storage = storageServiceClient.s3()
- .getS3Storage(S3StorageGetRequest.newBuilder().setStorageId(storageId).build());
- }
-
- S3Secret s3Secret;
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- secretServiceHost, secretServicePort)) {
- s3Secret = secretClient.s3().getS3Secret(S3SecretGetRequest.newBuilder().setSecretId(credentialToken).build());
- }
BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3Secret.getAccessKey(), s3Secret.getSecretKey());
AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java
index 1b89ca2..f6eebaf 100644
--- a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java
+++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java
@@ -43,38 +43,25 @@ public class S3OutgoingConnector implements OutgoingChunkedConnector {
this.resourcePath = cc.getResourcePath();
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(cc.getResourceServiceHost(), cc.getResourceServicePort())) {
+ s3Storage = cc.getStorage().getS3();
- s3Storage = storageServiceClient.s3()
- .getS3Storage(S3StorageGetRequest.newBuilder().setStorageId(cc.getStorageId()).build());
- }
+ S3Secret s3Secret = cc.getSecret().getS3();
- S3Secret s3Secret;
-
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- cc.getSecretServiceHost(), cc.getSecretServicePort())) {
-
- s3Secret = secretClient.s3().getS3Secret(S3SecretGetRequest.newBuilder()
- .setAuthzToken(cc.getAuthToken())
- .setSecretId(cc.getCredentialToken()).build());
-
- AWSCredentials awsCreds;
- if (s3Secret.getSessionToken() == null || s3Secret.getSessionToken().equals("")) {
- awsCreds = new BasicAWSCredentials(s3Secret.getAccessKey(), s3Secret.getSecretKey());
- } else {
- awsCreds = new BasicSessionCredentials(s3Secret.getAccessKey(),
- s3Secret.getSecretKey(),
- s3Secret.getSessionToken());
- }
-
- s3Client = AmazonS3ClientBuilder.standard()
- .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
- s3Storage.getEndpoint(), s3Storage.getRegion()))
- .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
- .build();
+ AWSCredentials awsCreds;
+ if (s3Secret.getSessionToken() == null || s3Secret.getSessionToken().equals("")) {
+ awsCreds = new BasicAWSCredentials(s3Secret.getAccessKey(), s3Secret.getSecretKey());
+ } else {
+ awsCreds = new BasicSessionCredentials(s3Secret.getAccessKey(),
+ s3Secret.getSecretKey(),
+ s3Secret.getSessionToken());
}
+ s3Client = AmazonS3ClientBuilder.standard()
+ .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
+ s3Storage.getEndpoint(), s3Storage.getRegion()))
+ .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
+ .build();
+
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(s3Storage.getBucketName(),
resourcePath);
initResponse = s3Client.initiateMultipartUpload(initRequest);
diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingStreamingConnector.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingStreamingConnector.java
index 34123a3..56b782d 100644
--- a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingStreamingConnector.java
+++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingStreamingConnector.java
@@ -62,41 +62,27 @@ public class S3OutgoingStreamingConnector implements OutgoingStreamingConnector
@Override
public void init(ConnectorConfig cc) throws Exception {
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(cc.getResourceServiceHost(), cc.getResourceServicePort())) {
-
- s3Storage = storageServiceClient.s3()
- .getS3Storage(S3StorageGetRequest.newBuilder().setStorageId(cc.getStorageId()).build());
- }
+ s3Storage = cc.getStorage().getS3();
this.resourcePath = cc.getResourcePath();
- S3Secret s3Secret;
-
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- cc.getSecretServiceHost(), cc.getSecretServicePort())) {
-
- s3Secret = secretClient.s3().getS3Secret(S3SecretGetRequest.newBuilder()
- .setAuthzToken(cc.getAuthToken())
- .setSecretId(cc.getCredentialToken()).build());
+ S3Secret s3Secret = cc.getSecret().getS3();
- AwsCredentials awsCreds;
- if (s3Secret.getSessionToken() == null || s3Secret.getSessionToken().equals("")) {
- awsCreds = AwsBasicCredentials.create(s3Secret.getAccessKey(), s3Secret.getSecretKey());
- } else {
- awsCreds = AwsSessionCredentials.create(s3Secret.getAccessKey(),
- s3Secret.getSecretKey(),
- s3Secret.getSessionToken());
- }
-
- S3Client s3Client = S3Client.builder()
- .region(Region.of(s3Storage.getRegion())).endpointOverride(new URI(s3Storage.getEndpoint()))
- .credentialsProvider(() -> awsCreds)
- .build();
+ AwsCredentials awsCreds;
+ if (s3Secret.getSessionToken() == null || s3Secret.getSessionToken().equals("")) {
+ awsCreds = AwsBasicCredentials.create(s3Secret.getAccessKey(), s3Secret.getSecretKey());
+ } else {
+ awsCreds = AwsSessionCredentials.create(s3Secret.getAccessKey(),
+ s3Secret.getSecretKey(),
+ s3Secret.getSessionToken());
+ }
- this.s3 = AwsS3ClientMultipartUpload.builder().s3(s3Client).build();
+ S3Client s3Client = S3Client.builder()
+ .region(Region.of(s3Storage.getRegion())).endpointOverride(new URI(s3Storage.getEndpoint()))
+ .credentialsProvider(() -> awsCreds)
+ .build();
- }
+ this.s3 = AwsS3ClientMultipartUpload.builder().s3(s3Client).build();
}
@Override
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPIncomingConnector.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPIncomingConnector.java
index 1013acc..368c58f 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPIncomingConnector.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPIncomingConnector.java
@@ -55,25 +55,11 @@ public final class SCPIncomingConnector implements IncomingStreamingConnector {
@Override
public void init(ConnectorConfig cc) throws Exception {
- SCPStorage scpStorage;
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(cc.getResourceServiceHost(), cc.getResourceServicePort())) {
-
- scpStorage = storageServiceClient.scp()
- .getSCPStorage(SCPStorageGetRequest.newBuilder().setStorageId(cc.getStorageId()).build());
- }
+ SCPStorage scpStorage = cc.getStorage().getScp();
this.resourcePath = cc.getResourcePath();
- SCPSecret scpSecret;
-
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- cc.getSecretServiceHost(), cc.getSecretServicePort())) {
-
- scpSecret = secretClient.scp().getSCPSecret(SCPSecretGetRequest.newBuilder()
- .setAuthzToken(cc.getAuthToken())
- .setSecretId(cc.getCredentialToken()).build());
- }
+ SCPSecret scpSecret = cc.getSecret().getScp();
logger.info("Creating a ssh session for {}@{}:{}", scpSecret.getUser(), scpStorage.getHost(), scpStorage.getPort());
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java
index f6a7ba6..b937875 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java
@@ -18,8 +18,8 @@
package org.apache.airavata.mft.transport.scp;
import net.schmizz.sshj.SSHClient;
-import net.schmizz.sshj.connection.channel.direct.Session;
import net.schmizz.sshj.sftp.FileAttributes;
+import net.schmizz.sshj.sftp.FileMode;
import net.schmizz.sshj.sftp.RemoteResourceInfo;
import net.schmizz.sshj.sftp.SFTPClient;
import net.schmizz.sshj.userauth.keyprovider.KeyProvider;
@@ -28,19 +28,10 @@ import net.schmizz.sshj.userauth.method.AuthMethod;
import net.schmizz.sshj.userauth.method.AuthPublickey;
import net.schmizz.sshj.userauth.method.ChallengeResponseProvider;
import net.schmizz.sshj.userauth.password.Resource;
-import org.apache.airavata.mft.common.AuthToken;
-import org.apache.airavata.mft.core.DirectoryResourceMetadata;
-import org.apache.airavata.mft.core.FileResourceMetadata;
+import org.apache.airavata.mft.agent.stub.*;
import org.apache.airavata.mft.core.api.MetadataCollector;
import org.apache.airavata.mft.credential.stubs.scp.SCPSecret;
-import org.apache.airavata.mft.credential.stubs.scp.SCPSecretGetRequest;
-import org.apache.airavata.mft.resource.client.StorageServiceClient;
-import org.apache.airavata.mft.resource.client.StorageServiceClientBuilder;
import org.apache.airavata.mft.resource.stubs.scp.storage.SCPStorage;
-import org.apache.airavata.mft.resource.stubs.scp.storage.SCPStorageGetRequest;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
-import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,178 +45,88 @@ public class SCPMetadataCollector implements MetadataCollector {
private static final Logger logger = LoggerFactory.getLogger(SCPMetadataCollector.class);
- private String resourceServiceHost;
- private int resourceServicePort;
- private String secretServiceHost;
- private int secretServicePort;
boolean initialized = false;
+ private SCPStorage scpStorage;
+ private SCPSecret scpSecret;
@Override
- public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) {
- this.resourceServiceHost = resourceServiceHost;
- this.resourceServicePort = resourceServicePort;
- this.secretServiceHost = secretServiceHost;
- this.secretServicePort = secretServicePort;
+ public void init(StorageWrapper storage, SecretWrapper secret) {
+ this.scpStorage = storage.getScp();
+ this.scpSecret = secret.getScp();
this.initialized = true;
}
- private void checkInitialized() {
- if (!initialized) {
- throw new IllegalStateException("SCP Metadata Collector is not initialized");
- }
- }
-
- private FileResourceMetadata getFileResourceMetadata(AuthToken authZToken, String resourcePath,
- SCPStorage scpStorage, SCPSecret scpSecret) throws Exception {
+ @Override
+ public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception {
+ ResourceMetadata.Builder resourceBuilder = ResourceMetadata.newBuilder();
try (SSHClient sshClient = getSSHClient(scpStorage, scpSecret)) {
-
- logger.info("Fetching metadata for resource {} in {}", resourcePath, scpStorage.getHost());
-
try (SFTPClient sftpClient = sshClient.newSFTPClient()) {
- FileAttributes lstat = sftpClient.lstat(resourcePath);
- sftpClient.close();
-
- FileResourceMetadata metadata = new FileResourceMetadata();
- metadata.setResourceSize(lstat.getSize());
- metadata.setCreatedTime(lstat.getAtime());
- metadata.setUpdateTime(lstat.getMtime());
- metadata.setFriendlyName(new File(resourcePath).getName());
- metadata.setResourcePath(resourcePath);
- try {
- // TODO calculate md5 using the binary based on the OS platform. Eg: MacOS has md5. Linux has md5sum
- // This only works for linux SCP resources. Improve to work in mac and windows resources
- Session.Command md5Command = sshClient.startSession().exec("md5sum " + resourcePath);
- StringWriter outWriter = new StringWriter();
- StringWriter errorWriter = new StringWriter();
-
- IOUtils.copy(md5Command.getInputStream(), outWriter, "UTF-8");
- Integer exitStatus = md5Command.getExitStatus(); // get exit status ofter reading std out
-
- if (exitStatus != null && exitStatus == 0) {
- metadata.setMd5sum(outWriter.toString().split(" ")[0]);
- } else {
- IOUtils.copy(md5Command.getErrorStream(), errorWriter, "UTF-8");
- logger.warn("MD5 fetch error out {}", errorWriter.toString());
- }
- } catch (Exception e) {
- logger.warn("Failed to fetch md5 for SCP resource with path {}", resourcePath, e);
+ if (sftpClient.statExistence(resourcePath) == null) {
+ resourceBuilder.setError(MetadataFetchError.NOT_FOUND);
+ return resourceBuilder.build();
}
- return metadata;
- }
- }
- }
-
- public FileResourceMetadata getFileResourceMetadata(AuthToken authZToken, String resourcePath, String storageId,
- String credentialToken) throws Exception {
-
- checkInitialized();
-
- SCPStorage scpStorage;
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(resourceServiceHost, resourceServicePort)) {
- scpStorage = storageServiceClient.scp()
- .getSCPStorage(SCPStorageGetRequest.newBuilder().setStorageId(storageId).build());
- }
-
- SCPSecret scpSecret;
-
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort)) {
- scpSecret = secretClient.scp().getSCPSecret(SCPSecretGetRequest.newBuilder()
- .setAuthzToken(authZToken).setSecretId(credentialToken).build());
- }
-
- return getFileResourceMetadata(authZToken, resourcePath, scpStorage, scpSecret);
- }
-
- private DirectoryResourceMetadata getDirectoryResourceMetadata(AuthToken authZToken, String resourcePath, SCPStorage scpStorage, SCPSecret scpSecret) throws Exception {
- try (SSHClient sshClient = getSSHClient(scpStorage, scpSecret)) {
-
- logger.info("Fetching metadata for resource {} in {}", resourcePath, scpStorage.getHost());
-
- try (SFTPClient sftpClient = sshClient.newSFTPClient()) {
- List<RemoteResourceInfo> lsOut = sftpClient.ls(resourcePath);
- FileAttributes lsStat = sftpClient.lstat(resourcePath);
- sftpClient.close();
+ FileAttributes lstat = sftpClient.lstat(resourcePath);
- DirectoryResourceMetadata.Builder dirMetadataBuilder = DirectoryResourceMetadata.Builder.getBuilder()
- .withLazyInitialized(false);
- for (RemoteResourceInfo rri : lsOut) {
- if (rri.isDirectory()) {
- DirectoryResourceMetadata.Builder childDirBuilder = DirectoryResourceMetadata.Builder.getBuilder()
- .withFriendlyName(rri.getName())
- .withResourcePath(rri.getPath())
- .withCreatedTime(rri.getAttributes().getAtime())
- .withUpdateTime(rri.getAttributes().getMtime());
- dirMetadataBuilder = dirMetadataBuilder.withDirectory(childDirBuilder.build());
+ if (lstat.getType() == FileMode.Type.REGULAR) {
+ FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
+ fileBuilder.setResourceSize(lstat.getSize());
+ fileBuilder.setCreatedTime(lstat.getAtime());
+ fileBuilder.setUpdateTime(lstat.getMtime());
+ fileBuilder.setFriendlyName(new File(resourcePath).getName());
+ fileBuilder.setResourcePath(resourcePath);
+ resourceBuilder.setFile(fileBuilder);
+ } else if (lstat.getType() == FileMode.Type.DIRECTORY) {
+
+ DirectoryMetadata.Builder dirBuilder = DirectoryMetadata.newBuilder();
+ dirBuilder.setResourcePath(resourcePath);
+ dirBuilder.setFriendlyName(new File(resourcePath).getName());
+ dirBuilder.setUpdateTime(lstat.getMtime());
+ dirBuilder.setCreatedTime(lstat.getAtime());
+
+ List<RemoteResourceInfo> dirList = sftpClient.ls(resourcePath);
+ for (RemoteResourceInfo rr: dirList) {
+ if (rr.isDirectory()) {
+ DirectoryMetadata.Builder subDir = DirectoryMetadata.newBuilder();
+ subDir.setResourcePath(rr.getPath());
+ subDir.setFriendlyName(rr.getName());
+ subDir.setCreatedTime(rr.getAttributes().getAtime());
+ subDir.setUpdateTime(rr.getAttributes().getMtime());
+ dirBuilder.addDirectories(subDir);
+ } else {
+ FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
+ fileBuilder.setResourceSize(rr.getAttributes().getSize());
+ fileBuilder.setCreatedTime(rr.getAttributes().getAtime());
+ fileBuilder.setUpdateTime(rr.getAttributes().getMtime());
+ fileBuilder.setFriendlyName(rr.getName());
+ fileBuilder.setResourcePath(rr.getPath());
+ dirBuilder.addFiles(fileBuilder);
+ }
}
- if (rri.isRegularFile()) {
- FileResourceMetadata.Builder childFileBuilder = FileResourceMetadata.Builder.newBuilder()
- .withFriendlyName(rri.getName())
- .withResourcePath(rri.getPath())
- .withCreatedTime(rri.getAttributes().getAtime())
- .withUpdateTime(rri.getAttributes().getMtime())
- .withResourceSize(rri.getAttributes().getSize());
-
- dirMetadataBuilder = dirMetadataBuilder.withFile(childFileBuilder.build());
- }
+ } else {
+ resourceBuilder.setError(MetadataFetchError.UNRECOGNIZED);
}
-
- dirMetadataBuilder = dirMetadataBuilder.withFriendlyName(new File(resourcePath).getName())
- .withResourcePath(resourcePath)
- .withCreatedTime(lsStat.getAtime())
- .withUpdateTime(lsStat.getMtime());
- return dirMetadataBuilder.build();
}
}
- }
-
- @Override
- public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthToken authZToken, String resourcePath,
- String storageId, String credentialToken) throws Exception {
-
- SCPStorage scpStorage;
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(resourceServiceHost, resourceServicePort)) {
-
- scpStorage = storageServiceClient.scp()
- .getSCPStorage(SCPStorageGetRequest.newBuilder().setStorageId(storageId).build());
- }
- SCPSecret scpSecret;
+ return resourceBuilder.build();
+ }
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort)) {
- scpSecret = secretClient.scp().getSCPSecret(SCPSecretGetRequest.newBuilder()
- .setAuthzToken(authZToken).setSecretId(credentialToken).build());
+ private void checkInitialized() {
+ if (!initialized) {
+ throw new IllegalStateException("SCP Metadata Collector is not initialized");
}
-
- return getDirectoryResourceMetadata(authZToken, resourcePath, scpStorage, scpSecret);
}
@Override
- public Boolean isAvailable(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception {
+ public Boolean isAvailable(String resourcePath) throws Exception {
checkInitialized();
- SCPStorage scpStorage;
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(resourceServiceHost, resourceServicePort)) {
-
- scpStorage = storageServiceClient.scp()
- .getSCPStorage(SCPStorageGetRequest.newBuilder().setStorageId(storageId).build());
- }
-
- SCPSecret scpSecret;
-
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort)) {
- scpSecret = secretClient.scp().getSCPSecret(SCPSecretGetRequest.newBuilder()
- .setAuthzToken(authZToken).setSecretId(credentialToken).build());
- }
-
try (SSHClient sshClient = getSSHClient(scpStorage, scpSecret)) {
logger.info("Checking the availability of file {}", resourcePath);
try (SFTPClient sftpClient = sshClient.newSFTPClient()) {
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPOutgoingConnector.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPOutgoingConnector.java
index a00b82e..93716d6 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPOutgoingConnector.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPOutgoingConnector.java
@@ -57,23 +57,8 @@ public final class SCPOutgoingConnector implements OutgoingStreamingConnector {
public void init(ConnectorConfig cc) throws Exception {
this.cc = cc;
- SCPStorage scpStorage;
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(cc.getResourceServiceHost(), cc.getResourceServicePort())) {
- scpStorage = storageServiceClient.scp()
- .getSCPStorage(SCPStorageGetRequest.newBuilder().setStorageId(cc.getStorageId()).build());
-
- }
-
- SCPSecret scpSecret;
-
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- cc.getSecretServiceHost(), cc.getSecretServicePort())) {
-
- scpSecret = secretClient.scp().getSCPSecret(SCPSecretGetRequest.newBuilder()
- .setAuthzToken(cc.getAuthToken())
- .setSecretId(cc.getCredentialToken()).build());
- }
+ SCPStorage scpStorage = cc.getStorage().getScp();
+ SCPSecret scpSecret = cc.getSecret().getScp();
logger.info("Creating a ssh session for {}@{}:{}", scpSecret.getUser(), scpStorage.getHost(), scpStorage.getPort());
@@ -98,7 +83,7 @@ public final class SCPOutgoingConnector implements OutgoingStreamingConnector {
@Override
public OutputStream fetchOutputStream() throws Exception {
- return fetchOutputStreamJCraft(escapeSpecialChars(cc.getResourcePath()), cc.getMetadata().getResourceSize());
+ return fetchOutputStreamJCraft(escapeSpecialChars(cc.getResourcePath()), cc.getMetadata().getFile().getResourceSize());
}
public OutputStream fetchOutputStreamJCraft(String resourcePath, long fileSize) throws Exception {
diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java
index 34db1b7..e4555c5 100644
--- a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java
+++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java
@@ -53,56 +53,41 @@ public class SwiftIncomingConnector implements IncomingChunkedConnector {
@Override
public void init(ConnectorConfig cc) throws Exception {
- SwiftStorage swiftStorage;
-
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(cc.getResourceServiceHost(), cc.getResourceServicePort())) {
-
- swiftStorage = storageServiceClient.swift()
- .getSwiftStorage(SwiftStorageGetRequest.newBuilder().setStorageId(cc.getStorageId()).build());
+ SwiftStorage swiftStorage = cc.getStorage().getSwift();
+
+ this.resourcePath = cc.getResourcePath();
+
+ SwiftSecret swiftSecret = cc.getSecret().getSwift();
+
+ String provider = "openstack-swift";
+
+ Properties overrides = new Properties();
+ overrides.put(KeystoneProperties.KEYSTONE_VERSION, swiftStorage.getKeystoneVersion() + "");
+
+ String identity = null;
+ String credential = null;
+ switch (swiftSecret.getSecretCase()) {
+ case PASSWORDSECRET:
+ identity = swiftSecret.getPasswordSecret().getDomainId() + ":" + swiftSecret.getPasswordSecret().getUserName();
+ credential = swiftSecret.getPasswordSecret().getPassword();
+ overrides.put(KeystoneProperties.SCOPE, "projectId:" + swiftSecret.getPasswordSecret().getProjectId());
+ overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.PASSWORD_CREDENTIALS);
+ break;
+ case AUTHCREDENTIALSECRET:
+ identity = swiftSecret.getAuthCredentialSecret().getCredentialId();
+ credential = swiftSecret.getAuthCredentialSecret().getCredentialSecret();
+ overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.API_ACCESS_KEY_CREDENTIALS);
+ break;
}
- this.resourcePath = resourcePath;
-
- SwiftSecret swiftSecret;
-
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- cc.getSecretServiceHost(), cc.getSecretServicePort())) {
-
- swiftSecret = secretClient.swift().getSwiftSecret(SwiftSecretGetRequest.newBuilder()
- .setAuthzToken(cc.getAuthToken())
- .setSecretId(cc.getCredentialToken()).build());
+ swiftApi = ContextBuilder.newBuilder(provider)
+ .endpoint(swiftStorage.getEndpoint())
+ .credentials(identity, credential)
+ .overrides(overrides)
+ .buildApi(SwiftApi.class);
- String provider = "openstack-swift";
+ objectApi = swiftApi.getObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer());
- Properties overrides = new Properties();
- overrides.put(KeystoneProperties.KEYSTONE_VERSION, swiftStorage.getKeystoneVersion() + "");
-
- String identity = null;
- String credential = null;
- switch (swiftSecret.getSecretCase()) {
- case PASSWORDSECRET:
- identity = swiftSecret.getPasswordSecret().getDomainId() + ":" + swiftSecret.getPasswordSecret().getUserName();
- credential = swiftSecret.getPasswordSecret().getPassword();
- overrides.put(KeystoneProperties.SCOPE, "projectId:" + swiftSecret.getPasswordSecret().getProjectId());
- overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.PASSWORD_CREDENTIALS);
- break;
- case AUTHCREDENTIALSECRET:
- identity = swiftSecret.getAuthCredentialSecret().getCredentialId();
- credential = swiftSecret.getAuthCredentialSecret().getCredentialSecret();
- overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.API_ACCESS_KEY_CREDENTIALS);
- break;
- }
-
- swiftApi = ContextBuilder.newBuilder(provider)
- .endpoint(swiftStorage.getEndpoint())
- .credentials(identity, credential)
- .overrides(overrides)
- .buildApi(SwiftApi.class);
-
- objectApi = swiftApi.getObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer());
-
- }
}
@Override
diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java
index 5d4210f..f264852 100644
--- a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java
+++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java
@@ -17,41 +17,29 @@
package org.apache.airavata.mft.transport.swift;
-import org.apache.airavata.mft.common.AuthToken;
-import org.apache.airavata.mft.core.DirectoryResourceMetadata;
-import org.apache.airavata.mft.core.FileResourceMetadata;
+import org.apache.airavata.mft.agent.stub.*;
import org.apache.airavata.mft.core.api.MetadataCollector;
import org.apache.airavata.mft.credential.stubs.swift.SwiftSecret;
-import org.apache.airavata.mft.credential.stubs.swift.SwiftSecretGetRequest;
-import org.apache.airavata.mft.resource.client.StorageServiceClient;
-import org.apache.airavata.mft.resource.client.StorageServiceClientBuilder;
import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage;
-import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorageGetRequest;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
import org.jclouds.ContextBuilder;
import org.jclouds.openstack.keystone.auth.config.CredentialTypes;
import org.jclouds.openstack.keystone.config.KeystoneProperties;
import org.jclouds.openstack.swift.v1.SwiftApi;
+import org.jclouds.openstack.swift.v1.domain.ObjectList;
import org.jclouds.openstack.swift.v1.domain.SwiftObject;
import org.jclouds.openstack.swift.v1.features.ObjectApi;
import java.util.Properties;
public class SwiftMetadataCollector implements MetadataCollector {
-
- private String resourceServiceHost;
- private int resourceServicePort;
- private String secretServiceHost;
- private int secretServicePort;
boolean initialized = false;
+ private SwiftStorage swiftStorage;
+ private SwiftSecret swiftSecret;
@Override
- public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) {
- this.resourceServiceHost = resourceServiceHost;
- this.resourceServicePort = resourceServicePort;
- this.secretServiceHost = secretServiceHost;
- this.secretServicePort = secretServicePort;
+ public void init(StorageWrapper storage, SecretWrapper secret) {
+ this.swiftStorage = storage.getSwift();
+ this.swiftSecret = secret.getSwift();
this.initialized = true;
}
@@ -91,61 +79,51 @@ public class SwiftMetadataCollector implements MetadataCollector {
}
@Override
- public FileResourceMetadata getFileResourceMetadata(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception {
+ public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception {
checkInitialized();
- SwiftStorage swiftStorage;
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(resourceServiceHost, resourceServicePort)) {
-
- swiftStorage = storageServiceClient.swift()
- .getSwiftStorage(SwiftStorageGetRequest.newBuilder().setStorageId(storageId).build());
- }
-
- SwiftSecret swiftSecret;
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- secretServiceHost, secretServicePort)) {
- swiftSecret = secretClient.swift().getSwiftSecret(SwiftSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
- }
-
SwiftApi swiftApi = getSwiftApi(swiftStorage, swiftSecret);
ObjectApi objectApi = swiftApi.getObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer());
- SwiftObject swiftObject = objectApi.get(resourcePath);
-
- FileResourceMetadata metadata = new FileResourceMetadata();
- metadata.setResourceSize(swiftObject.getPayload().getContentMetadata().getContentLength());
- metadata.setMd5sum(swiftObject.getETag());
- metadata.setUpdateTime(swiftObject.getLastModified().getTime());
- metadata.setCreatedTime(swiftObject.getLastModified().getTime());
- return metadata;
- }
-
- @Override
- public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthToken authZToken, String resourcePath,
- String storageId, String credentialToken) throws Exception {
- throw new UnsupportedOperationException("Method not implemented");
+ ResourceMetadata.Builder resourceBuilder = ResourceMetadata.newBuilder();
+ if ("".equals(resourcePath)) {
+ DirectoryMetadata.Builder rootDirBuilder = DirectoryMetadata.newBuilder();
+
+ ObjectList objectList = objectApi.list();
+ objectList.forEach(swiftObject -> {
+ FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
+ fileBuilder.setFriendlyName(swiftObject.getName());
+ fileBuilder.setResourcePath(swiftObject.getName());
+ fileBuilder.setCreatedTime(swiftObject.getLastModified().getTime());
+ fileBuilder.setUpdateTime(swiftObject.getLastModified().getTime());
+ fileBuilder.setResourceSize(swiftObject.getPayload().getContentMetadata().getContentLength());
+ rootDirBuilder.addFiles(fileBuilder);
+ });
+ resourceBuilder.setDirectory(rootDirBuilder);
+ } else {
+ SwiftObject swiftObject = objectApi.get(resourcePath);
+
+ if (swiftObject == null) {
+ resourceBuilder.setError(MetadataFetchError.NOT_FOUND);
+ return resourceBuilder.build();
+ }
+
+ FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
+ fileBuilder.setFriendlyName(swiftObject.getName());
+ fileBuilder.setResourcePath(swiftObject.getName());
+ fileBuilder.setCreatedTime(swiftObject.getLastModified().getTime());
+ fileBuilder.setUpdateTime(swiftObject.getLastModified().getTime());
+ fileBuilder.setResourceSize(swiftObject.getPayload().getContentMetadata().getContentLength());
+ resourceBuilder.setFile(fileBuilder);
+ }
+ return resourceBuilder.build();
}
@Override
- public Boolean isAvailable(AuthToken authZToken, String resourcePath, String storageId, String credentialToken) throws Exception {
+ public Boolean isAvailable(String resourcePath) throws Exception {
checkInitialized();
- SwiftStorage swiftStorage;
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(resourceServiceHost, resourceServicePort)) {
-
- swiftStorage = storageServiceClient.swift()
- .getSwiftStorage(SwiftStorageGetRequest.newBuilder().setStorageId(storageId).build());
- }
-
- SwiftSecret swiftSecret;
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- secretServiceHost, secretServicePort)) {
- swiftSecret = secretClient.swift().getSwiftSecret(SwiftSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
- }
-
SwiftApi swiftApi = getSwiftApi(swiftStorage, swiftSecret);
ObjectApi objectApi = swiftApi.getObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer());
diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java
index 565ab30..52f7a50 100644
--- a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java
+++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java
@@ -59,55 +59,41 @@ public class SwiftOutgoingConnector implements OutgoingChunkedConnector {
@Override
public void init(ConnectorConfig cc) throws Exception {
- SwiftStorage swiftStorage;
- try (StorageServiceClient storageServiceClient = StorageServiceClientBuilder
- .buildClient(cc.getResourceServiceHost(), cc.getResourceServicePort())) {
- swiftStorage = storageServiceClient.swift()
- .getSwiftStorage(SwiftStorageGetRequest.newBuilder().setStorageId(cc.getStorageId()).build());
-
- }
+ SwiftStorage swiftStorage = cc.getStorage().getSwift();
this.resourcePath = cc.getResourcePath();
- SwiftSecret swiftSecret;
-
- try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
- cc.getSecretServiceHost(), cc.getSecretServicePort())) {
-
- swiftSecret = secretClient.swift().getSwiftSecret(SwiftSecretGetRequest.newBuilder()
- .setAuthzToken(cc.getAuthToken())
- .setSecretId(cc.getCredentialToken()).build());
-
- String provider = "openstack-swift";
-
- Properties overrides = new Properties();
- overrides.put(KeystoneProperties.KEYSTONE_VERSION, swiftStorage.getKeystoneVersion() + "");
-
- String identity = null;
- String credential = null;
- switch (swiftSecret.getSecretCase()) {
- case PASSWORDSECRET:
- identity = swiftSecret.getPasswordSecret().getDomainId() + ":" + swiftSecret.getPasswordSecret().getUserName();
- credential = swiftSecret.getPasswordSecret().getPassword();
- overrides.put(KeystoneProperties.SCOPE, "projectId:" + swiftSecret.getPasswordSecret().getProjectId());
- overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.PASSWORD_CREDENTIALS);
- break;
- case AUTHCREDENTIALSECRET:
- identity = swiftSecret.getAuthCredentialSecret().getCredentialId();
- credential = swiftSecret.getAuthCredentialSecret().getCredentialSecret();
- overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.API_ACCESS_KEY_CREDENTIALS);
- break;
- }
-
- swiftApi = ContextBuilder.newBuilder(provider)
- .endpoint(swiftStorage.getEndpoint())
- .credentials(identity, credential)
- .overrides(overrides)
- .buildApi(SwiftApi.class);
-
- objectApi = swiftApi.getObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer());
- staticLargeObjectApi = swiftApi.getStaticLargeObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer());
+ SwiftSecret swiftSecret = cc.getSecret().getSwift();
+
+ String provider = "openstack-swift";
+
+ Properties overrides = new Properties();
+ overrides.put(KeystoneProperties.KEYSTONE_VERSION, swiftStorage.getKeystoneVersion() + "");
+
+ String identity = null;
+ String credential = null;
+ switch (swiftSecret.getSecretCase()) {
+ case PASSWORDSECRET:
+ identity = swiftSecret.getPasswordSecret().getDomainId() + ":" + swiftSecret.getPasswordSecret().getUserName();
+ credential = swiftSecret.getPasswordSecret().getPassword();
+ overrides.put(KeystoneProperties.SCOPE, "projectId:" + swiftSecret.getPasswordSecret().getProjectId());
+ overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.PASSWORD_CREDENTIALS);
+ break;
+ case AUTHCREDENTIALSECRET:
+ identity = swiftSecret.getAuthCredentialSecret().getCredentialId();
+ credential = swiftSecret.getAuthCredentialSecret().getCredentialSecret();
+ overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.API_ACCESS_KEY_CREDENTIALS);
+ break;
}
+
+ swiftApi = ContextBuilder.newBuilder(provider)
+ .endpoint(swiftStorage.getEndpoint())
+ .credentials(identity, credential)
+ .overrides(overrides)
+ .buildApi(SwiftApi.class);
+
+ objectApi = swiftApi.getObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer());
+ staticLargeObjectApi = swiftApi.getStaticLargeObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer());
}
@Override