You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2020/05/03 18:56:59 UTC
[airavata-mft] branch master updated (b292231 -> 130d5a9)
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git.
from b292231 Adding few more language specific ignore paths
new bf3be6d adding FTP secrets file sample data
new f803e24 secrets service proto FTP changes
new d951334 resources service proto FTP changes
new 3e95f8d adding FTP resource file sample data
new 0ce23fc FTP grpc method implementation and reading resources from different backends
new c435663 FTP secret grpc method implementation and reading resources from different backends
new a57b554 FTP transport implementation
new 7130037 Adding FTP in application properties of agent module
new d3520b9 adding FTP sender, receiver and metadatadata collector to the core
new a4635b5 adding missing dependency
new d391925 individual imports and license header addition
new 3d621d5 individual imports and license header addition
new 22054a1 using resource file variable
new e61eea9 adding FTP to the application properties
new 130d5a9 adding ftp module to the transport build
The 15 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
agent/pom.xml | 5 +
agent/src/main/resources/application.properties | 2 +-
.../distribution/conf/application.properties | 2 +-
.../samples/mft_api_client_sample.py | 0
.../airavata/mft/core/ConnectorResolver.java | 10 ++
.../mft/core/MetadataCollectorResolver.java | 3 +
.../resource/server/backend/ResourceBackend.java | 10 ++
.../backend/airavata/AiravataResourceBackend.java | 40 ++++++
.../backend/file/FileBasedResourceBackend.java | 73 +++++++++++
.../server/backend/sql/SQLResourceBackend.java | 113 ++++++++++++++++-
...PResourceEntity.java => FTPResourceEntity.java} | 37 +++---
...SCPStorageEntity.java => FTPStorageEntity.java} | 16 +--
...eRepository.java => FTPResourceRepository.java} | 6 +-
...geRepository.java => FTPStorageRepository.java} | 6 +-
.../server/handler/ResourceServiceHandler.java | 135 +++++++++++++++++++++
.../server/src/main/resources/resources.json | 10 ++
.../stub/src/main/proto/ResourceService.proto | 106 ++++++++++++++++
.../mft/secret/server/backend/SecretBackend.java | 5 +
.../backend/airavata/AiravataSecretBackend.java | 19 +++
.../backend/file/FileBasedSecretBackend.java | 42 +++++++
.../server/backend/sql/SQLSecretBackend.java | 28 +++++
.../{SCPSecretEntity.java => FTPSecretEntity.java} | 37 ++----
...retRepository.java => FTPSecretRepository.java} | 6 +-
.../server/handler/SecretServiceHandler.java | 53 ++++++++
.../server/src/main/resources/secrets.json | 6 +
.../stub/src/main/proto/SecretService.proto | 56 +++++++++
transport/{box-transport => ftp-transport}/pom.xml | 11 +-
.../mft/transport/ftp/FTPMetadataCollector.java | 123 +++++++++++++++++++
.../airavata/mft/transport/ftp/FTPReceiver.java} | 65 +++++-----
.../airavata/mft/transport/ftp/FTPSender.java} | 68 ++++++-----
.../mft/transport/ftp/FTPTransportUtil.java | 52 ++++++++
transport/pom.xml | 1 +
32 files changed, 1010 insertions(+), 136 deletions(-)
create mode 100644 client-sdk/mft-python-sdk/samples/mft_api_client_sample.py
copy services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/{SCPResourceEntity.java => FTPResourceEntity.java} (71%)
copy services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/{SCPStorageEntity.java => FTPStorageEntity.java} (87%)
copy services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/{SCPResourceRepository.java => FTPResourceRepository.java} (87%)
copy services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/{SCPStorageRepository.java => FTPStorageRepository.java} (87%)
copy services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/{SCPSecretEntity.java => FTPSecretEntity.java} (67%)
copy services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/{SecretRepository.java => FTPSecretRepository.java} (86%)
copy transport/{box-transport => ftp-transport}/pom.xml (89%)
create mode 100644 transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java
copy transport/{azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureReceiver.java => ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java} (59%)
copy transport/{dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxReceiver.java => ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java} (56%)
create mode 100644 transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPTransportUtil.java
[airavata-mft] 03/15: resources service proto FTP changes
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit d95133445460a8ca0e570bc7dab014c54724b02b
Author: Gopi Kiran <go...@gmail.com>
AuthorDate: Wed Apr 29 15:42:27 2020 -0400
resources service proto FTP changes
---
.../stub/src/main/proto/ResourceService.proto | 106 +++++++++++++++++++++
1 file changed, 106 insertions(+)
diff --git a/services/resource-service/stub/src/main/proto/ResourceService.proto b/services/resource-service/stub/src/main/proto/ResourceService.proto
index 6965913..8eda97d 100644
--- a/services/resource-service/stub/src/main/proto/ResourceService.proto
+++ b/services/resource-service/stub/src/main/proto/ResourceService.proto
@@ -236,6 +236,60 @@ message DropboxResourceDeleteRequest {
string resourceId = 1;
}
+// FTP Storage
+
+message FTPStorage {
+ string storageId = 1;
+ string host = 2;
+ int32 port = 3;
+}
+
+message FTPStorageGetRequest {
+ string storageId = 1;
+}
+
+message FTPStorageCreateRequest {
+ string host = 1;
+ int32 port = 2;
+}
+
+message FTPStorageUpdateRequest {
+ string storageId = 1;
+ string host = 2;
+ int32 port = 3;
+}
+
+message FTPStorageDeleteRequest {
+ string storageId = 1;
+}
+
+// FTP Resource
+
+message FTPResource {
+ string resourceId = 1;
+ FTPStorage ftpStorage = 2;
+ string resourcePath = 3;
+}
+
+message FTPResourceGetRequest {
+ string resourceId = 1;
+}
+
+message FTPResourceCreateRequest {
+ string ftpStorageId = 1;
+ string resourcePath = 2;
+}
+
+message FTPResourceUpdateRequest {
+ string resourceId = 1;
+ string FTPStorageId = 2;
+ string resourcePath = 3;
+}
+
+message FTPResourceDeleteRequest {
+ string resourceId = 1;
+}
+
service ResourceService {
// SCP Storage
@@ -444,4 +498,56 @@ service ResourceService {
};
}
+ // FTP Storage
+
+ rpc getFTPStorage (FTPStorageGetRequest) returns (FTPStorage) {
+ option (google.api.http) = {
+ get: "/v1.0/resource/ftp/storage"
+ };
+ }
+
+ rpc createFTPStorage (FTPStorageCreateRequest) returns (FTPStorage) {
+ option (google.api.http) = {
+ post: "/v1.0/resource/ftp/storage"
+ };
+ }
+
+ rpc updateFTPStorage (FTPStorageUpdateRequest) returns (google.protobuf.Empty) {
+ option (google.api.http) = {
+ put: "/v1.0/resource/ftp/storage"
+ };
+ }
+
+ rpc deleteFTPStorage (FTPStorageDeleteRequest) returns (google.protobuf.Empty) {
+ option (google.api.http) = {
+ delete: "/v1.0/resource/ftp/storage"
+ };
+ }
+
+ // FTP Resource
+
+ rpc getFTPResource (FTPResourceGetRequest) returns (FTPResource) {
+ option (google.api.http) = {
+ get: "/v1.0/resource/ftp"
+ };
+ }
+
+ rpc createFTPResource (FTPResourceCreateRequest) returns (FTPResource) {
+ option (google.api.http) = {
+ post: "/v1.0/resource/ftp"
+ };
+ }
+
+ rpc updateFTPResource (FTPResourceUpdateRequest) returns (google.protobuf.Empty) {
+ option (google.api.http) = {
+ put: "/v1.0/resource/ftp"
+ };
+ }
+
+ rpc deleteFTPResource (FTPResourceDeleteRequest) returns (google.protobuf.Empty) {
+ option (google.api.http) = {
+ delete: "/v1.0/resource/ftp"
+ };
+ }
+
}
\ No newline at end of file
[airavata-mft] 01/15: adding FTP secrets file sample data
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit bf3be6d856db9202e283047ea85292331a51dc24
Author: Gopi Kiran <go...@gmail.com>
AuthorDate: Wed Apr 29 15:38:08 2020 -0400
adding FTP secrets file sample data
---
services/secret-service/server/src/main/resources/secrets.json | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/services/secret-service/server/src/main/resources/secrets.json b/services/secret-service/server/src/main/resources/secrets.json
index 210ae85..c937452 100644
--- a/services/secret-service/server/src/main/resources/secrets.json
+++ b/services/secret-service/server/src/main/resources/secrets.json
@@ -32,5 +32,11 @@
"type": "DROPBOX",
"secretId": "dropbox-cred",
"accessToken": ""
+ },
+ {
+ "type": "FTP",
+ "secretId": "ftp-cred",
+ "userId": "dlpuser@dlptest.com",
+ "password": "SzMf7rTE4pCrf9dV286GuNe4N"
}
]
\ No newline at end of file
[airavata-mft] 11/15: individual imports and license header addition
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit d39192543a8bc08ff98855aa872aec6e72076ff6
Author: Gopi Kiran <go...@gmail.com>
AuthorDate: Fri May 1 15:22:43 2020 -0400
individual imports and license header addition
---
.../mft/transport/ftp/FTPMetadataCollector.java | 17 +++++++++++++++++
.../apache/airavata/mft/transport/ftp/FTPReceiver.java | 17 +++++++++++++++++
.../apache/airavata/mft/transport/ftp/FTPSender.java | 17 +++++++++++++++++
.../airavata/mft/transport/ftp/FTPTransportUtil.java | 17 +++++++++++++++++
4 files changed, 68 insertions(+)
diff --git a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java
index 8b7c9df..d6c5d23 100644
--- a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java
+++ b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.airavata.mft.transport.ftp;
import org.apache.airavata.mft.core.ResourceMetadata;
diff --git a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java
index 46909dd..f3fc4d8 100644
--- a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java
+++ b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.airavata.mft.transport.ftp;
import org.apache.airavata.mft.core.ConnectorContext;
diff --git a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java
index 41d4b8e..47c6419 100644
--- a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java
+++ b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.airavata.mft.transport.ftp;
import org.apache.airavata.mft.core.ConnectorContext;
diff --git a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPTransportUtil.java b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPTransportUtil.java
index 81608a4..b0ec235 100644
--- a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPTransportUtil.java
+++ b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPTransportUtil.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.airavata.mft.transport.ftp;
import org.apache.airavata.mft.resource.service.FTPResource;
[airavata-mft] 05/15: FTP grpc method implementation and reading
resources from different backends
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit 0ce23fc1b8e41572e3799b12e0096bacd398ac7c
Author: Gopi Kiran <go...@gmail.com>
AuthorDate: Wed Apr 29 16:02:11 2020 -0400
FTP grpc method implementation and reading resources from different backends
---
.../resource/server/backend/ResourceBackend.java | 10 ++
.../backend/airavata/AiravataResourceBackend.java | 40 ++++++
.../backend/file/FileBasedResourceBackend.java | 73 +++++++++++
.../server/backend/sql/SQLResourceBackend.java | 64 +++++++++-
.../backend/sql/entity/FTPResourceEntity.java | 57 +++++++++
.../backend/sql/entity/FTPStorageEntity.java | 48 ++++++++
.../sql/repository/FTPResourceRepository.java | 10 ++
.../sql/repository/FTPStorageRepository.java | 10 ++
.../server/handler/ResourceServiceHandler.java | 135 +++++++++++++++++++++
9 files changed, 441 insertions(+), 6 deletions(-)
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java
index 253fa16..fe6d311 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java
@@ -66,4 +66,14 @@ public interface ResourceBackend {
public DropboxResource createDropboxResource(DropboxResourceCreateRequest request) throws Exception;
public boolean updateDropboxResource(DropboxResourceUpdateRequest request) throws Exception;
public boolean deleteDropboxResource(DropboxResourceDeleteRequest request) throws Exception;
+
+ Optional<FTPStorage> getFTPStorage(FTPStorageGetRequest request) throws Exception;
+ FTPStorage createFTPStorage(FTPStorageCreateRequest request) throws Exception;
+ boolean updateFTPStorage(FTPStorageUpdateRequest request) throws Exception;
+ boolean deleteFTPStorage(FTPStorageDeleteRequest request) throws Exception;
+
+ Optional<FTPResource> getFTPResource(FTPResourceGetRequest request) throws Exception;
+ FTPResource createFTPResource(FTPResourceCreateRequest request) throws Exception;
+ boolean updateFTPResource(FTPResourceUpdateRequest request) throws Exception;
+ boolean deleteFTPResource(FTPResourceDeleteRequest request) throws Exception;
}
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/airavata/AiravataResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/airavata/AiravataResourceBackend.java
index 17b0bd1..1f05394 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/airavata/AiravataResourceBackend.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/airavata/AiravataResourceBackend.java
@@ -270,4 +270,44 @@ public class AiravataResourceBackend implements ResourceBackend {
public boolean deleteDropboxResource(DropboxResourceDeleteRequest request) throws Exception {
throw new UnsupportedOperationException("Operation is not supported in backend");
}
+
+ @Override
+ public Optional<FTPResource> getFTPResource(FTPResourceGetRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
+
+ @Override
+ public FTPResource createFTPResource(FTPResourceCreateRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
+
+ @Override
+ public boolean updateFTPResource(FTPResourceUpdateRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
+
+ @Override
+ public boolean deleteFTPResource(FTPResourceDeleteRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
+
+ @Override
+ public Optional<FTPStorage> getFTPStorage(FTPStorageGetRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
+
+ @Override
+ public FTPStorage createFTPStorage(FTPStorageCreateRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
+
+ @Override
+ public boolean updateFTPStorage(FTPStorageUpdateRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
+
+ @Override
+ public boolean deleteFTPStorage(FTPStorageDeleteRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
}
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java
index fd8a73f..ae29eb4 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java
@@ -25,12 +25,14 @@ import org.json.simple.parser.JSONParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
+@SuppressWarnings("unchecked")
public class FileBasedResourceBackend implements ResourceBackend {
private static final Logger logger = LoggerFactory.getLogger(FileBasedResourceBackend.class);
@@ -383,4 +385,75 @@ public class FileBasedResourceBackend implements ResourceBackend {
public boolean deleteDropboxResource(DropboxResourceDeleteRequest request) throws Exception {
throw new UnsupportedOperationException("Operation is not supported in backend");
}
+
+ @Override
+ public Optional<FTPResource> getFTPResource(FTPResourceGetRequest request) throws Exception {
+ InputStream inputStream = FileBasedResourceBackend.class.getClassLoader().getResourceAsStream("resources.json");
+
+ JSONParser jsonParser = new JSONParser();
+
+ if (inputStream == null) {
+ throw new IOException("resources file not found");
+ }
+
+ try (InputStreamReader reader = new InputStreamReader(inputStream)) {
+
+ Object obj = jsonParser.parse(reader);
+
+ JSONArray resourceList = (JSONArray) obj;
+
+ List<FTPResource> ftpResources = (List<FTPResource>) resourceList.stream()
+ .filter(resource -> "FTP".equals(((JSONObject) resource).get("type").toString()))
+ .map(resource -> {
+ JSONObject r = (JSONObject) resource;
+
+ FTPStorage storage = FTPStorage.newBuilder()
+ .setStorageId(((JSONObject)r.get("ftpStorage")).get("storageId").toString())
+ .setHost(((JSONObject)r.get("ftpStorage")).get("host").toString())
+ .setPort(Integer.parseInt(((JSONObject)r.get("ftpStorage")).get("port").toString())).build();
+
+ return FTPResource.newBuilder()
+ .setResourcePath(r.get("resourcePath").toString())
+ .setResourceId(r.get("resourceId").toString())
+ .setFtpStorage(storage).build();
+ }).collect(Collectors.toList());
+
+ return ftpResources.stream().filter(r -> request.getResourceId().equals(r.getResourceId())).findFirst();
+ }
+ }
+
+ @Override
+ public FTPResource createFTPResource(FTPResourceCreateRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
+
+ @Override
+ public boolean updateFTPResource(FTPResourceUpdateRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
+
+ @Override
+ public boolean deleteFTPResource(FTPResourceDeleteRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
+
+ @Override
+ public Optional<FTPStorage> getFTPStorage(FTPStorageGetRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
+
+ @Override
+ public FTPStorage createFTPStorage(FTPStorageCreateRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
+
+ @Override
+ public boolean updateFTPStorage(FTPStorageUpdateRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
+
+ @Override
+ public boolean deleteFTPStorage(FTPStorageDeleteRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
}
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java
index 1718ba8..adea1a0 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java
@@ -18,12 +18,8 @@
package org.apache.airavata.mft.resource.server.backend.sql;
import org.apache.airavata.mft.resource.server.backend.ResourceBackend;
-import org.apache.airavata.mft.resource.server.backend.sql.entity.LocalResourceEntity;
-import org.apache.airavata.mft.resource.server.backend.sql.entity.SCPResourceEntity;
-import org.apache.airavata.mft.resource.server.backend.sql.entity.SCPStorageEntity;
-import org.apache.airavata.mft.resource.server.backend.sql.repository.LocalResourceRepository;
-import org.apache.airavata.mft.resource.server.backend.sql.repository.SCPResourceRepository;
-import org.apache.airavata.mft.resource.server.backend.sql.repository.SCPStorageRepository;
+import org.apache.airavata.mft.resource.server.backend.sql.entity.*;
+import org.apache.airavata.mft.resource.server.backend.sql.repository.*;
import org.apache.airavata.mft.resource.service.*;
import org.dozer.DozerBeanMapper;
import org.slf4j.Logger;
@@ -45,6 +41,12 @@ public class SQLResourceBackend implements ResourceBackend {
@Autowired
private LocalResourceRepository localResourceRepository;
+ @Autowired
+ private FTPResourceRepository ftpResourceRepository;
+
+ @Autowired
+ private FTPStorageRepository ftpStorageRepository;
+
private DozerBeanMapper mapper = new DozerBeanMapper();
@Override
@@ -236,4 +238,54 @@ public class SQLResourceBackend implements ResourceBackend {
throw new UnsupportedOperationException("Operation is not supported in backend");
}
+ @Override
+ public Optional<FTPResource> getFTPResource(FTPResourceGetRequest request) {
+ Optional<FTPResourceEntity> resourceEntity = ftpResourceRepository.findByResourceId(request.getResourceId());
+
+ return resourceEntity.map(ftpResourceEntity -> mapper.map(ftpResourceEntity, FTPResource.newBuilder().getClass())
+ .setFtpStorage(mapper.map(ftpResourceEntity.getFtpStorage(), FTPStorage.newBuilder().getClass())).build());
+ }
+
+ @Override
+ public FTPResource createFTPResource(FTPResourceCreateRequest request) {
+ FTPResourceEntity savedEntity = ftpResourceRepository.save(mapper.map(request, FTPResourceEntity.class));
+ return getFTPResource(FTPResourceGetRequest.newBuilder().setResourceId(savedEntity.getResourceId()).build()).orElse(null);
+ }
+
+ @Override
+ public boolean updateFTPResource(FTPResourceUpdateRequest request) {
+ ftpResourceRepository.save(mapper.map(request, FTPResourceEntity.class));
+ return true;
+ }
+
+ @Override
+ public boolean deleteFTPResource(FTPResourceDeleteRequest request) {
+ ftpResourceRepository.deleteById(request.getResourceId());
+ return true;
+ }
+
+ @Override
+ public Optional<FTPStorage> getFTPStorage(FTPStorageGetRequest request) {
+ Optional<FTPStorageEntity> storageEty = ftpStorageRepository.findByStorageId(request.getStorageId());
+ return storageEty.map(ftpStorageEntity -> mapper.map(ftpStorageEntity, FTPStorage.newBuilder().getClass()).build());
+ }
+
+ @Override
+ public FTPStorage createFTPStorage(FTPStorageCreateRequest request) {
+ FTPStorageEntity savedEntity = ftpStorageRepository.save(mapper.map(request, FTPStorageEntity.class));
+ return mapper.map(savedEntity, FTPStorage.newBuilder().getClass()).build();
+ }
+
+ @Override
+ public boolean updateFTPStorage(FTPStorageUpdateRequest request) {
+ ftpStorageRepository.save(mapper.map(request, FTPStorageEntity.class));
+ return true;
+ }
+
+ @Override
+ public boolean deleteFTPStorage(FTPStorageDeleteRequest request) {
+ ftpResourceRepository.deleteById(request.getStorageId());
+ return true;
+ }
+
}
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/FTPResourceEntity.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/FTPResourceEntity.java
new file mode 100644
index 0000000..5aba915
--- /dev/null
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/FTPResourceEntity.java
@@ -0,0 +1,57 @@
+package org.apache.airavata.mft.resource.server.backend.sql.entity;
+
+import org.hibernate.annotations.GenericGenerator;
+
+import javax.persistence.*;
+
+@Entity
+public class FTPResourceEntity {
+
+ @Id
+ @Column(name = "FTP_RESOURCE_ID")
+ @GeneratedValue(generator = "uuid")
+ @GenericGenerator(name = "uuid", strategy = "uuid2")
+ private String resourceId;
+
+ @Column(name = "FTP_STORAGE_ID")
+ private String ftpStorageId;
+
+ @ManyToOne(fetch = FetchType.EAGER)
+ @JoinColumn(name = "FTP_STORAGE_ID", referencedColumnName = "FTP_STORAGE_ID", nullable = false, insertable = false, updatable = false)
+ private FTPStorageEntity ftpStorage;
+
+ @Column(name = "RESOURCE_PATH")
+ private String resourcePath;
+
+ public String getResourceId() {
+ return resourceId;
+ }
+
+ public void setResourceId(String resourceId) {
+ this.resourceId = resourceId;
+ }
+
+ public String getFtpStorageId() {
+ return ftpStorageId;
+ }
+
+ public void setFtpStorageId(String ftpStorageId) {
+ this.ftpStorageId = ftpStorageId;
+ }
+
+ public FTPStorageEntity getFtpStorage() {
+ return ftpStorage;
+ }
+
+ public void setFtpStorage(FTPStorageEntity ftpStorage) {
+ this.ftpStorage = ftpStorage;
+ }
+
+ public String getResourcePath() {
+ return resourcePath;
+ }
+
+ public void setResourcePath(String resourcePath) {
+ this.resourcePath = resourcePath;
+ }
+}
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/FTPStorageEntity.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/FTPStorageEntity.java
new file mode 100644
index 0000000..2c42a8c
--- /dev/null
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/FTPStorageEntity.java
@@ -0,0 +1,48 @@
+package org.apache.airavata.mft.resource.server.backend.sql.entity;
+
+import org.hibernate.annotations.GenericGenerator;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+
+@Entity
+public class FTPStorageEntity {
+
+ @Id
+ @Column(name = "FTP_STORAGE_ID")
+ @GeneratedValue(generator = "uuid")
+ @GenericGenerator(name = "uuid", strategy = "uuid2")
+ private String storageId;
+
+ @Column(name = "HOST")
+ private String host;
+
+ @Column(name = "PORT")
+ private int port;
+
+ public String getStorageId() {
+ return storageId;
+ }
+
+ public void setStorageId(String storageId) {
+ this.storageId = storageId;
+ }
+
+ public String getHost() {
+ return host;
+ }
+
+ public void setHost(String host) {
+ this.host = host;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ public void setPort(int port) {
+ this.port = port;
+ }
+}
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/FTPResourceRepository.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/FTPResourceRepository.java
new file mode 100644
index 0000000..ad1f855
--- /dev/null
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/FTPResourceRepository.java
@@ -0,0 +1,10 @@
+package org.apache.airavata.mft.resource.server.backend.sql.repository;
+
+import org.apache.airavata.mft.resource.server.backend.sql.entity.FTPResourceEntity;
+import org.springframework.data.repository.CrudRepository;
+
+import java.util.Optional;
+
+public interface FTPResourceRepository extends CrudRepository<FTPResourceEntity, String> {
+ Optional<FTPResourceEntity> findByResourceId(String resourceId);
+}
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/FTPStorageRepository.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/FTPStorageRepository.java
new file mode 100644
index 0000000..c80187a
--- /dev/null
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/FTPStorageRepository.java
@@ -0,0 +1,10 @@
+package org.apache.airavata.mft.resource.server.backend.sql.repository;
+
+import org.apache.airavata.mft.resource.server.backend.sql.entity.FTPStorageEntity;
+import org.springframework.data.repository.CrudRepository;
+
+import java.util.Optional;
+
+public interface FTPStorageRepository extends CrudRepository<FTPStorageEntity, String> {
+ Optional<FTPStorageEntity> findByStorageId(String storageId);
+}
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/ResourceServiceHandler.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/ResourceServiceHandler.java
index 7e0ae28..1b66319 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/ResourceServiceHandler.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/ResourceServiceHandler.java
@@ -571,4 +571,139 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
.asRuntimeException());
}
}
+
+ @Override
+ public void getFTPStorage(FTPStorageGetRequest request, StreamObserver<FTPStorage> responseObserver) {
+ try {
+ this.backend.getFTPStorage(request).ifPresentOrElse(storage -> {
+ responseObserver.onNext(storage);
+ responseObserver.onCompleted();
+ }, () -> responseObserver.onError(Status.INTERNAL
+ .withDescription("No FTP Storage with id " + request.getStorageId())
+ .asRuntimeException()));
+ } catch (Exception e) {
+ logger.error("Failed in retrieving FTP storage with id " + request.getStorageId(), e);
+
+ responseObserver.onError(Status.INTERNAL.withCause(e)
+ .withDescription("Failed in retrieving FTP storage with id " + request.getStorageId())
+ .asRuntimeException());
+ }
+ }
+
+ @Override
+ public void createFTPStorage(FTPStorageCreateRequest request, StreamObserver<FTPStorage> responseObserver) {
+ try {
+ responseObserver.onNext(this.backend.createFTPStorage(request));
+ responseObserver.onCompleted();
+ } catch (Exception e) {
+ logger.error("Failed in creating the FTP storage", e);
+
+ responseObserver.onError(Status.INTERNAL.withCause(e)
+ .withDescription("Failed in creating the FTP storage")
+ .asRuntimeException());
+ }
+ }
+
+ @Override
+ public void updateFTPStorage(FTPStorageUpdateRequest request, StreamObserver<Empty> responseObserver) {
+ try {
+ this.backend.updateFTPStorage(request);
+ responseObserver.onCompleted();
+ } catch (Exception e) {
+ logger.error("Failed in updating the FTP storage {}", request.getStorageId(), e);
+
+ responseObserver.onError(Status.INTERNAL.withCause(e)
+ .withDescription("Failed in updating the FTP storage")
+ .asRuntimeException());
+ }
+ }
+
+ @Override
+ public void deleteFTPStorage(FTPStorageDeleteRequest request, StreamObserver<Empty> responseObserver) {
+ try {
+ boolean res = this.backend.deleteFTPStorage(request);
+ if (res) {
+ responseObserver.onCompleted();
+ } else {
+ logger.error("Failed to delete FTP Storage with id " + request.getStorageId());
+
+ responseObserver.onError(Status.INTERNAL
+ .withDescription("Failed to delete FTP Storage with id " + request.getStorageId())
+ .asRuntimeException());
+ }
+ } catch (Exception e) {
+ logger.error("Failed in deleting the FTP storage {}", request.getStorageId(), e);
+
+ responseObserver.onError(Status.INTERNAL.withCause(e)
+ .withDescription("Failed in deleting the FTP storage")
+ .asRuntimeException());
+ }
+ }
+
+ @Override
+ public void getFTPResource(FTPResourceGetRequest request, StreamObserver<FTPResource> responseObserver) {
+ try {
+ this.backend.getFTPResource(request).ifPresentOrElse(resource -> {
+ responseObserver.onNext(resource);
+ responseObserver.onCompleted();
+ }, () -> responseObserver.onError(Status.INTERNAL
+ .withDescription("No FTP Resource with id " + request.getResourceId())
+ .asRuntimeException()));
+ } catch (Exception e) {
+ logger.error("Failed in retrieving FTP resource with id {}", request.getResourceId(), e);
+
+ responseObserver.onError(Status.INTERNAL.withCause(e)
+ .withDescription("Failed in retrieving FTP resource with id " + request.getResourceId())
+ .asRuntimeException());
+ }
+ }
+
+ @Override
+ public void createFTPResource(FTPResourceCreateRequest request, StreamObserver<FTPResource> responseObserver) {
+ try {
+ responseObserver.onNext(this.backend.createFTPResource(request));
+ responseObserver.onCompleted();
+ } catch (Exception e) {
+ logger.error("Failed in creating the FTP resource", e);
+
+ responseObserver.onError(Status.INTERNAL.withCause(e)
+ .withDescription("Failed in creating the FTP resource")
+ .asRuntimeException());
+ }
+ }
+
+ @Override
+ public void updateFTPResource(FTPResourceUpdateRequest request, StreamObserver<Empty> responseObserver) {
+ try {
+ this.backend.updateFTPResource(request);
+ responseObserver.onCompleted();
+ } catch (Exception e) {
+ logger.error("Failed in updating the FTP resource {}", request.getResourceId(), e);
+
+ responseObserver.onError(Status.INTERNAL.withCause(e)
+ .withDescription("Failed in updating the FTP resource")
+ .asRuntimeException());
+ }
+ }
+
+ @Override
+ public void deleteFTPResource(FTPResourceDeleteRequest request, StreamObserver<Empty> responseObserver) {
+ try {
+ boolean res = this.backend.deleteFTPResource(request);
+ if (res) {
+ responseObserver.onCompleted();
+ } else {
+
+ responseObserver.onError(Status.INTERNAL
+ .withDescription("Failed to delete FTP Resource with id " + request.getResourceId())
+ .asRuntimeException());
+ }
+ } catch (Exception e) {
+ logger.error("Failed in deleting the scp resource {}", request.getResourceId(), e);
+
+ responseObserver.onError(Status.INTERNAL.withCause(e)
+ .withDescription("Failed in deleting the FTP resource")
+ .asRuntimeException());
+ }
+ }
}
[airavata-mft] 07/15: FTP transport implementation
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit a57b554e5c5c7b419655fbbc6e8d21dc29694247
Author: Gopi Kiran <go...@gmail.com>
AuthorDate: Wed Apr 29 16:13:34 2020 -0400
FTP transport implementation
---
transport/ftp-transport/pom.xml | 49 ++++++++++
.../mft/transport/ftp/FTPMetadataCollector.java | 106 +++++++++++++++++++++
.../airavata/mft/transport/ftp/FTPReceiver.java | 89 +++++++++++++++++
.../airavata/mft/transport/ftp/FTPSender.java | 91 ++++++++++++++++++
.../mft/transport/ftp/FTPTransportUtil.java | 35 +++++++
5 files changed, 370 insertions(+)
diff --git a/transport/ftp-transport/pom.xml b/transport/ftp-transport/pom.xml
new file mode 100644
index 0000000..347cbc0
--- /dev/null
+++ b/transport/ftp-transport/pom.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>mft-transport</artifactId>
+ <groupId>org.apache.airavata</groupId>
+ <version>0.01-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>mft-ftp-transport</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>mft-core</artifactId>
+ <version>0.01-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-net</groupId>
+ <artifactId>commons-net</artifactId>
+ <version>3.6</version>
+ </dependency>
+
+ </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java
new file mode 100644
index 0000000..8b7c9df
--- /dev/null
+++ b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java
@@ -0,0 +1,106 @@
+package org.apache.airavata.mft.transport.ftp;
+
+import org.apache.airavata.mft.core.ResourceMetadata;
+import org.apache.airavata.mft.core.api.MetadataCollector;
+import org.apache.airavata.mft.resource.client.ResourceServiceClient;
+import org.apache.airavata.mft.resource.service.FTPResource;
+import org.apache.airavata.mft.resource.service.FTPResourceGetRequest;
+import org.apache.airavata.mft.resource.service.ResourceServiceGrpc;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.service.FTPSecret;
+import org.apache.airavata.mft.secret.service.FTPSecretGetRequest;
+import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
+import org.apache.commons.net.ftp.FTPClient;
+import org.apache.commons.net.ftp.FTPFile;
+import org.apache.commons.net.ftp.FTPReply;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.InputStream;
+
+public class FTPMetadataCollector implements MetadataCollector {
+
+ private static final Logger logger = LoggerFactory.getLogger(FTPMetadataCollector.class);
+
+ private String resourceServiceHost;
+ private int resourceServicePort;
+ private String secretServiceHost;
+ private int secretServicePort;
+ private boolean initialized = false;
+
+ @Override
+ public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) {
+ this.resourceServiceHost = resourceServiceHost;
+ this.resourceServicePort = resourceServicePort;
+ this.secretServiceHost = secretServiceHost;
+ this.secretServicePort = secretServicePort;
+ this.initialized = true;
+ }
+
+ private void checkInitialized() {
+ if (!initialized) {
+ throw new IllegalStateException("FTP Metadata Collector is not initialized");
+ }
+ }
+
+ @Override
+ public ResourceMetadata getGetResourceMetadata(String resourceId, String credentialToken) {
+
+ checkInitialized();
+ ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient(resourceServiceHost, resourceServicePort);
+ FTPResource ftpResource = resourceClient.getFTPResource(FTPResourceGetRequest.newBuilder().setResourceId(resourceId).build());
+ SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient(secretServiceHost, secretServicePort);
+ FTPSecret ftpSecret = secretClient.getFTPSecret(FTPSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+
+ ResourceMetadata resourceMetadata = new ResourceMetadata();
+ FTPClient ftpClient = null;
+ try {
+ ftpClient = FTPTransportUtil.getFTPClient(ftpResource, ftpSecret);
+ logger.info("Fetching metadata for resource {} in {}", ftpResource.getResourcePath(), ftpResource.getFtpStorage().getHost());
+
+ FTPFile ftpFile = ftpClient.mlistFile(ftpResource.getResourcePath());
+
+ if (ftpFile != null) {
+ resourceMetadata.setResourceSize(ftpFile.getSize());
+ resourceMetadata.setUpdateTime(ftpFile.getTimestamp().getTimeInMillis());
+ if (ftpClient.hasFeature("MD5") && FTPReply.isPositiveCompletion(ftpClient.sendCommand("MD5 " + ftpResource.getResourcePath()))) {
+ String[] replies = ftpClient.getReplyStrings();
+ resourceMetadata.setMd5sum(replies[0]);
+ } else {
+ logger.warn("MD5 fetch error out {}", ftpClient.getReplyString());
+ }
+ }
+ } catch (Exception e) {
+ logger.warn("Failed to fetch md5 for FTP resource {}", resourceId, e);
+ } finally {
+ FTPTransportUtil.disconnectFTP(ftpClient);
+ }
+
+ return resourceMetadata;
+ }
+
+ @Override
+ public Boolean isAvailable(String resourceId, String credentialToken) {
+
+ checkInitialized();
+
+ ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient(resourceServiceHost, resourceServicePort);
+ FTPResource ftpResource = resourceClient.getFTPResource(FTPResourceGetRequest.newBuilder().setResourceId(resourceId).build());
+ SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient(secretServiceHost, secretServicePort);
+ FTPSecret ftpSecret = secretClient.getFTPSecret(FTPSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+
+ FTPClient ftpClient = null;
+ try {
+ ftpClient = FTPTransportUtil.getFTPClient(ftpResource, ftpSecret);
+ InputStream inputStream = ftpClient.retrieveFileStream(ftpResource.getResourcePath());
+
+ return !(inputStream == null || ftpClient.getReplyCode() == 550);
+ } catch (Exception e) {
+ logger.error("FTP client initialization failed ", e);
+ return false;
+ } finally {
+ FTPTransportUtil.disconnectFTP(ftpClient);
+ }
+ }
+}
+
diff --git a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java
new file mode 100644
index 0000000..46909dd
--- /dev/null
+++ b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java
@@ -0,0 +1,89 @@
+package org.apache.airavata.mft.transport.ftp;
+
+import org.apache.airavata.mft.core.ConnectorContext;
+import org.apache.airavata.mft.core.api.Connector;
+import org.apache.airavata.mft.resource.client.ResourceServiceClient;
+import org.apache.airavata.mft.resource.service.FTPResource;
+import org.apache.airavata.mft.resource.service.FTPResourceGetRequest;
+import org.apache.airavata.mft.resource.service.ResourceServiceGrpc;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.service.FTPSecret;
+import org.apache.airavata.mft.secret.service.FTPSecretGetRequest;
+import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
+import org.apache.commons.net.ftp.FTPClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+
+public class FTPReceiver implements Connector {
+
+ private static final Logger logger = LoggerFactory.getLogger(FTPReceiver.class);
+
+ private FTPResource resource;
+ private boolean initialized;
+ private FTPClient ftpClient;
+
+ @Override
+ public void init(String resourceId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception {
+ this.initialized = true;
+
+ ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient(resourceServiceHost, resourceServicePort);
+ this.resource = resourceClient.getFTPResource(FTPResourceGetRequest.newBuilder().setResourceId(resourceId).build());
+
+ SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient(secretServiceHost, secretServicePort);
+ FTPSecret ftpSecret = secretClient.getFTPSecret(FTPSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+
+ this.ftpClient = FTPTransportUtil.getFTPClient(this.resource, ftpSecret);
+ }
+
+ @Override
+ public void destroy() {
+ FTPTransportUtil.disconnectFTP(ftpClient);
+ }
+
+ @Override
+ public void startStream(ConnectorContext context) throws Exception {
+ logger.info("Starting FTP receiver stream for transfer {}", context.getTransferId());
+
+ checkInitialized();
+ OutputStream streamOs = context.getStreamBuffer().getOutputStream();
+ InputStream inputStream = ftpClient.retrieveFileStream(resource.getResourcePath());
+
+ long fileSize = context.getMetadata().getResourceSize();
+
+ byte[] buf = new byte[1024];
+ while (true) {
+ int bufSize;
+
+ if (buf.length < fileSize) {
+ bufSize = buf.length;
+ } else {
+ bufSize = (int) fileSize;
+ }
+ bufSize = inputStream.read(buf, 0, bufSize);
+
+ if (bufSize < 0) {
+ break;
+ }
+
+ streamOs.write(buf, 0, bufSize);
+ streamOs.flush();
+
+ fileSize -= bufSize;
+ if (fileSize == 0L)
+ break;
+ }
+
+ inputStream.close();
+ streamOs.close();
+ logger.info("Completed FTP receiver stream for transfer {}", context.getTransferId());
+ }
+
+ private void checkInitialized() {
+ if (!initialized) {
+ throw new IllegalStateException("FTP Receiver is not initialized");
+ }
+ }
+}
diff --git a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java
new file mode 100644
index 0000000..41d4b8e
--- /dev/null
+++ b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java
@@ -0,0 +1,91 @@
+package org.apache.airavata.mft.transport.ftp;
+
+import org.apache.airavata.mft.core.ConnectorContext;
+import org.apache.airavata.mft.core.api.Connector;
+import org.apache.airavata.mft.resource.client.ResourceServiceClient;
+import org.apache.airavata.mft.resource.service.FTPResource;
+import org.apache.airavata.mft.resource.service.FTPResourceGetRequest;
+import org.apache.airavata.mft.resource.service.ResourceServiceGrpc;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.service.FTPSecret;
+import org.apache.airavata.mft.secret.service.FTPSecretGetRequest;
+import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
+import org.apache.commons.net.ftp.FTPClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+
+public class FTPSender implements Connector {
+
+ private static final Logger logger = LoggerFactory.getLogger(FTPReceiver.class);
+
+ private FTPResource resource;
+ private boolean initialized;
+ private FTPClient ftpClient;
+
+ @Override
+ public void init(String resourceId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception {
+ this.initialized = true;
+
+ ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient(resourceServiceHost, resourceServicePort);
+ this.resource = resourceClient.getFTPResource(FTPResourceGetRequest.newBuilder().setResourceId(resourceId).build());
+
+ SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient(secretServiceHost, secretServicePort);
+ FTPSecret ftpSecret = secretClient.getFTPSecret(FTPSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+
+ this.ftpClient = FTPTransportUtil.getFTPClient(this.resource, ftpSecret);
+ }
+
+ @Override
+ public void destroy() {
+ FTPTransportUtil.disconnectFTP(ftpClient);
+ }
+
+ @Override
+ public void startStream(ConnectorContext context) throws Exception {
+
+ logger.info("Starting FTP sender stream for transfer {}", context.getTransferId());
+
+ checkInitialized();
+ InputStream in = context.getStreamBuffer().getInputStream();
+ long fileSize = context.getMetadata().getResourceSize();
+ OutputStream outputStream = ftpClient.storeFileStream(resource.getResourcePath());
+
+ byte[] buf = new byte[1024];
+ while (true) {
+ int bufSize;
+
+ if (buf.length < fileSize) {
+ bufSize = buf.length;
+ } else {
+ bufSize = (int) fileSize;
+ }
+ bufSize = in.read(buf, 0, bufSize);
+
+ if (bufSize < 0) {
+ break;
+ }
+
+ outputStream.write(buf, 0, bufSize);
+ outputStream.flush();
+
+ fileSize -= bufSize;
+ if (fileSize == 0L)
+ break;
+ }
+
+ in.close();
+ outputStream.close();
+
+ logger.info("Completed FTP sender stream for transfer {}", context.getTransferId());
+
+ }
+
+ private void checkInitialized() {
+ if (!initialized) {
+ throw new IllegalStateException("FTP Sender is not initialized");
+ }
+ }
+}
diff --git a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPTransportUtil.java b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPTransportUtil.java
new file mode 100644
index 0000000..81608a4
--- /dev/null
+++ b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPTransportUtil.java
@@ -0,0 +1,35 @@
+package org.apache.airavata.mft.transport.ftp;
+
+import org.apache.airavata.mft.resource.service.FTPResource;
+import org.apache.airavata.mft.secret.service.FTPSecret;
+import org.apache.commons.net.ftp.FTPClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+class FTPTransportUtil {
+
+ private static final Logger logger = LoggerFactory.getLogger(FTPTransportUtil.class);
+
+ static FTPClient getFTPClient(FTPResource ftpResource, FTPSecret ftpSecret) throws IOException {
+
+ FTPClient ftpClient = new FTPClient();
+ ftpClient.connect(ftpResource.getFtpStorage().getHost(), ftpResource.getFtpStorage().getPort());
+ ftpClient.enterLocalActiveMode();
+ ftpClient.login(ftpSecret.getUserId(), ftpSecret.getPassword());
+
+ return ftpClient;
+ }
+
+ static void disconnectFTP(FTPClient ftpClient) {
+ try {
+ if (ftpClient != null) {
+ ftpClient.logout();
+ ftpClient.disconnect();
+ }
+ } catch (Exception e) {
+ logger.error("FTP client close operation failed", e);
+ }
+ }
+}
[airavata-mft] 08/15: Adding FTP in application properties of agent
module
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit 7130037dad616c97f685204ab1507653e81bfd34
Author: Gopi Kiran <go...@gmail.com>
AuthorDate: Wed Apr 29 18:09:16 2020 -0400
Adding FTP in application properties of agent module
---
agent/src/main/resources/application.properties | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/agent/src/main/resources/application.properties b/agent/src/main/resources/application.properties
index e8883b1..33d156e 100644
--- a/agent/src/main/resources/application.properties
+++ b/agent/src/main/resources/application.properties
@@ -19,7 +19,7 @@ spring.main.web-application-type=NONE
agent.id=agent0
agent.host=localhost
agent.user=dimuthu
-agent.supported.protocols=SCP,LOCAL
+agent.supported.protocols=SCP,LOCAL,FTP
consul.host=localhost
consul.port=8500
[airavata-mft] 09/15: adding FTP sender,
receiver and metadatadata collector to the core
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit d3520b9d74d1d8b08373cdf71760c85cc778c43e
Author: Gopi Kiran <go...@gmail.com>
AuthorDate: Wed Apr 29 18:12:46 2020 -0400
adding FTP sender, receiver and metadatadata collector to the core
---
.../java/org/apache/airavata/mft/core/ConnectorResolver.java | 10 ++++++++++
.../apache/airavata/mft/core/MetadataCollectorResolver.java | 3 +++
2 files changed, 13 insertions(+)
diff --git a/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java b/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java
index 6c21932..469d19e 100644
--- a/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java
+++ b/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java
@@ -97,6 +97,16 @@ public final class ConnectorResolver {
break;
}
break;
+ case "FTP":
+ switch (direction) {
+ case "IN":
+ className = "org.apache.airavata.mft.transport.ftp.FTPReceiver";
+ break;
+ case "OUT":
+ className = "org.apache.airavata.mft.transport.ftp.FTPSender";
+ break;
+ }
+ break;
}
if (className != null) {
diff --git a/core/src/main/java/org/apache/airavata/mft/core/MetadataCollectorResolver.java b/core/src/main/java/org/apache/airavata/mft/core/MetadataCollectorResolver.java
index fefcd5a..c9da731 100644
--- a/core/src/main/java/org/apache/airavata/mft/core/MetadataCollectorResolver.java
+++ b/core/src/main/java/org/apache/airavata/mft/core/MetadataCollectorResolver.java
@@ -48,6 +48,9 @@ public final class MetadataCollectorResolver {
case "DROPBOX":
className = "org.apache.airavata.mft.transport.dropbox.DropboxMetadataCollector";
break;
+ case "FTP":
+ className = "org.apache.airavata.mft.transport.ftp.FTPMetadataCollector";
+ break;
}
if (className != null) {
[airavata-mft] 12/15: individual imports and license header addition
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit 3d621d51791f3ea76e3913e8c02b46090c1c9727
Author: Gopi Kiran <go...@gmail.com>
AuthorDate: Fri May 1 15:23:05 2020 -0400
individual imports and license header addition
---
.../server/backend/sql/SQLResourceBackend.java | 65 +++++++++++++++++++++-
.../backend/sql/entity/FTPResourceEntity.java | 17 ++++++
.../backend/sql/entity/FTPStorageEntity.java | 17 ++++++
.../sql/repository/FTPResourceRepository.java | 17 ++++++
.../sql/repository/FTPStorageRepository.java | 17 ++++++
.../server/backend/sql/entity/FTPSecretEntity.java | 17 ++++++
.../sql/repository/FTPSecretRepository.java | 17 ++++++
7 files changed, 164 insertions(+), 3 deletions(-)
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java
index adea1a0..113a7ee 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java
@@ -17,10 +17,69 @@
package org.apache.airavata.mft.resource.server.backend.sql;
+
+
import org.apache.airavata.mft.resource.server.backend.ResourceBackend;
-import org.apache.airavata.mft.resource.server.backend.sql.entity.*;
-import org.apache.airavata.mft.resource.server.backend.sql.repository.*;
-import org.apache.airavata.mft.resource.service.*;
+import org.apache.airavata.mft.resource.server.backend.sql.entity.FTPResourceEntity;
+import org.apache.airavata.mft.resource.server.backend.sql.entity.FTPStorageEntity;
+import org.apache.airavata.mft.resource.server.backend.sql.entity.LocalResourceEntity;
+import org.apache.airavata.mft.resource.server.backend.sql.entity.SCPResourceEntity;
+import org.apache.airavata.mft.resource.server.backend.sql.entity.SCPStorageEntity;
+import org.apache.airavata.mft.resource.server.backend.sql.repository.FTPResourceRepository;
+import org.apache.airavata.mft.resource.server.backend.sql.repository.FTPStorageRepository;
+import org.apache.airavata.mft.resource.server.backend.sql.repository.LocalResourceRepository;
+import org.apache.airavata.mft.resource.server.backend.sql.repository.SCPResourceRepository;
+import org.apache.airavata.mft.resource.server.backend.sql.repository.SCPStorageRepository;
+import org.apache.airavata.mft.resource.service.AzureResource;
+import org.apache.airavata.mft.resource.service.AzureResourceCreateRequest;
+import org.apache.airavata.mft.resource.service.AzureResourceDeleteRequest;
+import org.apache.airavata.mft.resource.service.AzureResourceGetRequest;
+import org.apache.airavata.mft.resource.service.AzureResourceUpdateRequest;
+import org.apache.airavata.mft.resource.service.BoxResource;
+import org.apache.airavata.mft.resource.service.BoxResourceCreateRequest;
+import org.apache.airavata.mft.resource.service.BoxResourceDeleteRequest;
+import org.apache.airavata.mft.resource.service.BoxResourceGetRequest;
+import org.apache.airavata.mft.resource.service.BoxResourceUpdateRequest;
+import org.apache.airavata.mft.resource.service.DropboxResource;
+import org.apache.airavata.mft.resource.service.DropboxResourceCreateRequest;
+import org.apache.airavata.mft.resource.service.DropboxResourceDeleteRequest;
+import org.apache.airavata.mft.resource.service.DropboxResourceGetRequest;
+import org.apache.airavata.mft.resource.service.DropboxResourceUpdateRequest;
+import org.apache.airavata.mft.resource.service.FTPResource;
+import org.apache.airavata.mft.resource.service.FTPResourceCreateRequest;
+import org.apache.airavata.mft.resource.service.FTPResourceDeleteRequest;
+import org.apache.airavata.mft.resource.service.FTPResourceGetRequest;
+import org.apache.airavata.mft.resource.service.FTPResourceUpdateRequest;
+import org.apache.airavata.mft.resource.service.FTPStorage;
+import org.apache.airavata.mft.resource.service.FTPStorageCreateRequest;
+import org.apache.airavata.mft.resource.service.FTPStorageDeleteRequest;
+import org.apache.airavata.mft.resource.service.FTPStorageGetRequest;
+import org.apache.airavata.mft.resource.service.FTPStorageUpdateRequest;
+import org.apache.airavata.mft.resource.service.GCSResource;
+import org.apache.airavata.mft.resource.service.GCSResourceCreateRequest;
+import org.apache.airavata.mft.resource.service.GCSResourceDeleteRequest;
+import org.apache.airavata.mft.resource.service.GCSResourceGetRequest;
+import org.apache.airavata.mft.resource.service.GCSResourceUpdateRequest;
+import org.apache.airavata.mft.resource.service.LocalResource;
+import org.apache.airavata.mft.resource.service.LocalResourceCreateRequest;
+import org.apache.airavata.mft.resource.service.LocalResourceDeleteRequest;
+import org.apache.airavata.mft.resource.service.LocalResourceGetRequest;
+import org.apache.airavata.mft.resource.service.LocalResourceUpdateRequest;
+import org.apache.airavata.mft.resource.service.S3Resource;
+import org.apache.airavata.mft.resource.service.S3ResourceCreateRequest;
+import org.apache.airavata.mft.resource.service.S3ResourceDeleteRequest;
+import org.apache.airavata.mft.resource.service.S3ResourceGetRequest;
+import org.apache.airavata.mft.resource.service.S3ResourceUpdateRequest;
+import org.apache.airavata.mft.resource.service.SCPResource;
+import org.apache.airavata.mft.resource.service.SCPResourceCreateRequest;
+import org.apache.airavata.mft.resource.service.SCPResourceDeleteRequest;
+import org.apache.airavata.mft.resource.service.SCPResourceGetRequest;
+import org.apache.airavata.mft.resource.service.SCPResourceUpdateRequest;
+import org.apache.airavata.mft.resource.service.SCPStorage;
+import org.apache.airavata.mft.resource.service.SCPStorageCreateRequest;
+import org.apache.airavata.mft.resource.service.SCPStorageDeleteRequest;
+import org.apache.airavata.mft.resource.service.SCPStorageGetRequest;
+import org.apache.airavata.mft.resource.service.SCPStorageUpdateRequest;
import org.dozer.DozerBeanMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/FTPResourceEntity.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/FTPResourceEntity.java
index 5aba915..6f378aa 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/FTPResourceEntity.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/FTPResourceEntity.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.airavata.mft.resource.server.backend.sql.entity;
import org.hibernate.annotations.GenericGenerator;
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/FTPStorageEntity.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/FTPStorageEntity.java
index 2c42a8c..868ea7f 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/FTPStorageEntity.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/FTPStorageEntity.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.airavata.mft.resource.server.backend.sql.entity;
import org.hibernate.annotations.GenericGenerator;
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/FTPResourceRepository.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/FTPResourceRepository.java
index ad1f855..53d0222 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/FTPResourceRepository.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/FTPResourceRepository.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.airavata.mft.resource.server.backend.sql.repository;
import org.apache.airavata.mft.resource.server.backend.sql.entity.FTPResourceEntity;
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/FTPStorageRepository.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/FTPStorageRepository.java
index c80187a..152ae34 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/FTPStorageRepository.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/FTPStorageRepository.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.airavata.mft.resource.server.backend.sql.repository;
import org.apache.airavata.mft.resource.server.backend.sql.entity.FTPStorageEntity;
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/FTPSecretEntity.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/FTPSecretEntity.java
index 61ac8b8..7c3d839 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/FTPSecretEntity.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/FTPSecretEntity.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.airavata.mft.secret.server.backend.sql.entity;
import org.hibernate.annotations.GenericGenerator;
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/FTPSecretRepository.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/FTPSecretRepository.java
index 6236389..6eea16d 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/FTPSecretRepository.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/FTPSecretRepository.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.airavata.mft.secret.server.backend.sql.repository;
import org.apache.airavata.mft.secret.server.backend.sql.entity.FTPSecretEntity;
[airavata-mft] 06/15: FTP secret grpc method implementation and
reading resources from different backends
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit c43566349d41ca408f4c7b17d74f54a404e6dcbe
Author: Gopi Kiran <go...@gmail.com>
AuthorDate: Wed Apr 29 16:09:47 2020 -0400
FTP secret grpc method implementation and reading resources from different backends
---
.../mft/secret/server/backend/SecretBackend.java | 5 ++
.../backend/airavata/AiravataSecretBackend.java | 19 ++++++++
.../backend/file/FileBasedSecretBackend.java | 42 +++++++++++++++++
.../server/backend/sql/SQLSecretBackend.java | 28 ++++++++++++
.../server/backend/sql/entity/FTPSecretEntity.java | 48 ++++++++++++++++++++
.../sql/repository/FTPSecretRepository.java | 10 ++++
.../server/handler/SecretServiceHandler.java | 53 ++++++++++++++++++++++
7 files changed, 205 insertions(+)
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java
index 248f45b..81319ab 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java
@@ -55,4 +55,9 @@ public interface SecretBackend {
public DropboxSecret createDropboxSecret(DropboxSecretCreateRequest request) throws Exception;
public boolean updateDropboxSecret(DropboxSecretUpdateRequest request) throws Exception;
public boolean deleteDropboxSecret(DropboxSecretDeleteRequest request) throws Exception;
+
+ Optional<FTPSecret> getFTPSecret(FTPSecretGetRequest request) throws Exception;
+ FTPSecret createFTPSecret(FTPSecretCreateRequest request) throws Exception;
+ boolean updateFTPSecret(FTPSecretUpdateRequest request) throws Exception;
+ boolean deleteFTPSecret(FTPSecretDeleteRequest request) throws Exception;
}
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/airavata/AiravataSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/airavata/AiravataSecretBackend.java
index 83e8147..f2835df 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/airavata/AiravataSecretBackend.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/airavata/AiravataSecretBackend.java
@@ -180,5 +180,24 @@ public class AiravataSecretBackend implements SecretBackend {
throw new UnsupportedOperationException("Operation is not supported in backend");
}
+ @Override
+ public Optional<FTPSecret> getFTPSecret(FTPSecretGetRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
+
+ @Override
+ public FTPSecret createFTPSecret(FTPSecretCreateRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
+
+ @Override
+ public boolean updateFTPSecret(FTPSecretUpdateRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
+
+ @Override
+ public boolean deleteFTPSecret(FTPSecretDeleteRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
}
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/file/FileBasedSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/file/FileBasedSecretBackend.java
index bcd47c0..4dd23bf 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/file/FileBasedSecretBackend.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/file/FileBasedSecretBackend.java
@@ -33,6 +33,7 @@ import java.util.Optional;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
+@SuppressWarnings("unchecked")
public class FileBasedSecretBackend implements SecretBackend {
private static final Logger logger = LoggerFactory.getLogger(FileBasedSecretBackend.class);
@@ -292,5 +293,46 @@ public class FileBasedSecretBackend implements SecretBackend {
throw new UnsupportedOperationException("Operation is not supported in backend");
}
+ public Optional<FTPSecret> getFTPSecret(FTPSecretGetRequest request) throws Exception {
+ JSONParser jsonParser = new JSONParser();
+ InputStream inputStream = FileBasedSecretBackend.class.getClassLoader().getResourceAsStream(secretFile);
+
+ if (inputStream == null) {
+ throw new IOException("secrets file not found");
+ }
+
+ try (InputStreamReader reader = new InputStreamReader(inputStream)) {
+ Object obj = jsonParser.parse(reader);
+ JSONArray resourceList = (JSONArray) obj;
+
+ List<FTPSecret> ftpSecrets = (List<FTPSecret>) resourceList.stream()
+ .filter(resource -> "FTP".equals(((JSONObject) resource).get("type").toString()))
+ .map(resource -> {
+ JSONObject r = (JSONObject) resource;
+
+ return FTPSecret.newBuilder()
+ .setSecretId(r.get("secretId").toString())
+ .setUserId(r.get("userId").toString())
+ .setPassword(r.get("password").toString()).build();
+
+ }).collect(Collectors.toList());
+ return ftpSecrets.stream().filter(r -> request.getSecretId().equals(r.getSecretId())).findFirst();
+ }
+ }
+
+ @Override
+ public FTPSecret createFTPSecret(FTPSecretCreateRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
+
+ @Override
+ public boolean updateFTPSecret(FTPSecretUpdateRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
+
+ @Override
+ public boolean deleteFTPSecret(FTPSecretDeleteRequest request) {
+ throw new UnsupportedOperationException("Operation is not supported in backend");
+ }
}
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java
index f642f7c..23f55f0 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java
@@ -18,7 +18,9 @@
package org.apache.airavata.mft.secret.server.backend.sql;
import org.apache.airavata.mft.secret.server.backend.SecretBackend;
+import org.apache.airavata.mft.secret.server.backend.sql.entity.FTPSecretEntity;
import org.apache.airavata.mft.secret.server.backend.sql.entity.SCPSecretEntity;
+import org.apache.airavata.mft.secret.server.backend.sql.repository.FTPSecretRepository;
import org.apache.airavata.mft.secret.server.backend.sql.repository.SecretRepository;
import org.apache.airavata.mft.secret.service.*;
import org.dozer.DozerBeanMapper;
@@ -35,6 +37,9 @@ public class SQLSecretBackend implements SecretBackend {
@Autowired
private SecretRepository secretRepository;
+ @Autowired
+ private FTPSecretRepository ftpSecretRepository;
+
private DozerBeanMapper mapper = new DozerBeanMapper();
@Override
@@ -171,4 +176,27 @@ public class SQLSecretBackend implements SecretBackend {
throw new UnsupportedOperationException("Operation is not supported in backend");
}
+ @Override
+ public Optional<FTPSecret> getFTPSecret(FTPSecretGetRequest request) {
+ Optional<FTPSecretEntity> secretEty = ftpSecretRepository.findBySecretId(request.getSecretId());
+ return secretEty.map(ftpSecretEntity -> mapper.map(ftpSecretEntity, FTPSecret.newBuilder().getClass()).build());
+ }
+
+ @Override
+ public FTPSecret createFTPSecret(FTPSecretCreateRequest request) {
+ FTPSecretEntity savedEntity = ftpSecretRepository.save(mapper.map(request, FTPSecretEntity.class));
+ return mapper.map(savedEntity, FTPSecret.newBuilder().getClass()).build();
+ }
+
+ @Override
+ public boolean updateFTPSecret(FTPSecretUpdateRequest request) {
+ ftpSecretRepository.save(mapper.map(request, FTPSecretEntity.class));
+ return true;
+ }
+
+ @Override
+ public boolean deleteFTPSecret(FTPSecretDeleteRequest request) {
+ ftpSecretRepository.deleteById(request.getSecretId());
+ return true;
+ }
}
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/FTPSecretEntity.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/FTPSecretEntity.java
new file mode 100644
index 0000000..61ac8b8
--- /dev/null
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/FTPSecretEntity.java
@@ -0,0 +1,48 @@
+package org.apache.airavata.mft.secret.server.backend.sql.entity;
+
+import org.hibernate.annotations.GenericGenerator;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+
+@Entity
+public class FTPSecretEntity {
+
+ @Id
+ @Column(name = "SECRET_ID")
+ @GeneratedValue(generator = "uuid")
+ @GenericGenerator(name = "uuid", strategy = "uuid2")
+ private String secretId;
+
+ @Column(name = "USER_ID")
+ private String userId;
+
+ @Column(name = "PASSWORD")
+ private String password;
+
+ public String getSecretId() {
+ return secretId;
+ }
+
+ public void setSecretId(String secretId) {
+ this.secretId = secretId;
+ }
+
+ public String getUserId() {
+ return userId;
+ }
+
+ public void setUserId(String userId) {
+ this.userId = userId;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+}
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/FTPSecretRepository.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/FTPSecretRepository.java
new file mode 100644
index 0000000..6236389
--- /dev/null
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/FTPSecretRepository.java
@@ -0,0 +1,10 @@
+package org.apache.airavata.mft.secret.server.backend.sql.repository;
+
+import org.apache.airavata.mft.secret.server.backend.sql.entity.FTPSecretEntity;
+import org.springframework.data.repository.CrudRepository;
+
+import java.util.Optional;
+
+public interface FTPSecretRepository extends CrudRepository<FTPSecretEntity, String> {
+ Optional<FTPSecretEntity> findBySecretId(String secretId);
+}
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java
index 3345db7..cac5373 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java
@@ -365,5 +365,58 @@ public class SecretServiceHandler extends SecretServiceGrpc.SecretServiceImplBas
}
}
+ @Override
+ public void getFTPSecret(FTPSecretGetRequest request, StreamObserver<FTPSecret> responseObserver) {
+ try {
+ this.backend.getFTPSecret(request).ifPresentOrElse(secret -> {
+ responseObserver.onNext(secret);
+ responseObserver.onCompleted();
+ }, () -> responseObserver.onError(Status.INTERNAL
+ .withDescription("No FTP Secret with id " + request.getSecretId())
+ .asRuntimeException()));
+
+ } catch (Exception e) {
+ logger.error("Error in retrieving FTP Secret with id " + request.getSecretId(), e);
+ responseObserver.onError(Status.INTERNAL.withCause(e)
+ .withDescription("Error in retrieving FTP Secret with id " + request.getSecretId())
+ .asRuntimeException());
+ }
+ }
+
+ @Override
+ public void createFTPSecret(FTPSecretCreateRequest request, StreamObserver<FTPSecret> responseObserver) {
+ try {
+ this.backend.createFTPSecret(request);
+ } catch (Exception e) {
+ logger.error("Error in creating FTP Secret", e);
+ responseObserver.onError(Status.INTERNAL.withCause(e)
+ .withDescription("Error in creating FTP Secret")
+ .asRuntimeException());
+ }
+ }
+
+ @Override
+ public void updateFTPSecret(FTPSecretUpdateRequest request, StreamObserver<Empty> responseObserver) {
+ try {
+ this.backend.updateFTPSecret(request);
+ } catch (Exception e) {
+ logger.error("Error in updating FTP Secret with id {}", request.getSecretId(), e);
+ responseObserver.onError(Status.INTERNAL.withCause(e)
+ .withDescription("Error in updating FTP Secret with id " + request.getSecretId())
+ .asRuntimeException());
+ }
+ }
+
+ @Override
+ public void deleteFTPSecret(FTPSecretDeleteRequest request, StreamObserver<Empty> responseObserver) {
+ try {
+ this.backend.deleteFTPSecret(request);
+ } catch (Exception e) {
+ logger.error("Error in deleting FTP Secret with id {}", request.getSecretId(), e);
+ responseObserver.onError(Status.INTERNAL.withCause(e)
+ .withDescription("Error in deleting FTP Secret with id " + request.getSecretId())
+ .asRuntimeException());
+ }
+ }
}
[airavata-mft] 13/15: using resource file variable
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit 22054a1469ff379459560b301caca72f5f2cb913
Author: Gopi Kiran <go...@gmail.com>
AuthorDate: Fri May 1 15:28:54 2020 -0400
using resource file variable
---
.../mft/resource/server/backend/file/FileBasedResourceBackend.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java
index ae29eb4..60a3df9 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java
@@ -388,7 +388,7 @@ public class FileBasedResourceBackend implements ResourceBackend {
@Override
public Optional<FTPResource> getFTPResource(FTPResourceGetRequest request) throws Exception {
- InputStream inputStream = FileBasedResourceBackend.class.getClassLoader().getResourceAsStream("resources.json");
+ InputStream inputStream = FileBasedResourceBackend.class.getClassLoader().getResourceAsStream(resourceFile);
JSONParser jsonParser = new JSONParser();
[airavata-mft] 02/15: secrets service proto FTP changes
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit f803e24c6ee72f05f77733569a53aac0c8fe66ef
Author: Gopi Kiran <go...@gmail.com>
AuthorDate: Wed Apr 29 15:40:31 2020 -0400
secrets service proto FTP changes
---
.../stub/src/main/proto/SecretService.proto | 56 ++++++++++++++++++++++
1 file changed, 56 insertions(+)
diff --git a/services/secret-service/stub/src/main/proto/SecretService.proto b/services/secret-service/stub/src/main/proto/SecretService.proto
index fd7e897..b933df9 100644
--- a/services/secret-service/stub/src/main/proto/SecretService.proto
+++ b/services/secret-service/stub/src/main/proto/SecretService.proto
@@ -200,6 +200,36 @@ message DropboxSecretDeleteRequest {
AuthToken authzToken = 2;
}
+// FTP
+message FTPSecret {
+ string secretId = 1;
+ string userId = 2;
+ string password = 3;
+}
+
+message FTPSecretGetRequest {
+ string secretId = 1;
+ AuthToken authzToken = 2;
+}
+
+message FTPSecretCreateRequest {
+ string userId = 1;
+ string password = 2;
+ AuthToken authzToken = 3;
+}
+
+message FTPSecretUpdateRequest {
+ string secretId = 1;
+ string userId = 2;
+ string password = 3;
+ AuthToken authzToken = 4;
+}
+
+message FTPSecretDeleteRequest {
+ string secretId = 1;
+ AuthToken authzToken = 4;
+}
+
service SecretService {
rpc getSCPSecret (SCPSecretGetRequest) returns (SCPSecret) {
option (google.api.http) = {
@@ -351,4 +381,30 @@ service SecretService {
};
}
+ // FTP
+
+ rpc getFTPSecret (FTPSecretGetRequest) returns (FTPSecret) {
+ option (google.api.http) = {
+ get: "/v1.0/secret/ftp"
+ };
+ }
+
+ rpc createFTPSecret (FTPSecretCreateRequest) returns (FTPSecret) {
+ option (google.api.http) = {
+ post: "/v1.0/secret/ftp"
+ };
+ }
+
+ rpc updateFTPSecret (FTPSecretUpdateRequest) returns (google.protobuf.Empty) {
+ option (google.api.http) = {
+ put: "/v1.0/secret/ftp"
+ };
+ }
+
+ rpc deleteFTPSecret (FTPSecretDeleteRequest) returns (google.protobuf.Empty) {
+ option (google.api.http) = {
+ delete: "/v1.0/secret/ftp"
+ };
+ }
+
}
\ No newline at end of file
[airavata-mft] 14/15: adding FTP to the application properties
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit e61eea9610cb80911fd5733af7832e40c175e0fe
Author: Gopi Kiran <go...@gmail.com>
AuthorDate: Sat May 2 16:15:58 2020 -0400
adding FTP to the application properties
---
agent/src/main/resources/distribution/conf/application.properties | 2 +-
client-sdk/mft-python-sdk/samples/mft_api_client_sample.py | 0
2 files changed, 1 insertion(+), 1 deletion(-)
diff --git a/agent/src/main/resources/distribution/conf/application.properties b/agent/src/main/resources/distribution/conf/application.properties
index e8883b1..33d156e 100644
--- a/agent/src/main/resources/distribution/conf/application.properties
+++ b/agent/src/main/resources/distribution/conf/application.properties
@@ -19,7 +19,7 @@ spring.main.web-application-type=NONE
agent.id=agent0
agent.host=localhost
agent.user=dimuthu
-agent.supported.protocols=SCP,LOCAL
+agent.supported.protocols=SCP,LOCAL,FTP
consul.host=localhost
consul.port=8500
diff --git a/client-sdk/mft-python-sdk/samples/mft_api_client_sample.py b/client-sdk/mft-python-sdk/samples/mft_api_client_sample.py
new file mode 100644
index 0000000..e69de29
[airavata-mft] 10/15: adding missing dependency
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit a4635b5f74612cb161c30911f4792dbf864e2b47
Author: Gopi Kiran <go...@gmail.com>
AuthorDate: Thu Apr 30 18:59:58 2020 -0400
adding missing dependency
---
agent/pom.xml | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/agent/pom.xml b/agent/pom.xml
index 3499db1..7ca3753 100644
--- a/agent/pom.xml
+++ b/agent/pom.xml
@@ -70,6 +70,11 @@
</dependency>
<dependency>
<groupId>org.apache.airavata</groupId>
+ <artifactId>mft-ftp-transport</artifactId>
+ <version>0.01-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
<artifactId>mft-admin</artifactId>
<version>0.01-SNAPSHOT</version>
</dependency>
[airavata-mft] 15/15: adding ftp module to the transport build
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit 130d5a9541a3121430e14b97bf2318f76eb3bbe1
Author: Gopi Kiran <go...@gmail.com>
AuthorDate: Sat May 2 16:16:43 2020 -0400
adding ftp module to the transport build
---
transport/pom.xml | 1 +
1 file changed, 1 insertion(+)
diff --git a/transport/pom.xml b/transport/pom.xml
index 0db98a5..d53cfca 100755
--- a/transport/pom.xml
+++ b/transport/pom.xml
@@ -39,6 +39,7 @@
<module>box-transport</module>
<module>azure-transport</module>
<module>gcp-transport</module>
+ <module>ftp-transport</module>
<module>dropbox-transport</module>
</modules>
<dependencies>
[airavata-mft] 04/15: adding FTP resource file sample data
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit 3e95f8d185e369aa746964a4ca90c5534a6d6ca6
Author: Gopi Kiran <go...@gmail.com>
AuthorDate: Wed Apr 29 15:43:55 2020 -0400
adding FTP resource file sample data
---
.../resource-service/server/src/main/resources/resources.json | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/services/resource-service/server/src/main/resources/resources.json b/services/resource-service/server/src/main/resources/resources.json
index 8efb247..5c960fb 100644
--- a/services/resource-service/server/src/main/resources/resources.json
+++ b/services/resource-service/server/src/main/resources/resources.json
@@ -59,5 +59,15 @@
"type": "DROPBOX",
"resourceId": "dropbox-file",
"resourcePath": "/test.txt"
+ },
+ {
+ "type": "FTP",
+ "resourceId": "ftp-resource",
+ "resourcePath": "mft-1mb.txt",
+ "ftpStorage": {
+ "storageId": "ftp-resource",
+ "host": "ftp.dlptest.com",
+ "port": "21"
+ }
}
]
\ No newline at end of file