You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by is...@apache.org on 2021/07/09 15:18:22 UTC
[airavata-data-lake] branch master updated: drms storage and
storage-preference
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
The following commit(s) were added to refs/heads/master by this push:
new 1139218 drms storage and storage-preference
new ccbb746 Merge pull request #21 from isururanawaka/workflow_invocation
1139218 is described below
commit 1139218e74413b30ddcb9a9bb06e28b09b3d9903
Author: Isuru Ranawaka <ir...@gmail.com>
AuthorDate: Fri Jul 9 11:16:02 2021 -0400
drms storage and storage-preference
---
.../src/main/resources/config.yml | 2 +-
.../drms/api/handlers/ResourceServiceHandler.java | 45 ++++++++++++---------
.../handlers/StoragePreferenceServiceHandler.java | 44 +++++++++++++++-----
.../drms-rest-proxy/src/main/resources/drms.pb | Bin 109380 -> 109388 bytes
.../src/main/proto/resource/DRMSResource.proto | 10 ++---
5 files changed, 65 insertions(+), 36 deletions(-)
diff --git a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/resources/config.yml b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/resources/config.yml
index 2d11996..43fb605 100644
--- a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/resources/config.yml
+++ b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/resources/config.yml
@@ -12,6 +12,6 @@ outboundEventProcessor:
drmsPort: 7070
consumer:
brokerURL: "149.165.157.235:9092"
- consumerGroup: "DataOrchestratorGroup"
+ consumerGroup: "DataOrchestratorGroup2"
topic: "data-orchestrator-file-events"
maxPollRecordsConfig: 10
\ No newline at end of file
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
index 214a6d7..d97f29d 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
@@ -29,8 +29,8 @@ import org.apache.airavata.drms.api.utils.CustosUtils;
import org.apache.airavata.drms.api.utils.Utils;
import org.apache.airavata.drms.core.Neo4JConnector;
import org.apache.airavata.drms.core.constants.StoragePreferenceConstants;
+import org.apache.airavata.drms.core.deserializer.AnyStorageDeserializer;
import org.apache.airavata.drms.core.deserializer.GenericResourceDeserializer;
-import org.apache.airavata.drms.core.deserializer.TransferMappingDeserializer;
import org.apache.airavata.drms.core.serializer.GenericResourceSerializer;
import org.apache.custos.clients.CustosClientProvider;
import org.apache.custos.sharing.service.Entity;
@@ -90,28 +90,16 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
List<GenericResource> genericResourceList = GenericResourceDeserializer.deserializeList(records);
ResourceFetchResponse.Builder builder = ResourceFetchResponse.newBuilder();
if (!genericResourceList.isEmpty()) {
- // TODO: Move to Storage
- String searchQuery = "Match (srcStr:Storage)<-[:CHILD_OF]-" +
- "(srcSp:StoragePreference)-[:TRANSFER_OUT]->(t:TransferMapping" +
- "{scope:'GLOBAL', tenantId:$tenantId})-[:TRANSFER_IN]->(dstSp:StoragePreference)-[:CHILD_OF]->(dstStr:Storage)" +
- " return srcStr, srcSp, dstStr, dstSp, t";
-
- List<Record> globalRecords = this.neo4JConnector.searchNodes(userProps, searchQuery);
-
- if (!globalRecords.isEmpty()) {
- List<TransferMapping> transferMappings = TransferMappingDeserializer.deserializeList(globalRecords);
- if (!transferMappings.isEmpty()) {
- AnyStoragePreference anyStoragePreference = transferMappings.get(0)
- .getSourceStoragePreference();
- GenericResource resource = genericResourceList.get(0)
- .toBuilder()
- .setSshPreference(anyStoragePreference.getSshStoragePreference())
- .build();
- builder.setResource(resource);
+ Optional<AnyStorage> anyStorage = findStorage(resourceId, type, callUser.getTenantId());
+ GenericResource resource = genericResourceList.get(0);
+ if (anyStorage.isPresent()) {
+ if (anyStorage.get().getStorageCase().equals(AnyStorage.StorageCase.SSH_STORAGE)) {
+ resource = resource.toBuilder().setSshStorage(anyStorage.get().getSshStorage()).build();
} else {
- builder.setResource(genericResourceList.get(0));
+ resource = resource.toBuilder().setS3Storage(anyStorage.get().getS3Storage()).build();
}
}
+ builder.setResource(resource);
}
responseObserver.onNext(builder.build());
responseObserver.onCompleted();
@@ -839,4 +827,21 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
return Optional.empty();
}
+
+ private Optional<AnyStorage> findStorage(String entityId, String type, String tenantId) throws Exception {
+ Map<String, Object> parameters = new HashMap<>();
+ parameters.put("entityId", entityId);
+ parameters.put("tenantId", tenantId);
+ String query = " Match (r" + type + ") where r.entityId=$entityId and r.tenantId=$tenantId " +
+ " Match (s:Storage)<-[:CHILD_OF*]-(r) return (s)";
+
+ List<Record> records = this.neo4JConnector.searchNodes(parameters, query);
+ if (!records.isEmpty()) {
+ List<AnyStorage> storageList = AnyStorageDeserializer.deserializeList(records);
+ return Optional.ofNullable(storageList.get(0));
+
+ }
+ return Optional.empty();
+ }
+
}
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java
index ef07212..7221e6d 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java
@@ -221,16 +221,40 @@ public class StoragePreferenceServiceHandler extends StoragePreferenceServiceGrp
userProps.put("username", callUser.getUsername());
userProps.put("tenantId", callUser.getTenantId());
- List<Record> records = this.neo4JConnector.searchNodes(userProps,
- " MATCH (u:User) where u.username = $username AND u.tenantId = $tenantId" +
- " OPTIONAL MATCH (u)<-[:SHARED_WITH]-(s1:Storage)<-[:CHILD_OF]->(sp1:StoragePreference)" +
- " OPTIONAL MATCH (cg:Group)-[:CHILD_OF*]->(g:Group)<-[:MEMBER_OF]-(u)" +
- " OPTIONAL MATCH (sp2:StoragePreference)-[:CHILD_OF]->(s2:Storage)-[:SHARED_WITH]->(cg) " +
- " OPTIONAL MATCH (sp3:StoragePreference)-[:CHILD_OF]->(s3:Storage)-[:SHARED_WITH]->(g)" +
- " OPTIONAL MATCH (s4:Storage)<-[:CHILD_OF]->(sp4:StoragePreference)-[:SHARED_WITH]->(u)" +
- " OPTIONAL MATCH (s5:Storage)<-[:CHILD_OF]->(sp5:StoragePreference)-[:SHARED_WITH]->(cg)" +
- " OPTIONAL MATCH (s6:Storage)<-[:CHILD_OF]->(sp6:StoragePreference)-[:SHARED_WITH]->(g)" +
- " return distinct s1, sp1, s2, sp2, s3, sp3, s4,sp4, s5,sp5, s6,sp6");
+ String query = " MATCH (u:User) where u.username = $username AND u.tenantId = $tenantId" +
+ " OPTIONAL MATCH (u)<-[:SHARED_WITH]-(s1:Storage)<-[:CHILD_OF]->(sp1:StoragePreference)" +
+ " OPTIONAL MATCH (cg:Group)-[:CHILD_OF*]->(g:Group)<-[:MEMBER_OF]-(u)" +
+ " OPTIONAL MATCH (sp2:StoragePreference)-[:CHILD_OF]->(s2:Storage)-[:SHARED_WITH]->(cg) " +
+ " OPTIONAL MATCH (sp3:StoragePreference)-[:CHILD_OF]->(s3:Storage)-[:SHARED_WITH]->(g)" +
+ " OPTIONAL MATCH (s4:Storage)<-[:CHILD_OF]->(sp4:StoragePreference)-[:SHARED_WITH]->(u)" +
+ " OPTIONAL MATCH (s5:Storage)<-[:CHILD_OF]->(sp5:StoragePreference)-[:SHARED_WITH]->(cg)" +
+ " OPTIONAL MATCH (s6:Storage)<-[:CHILD_OF]->(sp6:StoragePreference)-[:SHARED_WITH]->(g)" +
+ " return distinct s1, sp1, s2, sp2, s3, sp3, s4,sp4, s5,sp5, s6,sp6";
+
+
+ String storageId = null;
+ List<StoragePreferenceSearchQuery> storagePreferenceSearchQueries = request.getQueriesList();
+
+ for (StoragePreferenceSearchQuery searchQuery : storagePreferenceSearchQueries) {
+ if (searchQuery.getField().equals("storageId")) {
+ storageId = searchQuery.getValue();
+ }
+ }
+
+ if (storageId != null) {
+ query = " MATCH (u:User) where u.username = $username AND u.tenantId = $tenantId" +
+ " OPTIONAL MATCH (u)<-[:SHARED_WITH]-(s1:Storage{storageId:'" + storageId + "'})<-[:CHILD_OF]->(sp1:StoragePreference)" +
+ " OPTIONAL MATCH (cg:Group)-[:CHILD_OF*]->(g:Group)<-[:MEMBER_OF]-(u)" +
+ " OPTIONAL MATCH (sp2:StoragePreference)-[:CHILD_OF]->(s2:Storage{storageId:'" + storageId + "'})-[:SHARED_WITH]->(cg) " +
+ " OPTIONAL MATCH (sp3:StoragePreference)-[:CHILD_OF]->(s3:Storage{storageId:'" + storageId + "'})-[:SHARED_WITH]->(g)" +
+ " OPTIONAL MATCH (s4:Storage{storageId:'" + storageId + "'})<-[:CHILD_OF]->(sp4:StoragePreference)-[:SHARED_WITH]->(u)" +
+ " OPTIONAL MATCH (s5:Storage{storageId:'" + storageId + "'})<-[:CHILD_OF]->(sp5:StoragePreference)-[:SHARED_WITH]->(cg)" +
+ " OPTIONAL MATCH (s6:Storage{storageId:'" + storageId + "'})<-[:CHILD_OF]->(sp6:StoragePreference)-[:SHARED_WITH]->(g)" +
+ " return distinct s1, sp1, s2, sp2, s3, sp3, s4,sp4, s5,sp5, s6,sp6";
+ }
+
+ List<Record> records = this.neo4JConnector.searchNodes(userProps, query);
+
if (!records.isEmpty()) {
try {
List keyList = new ArrayList();
diff --git a/data-resource-management-service/drms-rest-proxy/src/main/resources/drms.pb b/data-resource-management-service/drms-rest-proxy/src/main/resources/drms.pb
index fd2c03a..21a39a1 100644
Binary files a/data-resource-management-service/drms-rest-proxy/src/main/resources/drms.pb and b/data-resource-management-service/drms-rest-proxy/src/main/resources/drms.pb differ
diff --git a/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResource.proto b/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResource.proto
index db7741a..c696cb0 100644
--- a/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResource.proto
+++ b/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResource.proto
@@ -21,15 +21,15 @@ option java_multiple_files = true;
package org.apache.airavata.datalake.drms.resource;
import "google/api/annotations.proto";
-import "preference/S3Preference.proto";
-import "preference/SSHPreference.proto";
+import "storage/S3Storage.proto";
+import "storage/SSHStorage.proto";
message GenericResource {
string resource_id = 1;
string resource_path = 2;
- oneof storagePreference {
- org.apache.airavata.datalake.drms.storage.preference.s3.S3StoragePreference s3_preference = 3;
- org.apache.airavata.datalake.drms.storage.preference.ssh.SSHStoragePreference ssh_preference = 4;
+ oneof storage {
+ org.apache.airavata.datalake.drms.storage.ssh.SSHStorage ssh_storage = 3;
+ org.apache.airavata.datalake.drms.storage.s3.S3Storage s3_storage = 4;
}
map<string, string> properties = 5;
string type=6;