You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by is...@apache.org on 2021/07/26 21:15:42 UTC
[airavata-data-lake] branch master updated: storage based resource
search
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
The following commit(s) were added to refs/heads/master by this push:
new 40b5cec storage based resource search
new e3ef5d8 Merge pull request #32 from isururanawaka/workflow_merge_23_jul_2021
40b5cec is described below
commit 40b5cece5fa52f949c232197d02b2cae2b4294b4
Author: Isuru Ranawaka <ir...@gmail.com>
AuthorDate: Mon Jul 26 17:15:06 2021 -0400
storage based resource search
---
.../drms/api/handlers/ResourceServiceHandler.java | 104 ++++++++++++++++-----
.../drms/api/handlers/StorageServiceHandler.java | 2 +-
.../org/apache/airavata/drms/api/utils/Utils.java | 21 ++++-
3 files changed, 96 insertions(+), 31 deletions(-)
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
index ed6328f..2c652d2 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
@@ -32,6 +32,7 @@ import org.apache.airavata.drms.core.Neo4JConnector;
import org.apache.airavata.drms.core.constants.StoragePreferenceConstants;
import org.apache.airavata.drms.core.deserializer.AnyStorageDeserializer;
import org.apache.airavata.drms.core.deserializer.GenericResourceDeserializer;
+import org.apache.airavata.drms.core.deserializer.TransferMappingDeserializer;
import org.apache.airavata.drms.core.serializer.GenericResourceSerializer;
import org.apache.custos.clients.CustosClientProvider;
import org.apache.custos.sharing.service.Entity;
@@ -152,11 +153,11 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
serializedMap.put("owner", exEntity.get().getOwnerId());
if (serializedMap.containsKey("properties") && serializedMap.get("properties") instanceof List) {
- List propertiesList = (List) serializedMap.get("properties");
- propertiesList.forEach(property-> {
- MapEntry entry = (MapEntry) property;
- serializedMap.put(entry.getKey().toString(),entry.getValue());
- });
+ List propertiesList = (List) serializedMap.get("properties");
+ propertiesList.forEach(property -> {
+ MapEntry entry = (MapEntry) property;
+ serializedMap.put(entry.getKey().toString(), entry.getValue());
+ });
}
serializedMap.remove("properties");
if (!parentId.isEmpty()) {
@@ -375,8 +376,26 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
int depth = request.getDepth();
String value = request.getType();
+ if (value == null || value.isEmpty()) {
+ logger.error("Resource type required to continue search ");
+ responseObserver
+ .onError(Status.FAILED_PRECONDITION.withDescription("Resource type required to continue search")
+ .asRuntimeException());
+ return;
+ }
+
+
+ String storageId = "";
+ Optional<String> globalStorage = getGlobalSourceStorage(callUser.getTenantId());
+ if (globalStorage.isPresent()) {
+ storageId = globalStorage.get();
+ }
+
if (!resourceSearchQueries.isEmpty()) {
for (ResourceSearchQuery qry : resourceSearchQueries) {
+ if (qry.getField().equals("storageId")) {
+ storageId = qry.getValue();
+ }
if (qry.getField().equals("sharedBy")) {
String val = qry.getValue();
String query = " Match (m:" + value + ")-[r:SHARED_WITH]->(l) " +
@@ -421,16 +440,9 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
}
}
- if (value == null || value.isEmpty()) {
- logger.error("Errored while searching generic resources");
- responseObserver
- .onError(Status.INTERNAL.withDescription("Errored while searching generic resources ")
- .asRuntimeException());
- return;
- }
- Optional<String> metadataSearchQueryOP = Utils.getMetadataSearchQuery(resourceSearchQueries, value);
- Optional<String> ownPropertySearchQuery = Utils.getPropertySearchQuery(resourceSearchQueries, value);
+ Optional<String> metadataSearchQueryOP = Utils.getMetadataSearchQuery(resourceSearchQueries, value, storageId);
+ Optional<String> ownPropertySearchQuery = Utils.getPropertySearchQuery(resourceSearchQueries, value, storageId);
if (metadataSearchQueryOP.isPresent()) {
String query = metadataSearchQueryOP.get();
@@ -482,20 +494,38 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
userProps.put("username", callUser.getUsername());
userProps.put("tenantId", callUser.getTenantId());
- String query = " MATCH (u:User) where u.username = $username AND u.tenantId = $tenantId" +
- " OPTIONAL MATCH (g:Group)<-[:MEMBER_OF]-(u) " +
- " OPTIONAL MATCH (u)<-[:SHARED_WITH]-(m)<-[:CHILD_OF*]-(rm:" + value + ")" +
- " , (r:" + value + ")-[:SHARED_WITH]->(u)" +
- " OPTIONAL MATCH (g)<-[:SHARED_WITH]-(mg)<-[:CHILD_OF*]-(rmg:" + value + ")" +
- " , (rg:" + value + ")-[:SHARED_WITH]->(g)" +
- " return distinct rm, r,rmg,rg ";
+ String query = "";
+ if ((value.equals("FILE") || value.equals("COLLECTION")) && !storageId.isEmpty()) {
+ query = " MATCH (u:User) where u.username = $username AND u.tenantId = $tenantId" +
+ " OPTIONAL MATCH (g:Group)<-[:MEMBER_OF]-(u) " +
+ " OPTIONAL MATCH (u)<-[:SHARED_WITH]-(m)<-[:CHILD_OF*]-(rm:" + value + ")-[:CHILD_OF*]->(s:Storage{entityId:'" + storageId + "'})" +
+ " , (s:Storage{entityId:'" + storageId + "'})<-[:CHILD_OF*]-(r:" + value + ")-[:SHARED_WITH]->(u)" +
+ " OPTIONAL MATCH (g)<-[:SHARED_WITH]-(mg)<-[:CHILD_OF*]-(rmg:" + value + ")-[:CHILD_OF*]->(s:Storage{entityId:'" + storageId + "'})" +
+ " , (s:Storage{entityId:'" + storageId + "'})<-[:CHILD_OF*]-(rg:" + value + ")-[:SHARED_WITH]->(g)" +
+ " return distinct rm, r,rmg,rg ";
+ if (depth == 1) {
+ query = " MATCH (u:User) where u.username = $username AND u.tenantId = $tenantId" +
+ " OPTIONAL MATCH (g:Group)<-[:MEMBER_OF]-(u) " +
+ " OPTIONAL MATCH (s:Storage{entityId:'" + storageId + "'})<-[:CHILD_OF*]-(r:" + value + ")-[:SHARED_WITH]->(u)" +
+ " OPTIONAL MATCH (s:Storage{entityId:'" + storageId + "'})<-[:CHILD_OF*]-(rg:" + value + ")-[:SHARED_WITH]->(g)" +
+ " return distinct r, rg ";
+ }
- if (depth == 1) {
+ } else {
query = " MATCH (u:User) where u.username = $username AND u.tenantId = $tenantId" +
" OPTIONAL MATCH (g:Group)<-[:MEMBER_OF]-(u) " +
- " OPTIONAL MATCH (r:" + value + ")-[:SHARED_WITH]->(u)" +
- " OPTIONAL MATCH (rg:" + value + ")-[:SHARED_WITH]->(g)" +
- " return distinct r, rg ";
+ " OPTIONAL MATCH (u)<-[:SHARED_WITH]-(m)<-[:CHILD_OF*]-(rm:" + value + ")" +
+ " , (r:" + value + ")-[:SHARED_WITH]->(u)" +
+ " OPTIONAL MATCH (g)<-[:SHARED_WITH]-(mg)<-[:CHILD_OF*]-(rmg:" + value + ")" +
+ " , (rg:" + value + ")-[:SHARED_WITH]->(g)" +
+ " return distinct rm, r,rmg,rg ";
+ if (depth == 1) {
+ query = " MATCH (u:User) where u.username = $username AND u.tenantId = $tenantId" +
+ " OPTIONAL MATCH (g:Group)<-[:MEMBER_OF]-(u) " +
+ " OPTIONAL MATCH (r:" + value + ")-[:SHARED_WITH]->(u)" +
+ " OPTIONAL MATCH (rg:" + value + ")-[:SHARED_WITH]->(g)" +
+ " return distinct r, rg ";
+ }
}
List<Record> records = this.neo4JConnector.searchNodes(userProps, query);
@@ -905,4 +935,28 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
return Optional.empty();
}
+ private Optional<String> getGlobalSourceStorage(String tenantId) throws Exception {
+ Map<String, Object> properties = new HashMap<>();
+ properties.put("tenantId", tenantId);
+ properties.put("scope", TransferScope.GLOBAL.name());
+ String query = " Match (srcStr:Storage)-[:TRANSFER_OUT]->(tm:TransferMapping) where tm.tenantId=$tenantId AND tm.scope='GLOBAL' " +
+ " AND srcStr.tenantId=$tenantId return srcStr, tm";
+ List<Record> records = this.neo4JConnector.searchNodes(properties, query);
+ List<TransferMapping> sourceTransfers = TransferMappingDeserializer.deserializeListExceptDestinationStorage(records);
+ if (sourceTransfers.isEmpty()) {
+ return Optional.empty();
+ }
+ TransferMapping transferMapping = sourceTransfers.get(0);
+ return Optional.ofNullable(getStorageId(transferMapping.getSourceStorage()));
+ }
+
+ private String getStorageId(AnyStorage storage) {
+ if (storage.getStorageCase()
+ .equals(AnyStorage.StorageCase.S3_STORAGE)) {
+ return storage.getS3Storage().getStorageId();
+ } else {
+ return storage.getSshStorage().getStorageId();
+ }
+ }
+
}
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java
index bfe7c9b..dea8e26 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java
@@ -339,7 +339,7 @@ public class StorageServiceHandler extends StorageServiceGrpc.StorageServiceImpl
" return srcStr, dstStr, t";
List<Record> records = this.neo4JConnector.searchNodes(properties, query);
properties.put("scope", TransferScope.GLOBAL.name());
- String queryFetchGlobal = "Match (srcStr:Storage)<-[:TRANSFER_OUT]->(t:TransferMapping{scope:$scope, tenantId:$tenantId})-[:TRANSFER_IN]->(dstStr:Storage)" +
+ String queryFetchGlobal = "Match (srcStr:Storage)-[:TRANSFER_OUT]->(t:TransferMapping{scope:$scope, tenantId:$tenantId})-[:TRANSFER_IN]->(dstStr:Storage)" +
" return srcStr, dstStr, t";
List<Record> globalRecords = this.neo4JConnector.searchNodes(properties, queryFetchGlobal);
if (!records.isEmpty()) {
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/utils/Utils.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/utils/Utils.java
index f6675b9..d765ebf 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/utils/Utils.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/utils/Utils.java
@@ -28,11 +28,16 @@ public class Utils {
}
- public static Optional<String> getMetadataSearchQuery(List<ResourceSearchQuery> resourceSearchQueries, String type) {
+ public static Optional<String> getMetadataSearchQuery(List<ResourceSearchQuery> resourceSearchQueries, String type, String storageId) {
if (!resourceSearchQueries.isEmpty()) {
String preRegex = "'(?i).*";
String postRegex = ".*'";
- String query = " MATCH (r:" + type + ")-[:HAS_METADATA*]->(m) WHERE ";
+ String query = "";
+ if ((type.equals("FILE") || type.equals("COLLECTION")) && !storageId.isEmpty()) {
+ query = " MATCH (s:Storage{entityId:'" + storageId + "'})<-[:CHILD_OF*]-(r:" + type + ")-[:HAS_METADATA*]->(m) WHERE ";
+ } else {
+ query = " MATCH (r:" + type + ")-[:HAS_METADATA*]->(m) WHERE ";
+ }
//TODO: works only for one property
for (ResourceSearchQuery qry : resourceSearchQueries) {
if (qry.getField().contains(" ")) {
@@ -52,11 +57,17 @@ public class Utils {
return Optional.empty();
}
- public static Optional<String> getPropertySearchQuery(List<ResourceSearchQuery> resourceSearchQueries, String type) {
+ public static Optional<String> getPropertySearchQuery(List<ResourceSearchQuery> resourceSearchQueries, String type, String storageId) {
if (!resourceSearchQueries.isEmpty()) {
for (ResourceSearchQuery qry : resourceSearchQueries) {
- String query = " MATCH (r:" + type +
- ") where r." + qry.getField() + " contains '" + qry.getValue() + "' Return r ";
+ String query = "";
+ if ((type.equals("FILE") || type.equals("COLLECTION")) && !storageId.isEmpty()) {
+ query = " MATCH (s:Storage{entityId:'" + storageId + "'})<-[:CHILD_OF*]-(r:" + type +
+ ") where r." + qry.getField() + " contains '" + qry.getValue() + "' Return r ";
+ } else {
+ query = " MATCH (r:" + type +
+ ") where r." + qry.getField() + " contains '" + qry.getValue() + "' Return r ";
+ }
return Optional.ofNullable(query);
}
}