You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by is...@apache.org on 2021/07/12 15:47:34 UTC
[airavata-data-lake] branch master updated: Bug fix in
transferMappings
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
The following commit(s) were added to refs/heads/master by this push:
new 5d6d131 Bug fix in transferMappings
new 26a3a80 Merge pull request #25 from isururanawaka/workflow_invocation
5d6d131 is described below
commit 5d6d1317a1c5f6d256c2e83e0cb254adcb7bc751
Author: Isuru Ranawaka <ir...@gmail.com>
AuthorDate: Mon Jul 12 08:58:56 2021 -0400
Bug fix in transferMappings
---
.../drms/api/handlers/ResourceServiceHandler.java | 9 +-
.../drms/api/handlers/StorageServiceHandler.java | 158 ++++++++++++++-------
.../org/apache/airavata/drms/api/utils/Utils.java | 24 ++++
.../deserializer/TransferMappingDeserializer.java | 29 ++++
4 files changed, 164 insertions(+), 56 deletions(-)
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
index 3be511a..2b5ad12 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
@@ -368,7 +368,8 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
for (ResourceSearchQuery qry : resourceSearchQueries) {
if (qry.getField().equals("sharedBy")) {
String val = qry.getValue();
- String query = " Match (m)-[r:SHARED_WITH]->(l) where r.sharedBy=$sharedBy AND m.tenantId=$tenantId and l.tenantId=$tenantId " +
+ String query = " Match (m:" + value + ")-[r:SHARED_WITH]->(l) " +
+ "where r.sharedBy=$sharedBy AND m.tenantId=$tenantId and l.tenantId=$tenantId " +
"return (m) ";
Map<String, Object> objectMap = new HashMap<>();
objectMap.put("sharedBy", val);
@@ -387,9 +388,11 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
String val = qry.getValue();
String query = "MATCH (u:User) where u.username = $username AND u.tenantId = $tenantId " +
" OPTIONAL MATCH (g:Group)<-[:MEMBER_OF]-(u) " +
- " OPTIONAL MATCH (u)<-[:SHARED_WITH]-(m)<-[:CHILD_OF*]-(rm) , (r)-[:SHARED_WITH]->(u) where NOT r.owner = '" + val + "'" +
+ " OPTIONAL MATCH (u)<-[:SHARED_WITH]-(m)<-[:CHILD_OF*]-(rm:" + value + ") , " +
+ "(r:" + value + ")-[:SHARED_WITH]->(u) where NOT r.owner = '" + val + "'" +
" AND NOT rm.owner='" + val + "' " +
- " OPTIONAL MATCH (g)<-[:SHARED_WITH]-(mg)<-[:CHILD_OF*]-(rmg), (rg)-[:SHARED_WITH]->(g) where NOT rg.owner = '" + val + "'" +
+ " OPTIONAL MATCH (g)<-[:SHARED_WITH]-(mg)<-[:CHILD_OF*]-(rmg:" + value + ")," +
+ " (rg:" + value + ")-[:SHARED_WITH]->(g) where NOT rg.owner = '" + val + "'" +
" AND NOT rmg.owner='" + val + "' " +
" return distinct r, rm, rmg, rg ";
Map<String, Object> objectMap = new HashMap<>();
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java
index a9dc2d0..bfe7c9b 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java
@@ -17,11 +17,11 @@
package org.apache.airavata.drms.api.handlers;
import com.google.protobuf.Empty;
-import io.grpc.Status;
import io.grpc.stub.StreamObserver;
import org.apache.airavata.datalake.drms.AuthenticatedUser;
import org.apache.airavata.datalake.drms.storage.*;
import org.apache.airavata.drms.api.utils.CustosUtils;
+import org.apache.airavata.drms.api.utils.Utils;
import org.apache.airavata.drms.core.Neo4JConnector;
import org.apache.airavata.drms.core.constants.StorageConstants;
import org.apache.airavata.drms.core.deserializer.AnyStorageDeserializer;
@@ -196,81 +196,133 @@ public class StorageServiceHandler extends StorageServiceGrpc.StorageServiceImpl
public void createTransferMapping(CreateTransferMappingRequest request, StreamObserver<CreateTransferMappingResponse> responseObserver) {
try {
AuthenticatedUser authenticatedUser = request.getAuthToken().getAuthenticatedUser();
- AnyStorage sourceStoragePreference = request.getTransferMapping().getSourceStorage();
- AnyStorage destinationStoragePreference = request.getTransferMapping().getDestinationStorage();
- String sourceId = getStorageId(sourceStoragePreference);
- String destinationId = getStorageId(destinationStoragePreference);
-
+ AnyStorage sourceStorage = request.getTransferMapping().getSourceStorage();
+ AnyStorage destinationStorage = request.getTransferMapping().getDestinationStorage();
TransferScope scope = request.getTransferMapping().getTransferScope();
Map<String, Object> properties = new HashMap<>();
Map<String, Object> props = new HashMap<>();
props.put("tenantId", authenticatedUser.getTenantId());
props.put("owner", authenticatedUser.getUsername());
- props.put("srcStorageId", sourceId);
- props.put("dstStorageId", destinationId);
- String entityId = sourceId + "_" + destinationId;
+ properties.put("tenantId", authenticatedUser.getTenantId());
+ properties.put("username", authenticatedUser.getUsername());
+ properties.put("owner", authenticatedUser.getUsername());
+ List<TransferMapping> transferMappings = new ArrayList<>();
+
if (scope.equals(TransferScope.GLOBAL)) {
props.put("scope", TransferScope.GLOBAL.name());
} else {
props.put("scope", TransferScope.USER.name());
}
- properties.put("props", props);
- properties.put("tenantId", authenticatedUser.getTenantId());
- properties.put("entityId", entityId);
- properties.put("username", authenticatedUser.getUsername());
- properties.put("srcStorageId", sourceId);
- properties.put("dstStorageId", destinationId);
- properties.put("owner", authenticatedUser.getUsername());
+ if (sourceStorage.getSshStorage() != null && !sourceStorage.getSshStorage().getStorageId().isEmpty() &&
+ destinationStorage.getSshStorage() == null || destinationStorage.getSshStorage().getStorageId().isEmpty()) {
+ String sourceId = sourceStorage.getSshStorage().getStorageId();
+ properties.put("srcStorageId", sourceId);
+ props.put("srcStorageId", sourceId);
+ String messageId = authenticatedUser.getUsername() + "_" + sourceId;
+ String entityId = Utils.getId(messageId);
+ properties.put("props", props);
+ properties.put("entityId", entityId);
+ props.put("entityId", entityId);
+ if (hasAccess(authenticatedUser.getUsername(), authenticatedUser.getTenantId(), sourceId)) {
+ String query = " Match (u:User), (srcSp:Storage) where " +
+ " u.username=$username AND u.tenantId=$tenantId AND " +
+ "srcSp.storageId=$srcStorageId AND " +
+ "srcSp.tenantId = $tenantId " +
+ " Merge (u)-[:HAS_TRANSFER_MAPPING]->(tm:TransferMapping{entityId:$entityId, tenantId:$tenantId, " +
+ "srcStorageId:$srcStorageId," +
+ "owner:$owner}) set tm += $props" +
+ " Merge (tm)<-[:TRANSFER_OUT]-(srcSp)" +
+ " return (tm)";
+ this.neo4JConnector.runTransactionalQuery(properties, query);
+
+ String searchQuery = " Match (srcStr:Storage)-[:TRANSFER_OUT]->(tm:TransferMapping)" +
+ " where " +
+ " tm.entityId=$entityId AND tm.tenantId=$tenantId return srcStr, tm";
+ List<Record> records = this.neo4JConnector.searchNodes(properties, searchQuery);
+ if (!records.isEmpty()) {
+ transferMappings = TransferMappingDeserializer.deserializeListExceptDestinationStorage(records);
+ }
+ }
- if (hasAccess(authenticatedUser.getUsername(), authenticatedUser.getTenantId(), sourceId) &&
- hasAccess(authenticatedUser.getUsername(), authenticatedUser.getTenantId(), destinationId)) {
- String query = " Match (u:User), (srcSp:Storage), (dstSp:Storage) where " +
- " u.username=$username AND u.tenantId=$tenantId AND " +
- "srcSp.storageId=$srcStorageId AND " +
- "srcSp.tenantId = $tenantId AND dstSp.storageId=$dstStorageId " +
- "AND dstSp.tenantId =$tenantId " +
- " Merge (u)-[:HAS_TRANSFER_MAPPING]->(tm:TransferMapping{entityId:$entityId, tenantId:$tenantId, " +
- "srcStorageId:$srcStorageId," +
- "dstStorageId:$dstStorageId,owner:$owner}) set tm += $props" +
- " Merge (tm)<-[:TRANSFER_OUT]-(srcSp)" +
- " Merge (tm)-[:TRANSFER_IN]->(dstSp) return (tm)";
- this.neo4JConnector.runTransactionalQuery(properties, query);
-
- String searchQuery = " Match (srcStr:Storage)-[:TRANSFER_OUT]->(tm:TransferMapping)" +
- "-[:TRANSFER_IN]->(dstStr:Storage) where " +
- " tm.entityId=$entityId AND tm.tenantId=$tenantId return srcStr, dstStr, tm";
- List<Record> records = this.neo4JConnector.searchNodes(properties, searchQuery);
- if (!records.isEmpty()) {
- List<TransferMapping> transferMappings = TransferMappingDeserializer.deserializeList(records);
- if (!transferMappings.isEmpty()) {
- CreateTransferMappingResponse response = CreateTransferMappingResponse
- .newBuilder()
- .setTransferMapping(transferMappings.get(0))
- .build();
- responseObserver.onNext(response);
- responseObserver.onCompleted();
- } else {
- String msg = "Errored while creating transfer mapping; Message:";
- logger.error("Errored while creating transfer mapping; Message:");
+ } else {
+ String sourceId = null;
+
+ if (sourceStorage.getSshStorage() == null || sourceStorage.getSshStorage().getStorageId().isEmpty()) {
+ String query = " Match (srcStr:Storage)-[:TRANSFER_OUT]->(tm:TransferMapping) where tm.tenantId=$tenantId AND tm.scope='GLOBAL' " +
+ " AND srcStr.tenantId=$tenantId return srcStr, tm";
+ List<Record> sourceRecords = this.neo4JConnector.searchNodes(properties, query);
+ List<TransferMapping> sourceTransfers = TransferMappingDeserializer.deserializeListExceptDestinationStorage(sourceRecords);
+ if (sourceTransfers.isEmpty()) {
+ String msg = "Errored while creating transfer mapping; Message: Cannot find global source storage ";
+ logger.error("Errored while creating transfer mapping; Message: Cannot find global source storage ");
responseObserver.onError(io.grpc.Status.INTERNAL.withDescription(msg).asRuntimeException());
+ return;
}
+ TransferMapping transferMapping = sourceTransfers.get(0);
+ sourceId = getStorageId(transferMapping.getSourceStorage());
+
} else {
- String msg = "Errored while creating transfer mapping; Message:";
- logger.error("Errored while creating transfer mapping; Message:");
- responseObserver.onError(io.grpc.Status.INTERNAL.withDescription(msg).asRuntimeException());
+ sourceId = getStorageId(sourceStorage);
}
- } else {
- String msg = "User does not have permission to create mapping ";
- logger.error("User does not have permission to create mapping ");
- responseObserver.onError(Status.PERMISSION_DENIED.withDescription(msg).asRuntimeException());
+ String destinationId = getStorageId(destinationStorage);
+ props.put("srcStorageId", sourceId);
+ props.put("dstStorageId", destinationId);
+ String messageId = authenticatedUser.getUsername() + "_" + sourceId + "_" + destinationId;
+ String entityId = Utils.getId(messageId);
+ properties.put("props", props);
+ properties.put("tenantId", authenticatedUser.getTenantId());
+ properties.put("entityId", entityId);
+ properties.put("username", authenticatedUser.getUsername());
+ properties.put("srcStorageId", sourceId);
+ properties.put("dstStorageId", destinationId);
+ properties.put("owner", authenticatedUser.getUsername());
+ properties.put("entityId", entityId);
+ props.put("entityId", entityId);
+
+ if (hasAccess(authenticatedUser.getUsername(), authenticatedUser.getTenantId(), sourceId) &&
+ hasAccess(authenticatedUser.getUsername(), authenticatedUser.getTenantId(), destinationId)) {
+ String query = " Match (u:User), (srcSp:Storage), (dstSp:Storage) where " +
+ " u.username=$username AND u.tenantId=$tenantId AND " +
+ "srcSp.storageId=$srcStorageId AND " +
+ "srcSp.tenantId = $tenantId AND dstSp.storageId=$dstStorageId " +
+ "AND dstSp.tenantId =$tenantId " +
+ " Merge (u)-[:HAS_TRANSFER_MAPPING]->(tm:TransferMapping{entityId:$entityId, tenantId:$tenantId, " +
+ "srcStorageId:$srcStorageId," +
+ "dstStorageId:$dstStorageId,owner:$owner}) set tm += $props" +
+ " Merge (tm)<-[:TRANSFER_OUT]-(srcSp)" +
+ " Merge (tm)-[:TRANSFER_IN]->(dstSp) return (tm)";
+ this.neo4JConnector.runTransactionalQuery(properties, query);
+
+ String searchQuery = " Match (srcStr:Storage)-[:TRANSFER_OUT]->(tm:TransferMapping)" +
+ "-[:TRANSFER_IN]->(dstStr:Storage) where " +
+ " tm.entityId=$entityId AND tm.tenantId=$tenantId return srcStr, dstStr, tm";
+ List<Record> records = this.neo4JConnector.searchNodes(properties, searchQuery);
+ if (!records.isEmpty()) {
+ transferMappings = TransferMappingDeserializer.deserializeList(records);
+ }
+ }
+ }
+
+ if (!transferMappings.isEmpty()) {
+ CreateTransferMappingResponse response = CreateTransferMappingResponse
+ .newBuilder()
+ .setTransferMapping(transferMappings.get(0))
+ .build();
+ responseObserver.onNext(response);
+ responseObserver.onCompleted();
+ } else {
+ String msg = "Errored while creating transfer mapping; Message:";
+ logger.error("Errored while creating transfer mapping; Message:");
+ responseObserver.onError(io.grpc.Status.INTERNAL.withDescription(msg).asRuntimeException());
}
} catch (Exception e) {
String msg = "Errored while creating transfer mapping; Message:" + e.getMessage();
logger.error("Errored while creating transfer mapping; Message: {}", e.getMessage(), e);
responseObserver.onError(io.grpc.Status.INTERNAL.withDescription(msg).asRuntimeException());
}
+
}
@Override
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/utils/Utils.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/utils/Utils.java
index d651d75..f6675b9 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/utils/Utils.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/utils/Utils.java
@@ -3,6 +3,10 @@ package org.apache.airavata.drms.api.utils;
import io.grpc.Context;
import org.apache.airavata.datalake.drms.storage.ResourceSearchQuery;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -88,5 +92,25 @@ public class Utils {
return Optional.empty();
}
+ public static String getId(String message) throws NoSuchAlgorithmException {
+ MessageDigest md = MessageDigest.getInstance("SHA-256");
+ // digest() method called
+ // to calculate message digest of an input
+ // and return array of byte
+ byte[] array = md.digest(message.getBytes(StandardCharsets.UTF_8));
+ // Convert byte array into signum representation
+ BigInteger number = new BigInteger(1, array);
+
+ // Convert message digest into hex value
+ StringBuilder hexString = new StringBuilder(number.toString(16));
+
+ // Pad with leading zeros
+ while (hexString.length() < 32) {
+ hexString.insert(0, '0');
+ }
+
+ return hexString.toString();
+ }
+
}
diff --git a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/TransferMappingDeserializer.java b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/TransferMappingDeserializer.java
index 51e9a57..6414730 100644
--- a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/TransferMappingDeserializer.java
+++ b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/TransferMappingDeserializer.java
@@ -46,4 +46,33 @@ public class TransferMappingDeserializer {
return transferMappings;
}
+ public static List<TransferMapping> deserializeListExceptDestinationStorage(List<Record> neo4jRecords) throws Exception {
+ List<TransferMapping> transferMappings = new ArrayList<>();
+ for (Record record : neo4jRecords) {
+ InternalRecord internalRecord = (InternalRecord) record;
+ List<Value> values = internalRecord.values();
+
+ if (values.size() == 2) {
+ Value srcStr = values.get(0);
+ Value tm = values.get(1);
+
+ if (!srcStr.isNull() && !tm.isNull()) {
+ AnyStorage storage = AnyStorageDeserializer.deriveStorageFromMap(srcStr.asMap());
+
+ Map<String, Object> map = tm.asMap();
+ TransferMapping transferMapping = TransferMapping.newBuilder()
+ .setTransferScope(TransferScope.valueOf(map.get("scope").toString()))
+ .setId(map.get("entityId").toString())
+ .setSourceStorage(storage)
+ .setUserId(map.get("owner").toString())
+ .build();
+
+ transferMappings.add(transferMapping);
+
+ }
+ }
+ }
+ return transferMappings;
+ }
+
}