You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by is...@apache.org on 2021/07/27 03:54:51 UTC
[airavata-data-lake] branch master updated: drms service
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
The following commit(s) were added to refs/heads/master by this push:
new f0a9424 drms service
new 59fb3df Merge pull request #33 from isururanawaka/workflow_merge_23_jul_2021
f0a9424 is described below
commit f0a9424b486db4c24dda2b136d329fb84aa80619
Author: Isuru Ranawaka <ir...@gmail.com>
AuthorDate: Mon Jul 26 23:53:35 2021 -0400
drms service
---
.../drms/api/handlers/ResourceServiceHandler.java | 77 ++++++++++++++--------
.../deserializer/GenericResourceDeserializer.java | 60 ++++++++++++++++-
2 files changed, 107 insertions(+), 30 deletions(-)
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
index 2c652d2..625ba8b 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
@@ -390,7 +390,7 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
if (globalStorage.isPresent()) {
storageId = globalStorage.get();
}
-
+ List keyList = new ArrayList();
if (!resourceSearchQueries.isEmpty()) {
for (ResourceSearchQuery qry : resourceSearchQueries) {
if (qry.getField().equals("storageId")) {
@@ -400,13 +400,14 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
String val = qry.getValue();
String query = " Match (m:" + value + ")-[r:SHARED_WITH]->(l) " +
"where r.sharedBy=$sharedBy AND m.tenantId=$tenantId and l.tenantId=$tenantId " +
- "return (m) ";
+ "return m, r ";
Map<String, Object> objectMap = new HashMap<>();
objectMap.put("sharedBy", val);
objectMap.put("tenantId", callUser.getTenantId());
List<Record> records = this.neo4JConnector.searchNodes(objectMap, query);
-
- List<GenericResource> genericResourceList = GenericResourceDeserializer.deserializeList(records);
+ keyList = new ArrayList();
+ keyList.add("m:r");
+ List<GenericResource> genericResourceList = GenericResourceDeserializer.deserializeList(records, keyList);
ResourceSearchResponse.Builder builder = ResourceSearchResponse.newBuilder();
builder.addAllResources(genericResourceList);
responseObserver.onNext(builder.build());
@@ -418,19 +419,23 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
String val = qry.getValue();
String query = "MATCH (u:User) where u.username = $username AND u.tenantId = $tenantId " +
" OPTIONAL MATCH (g:Group)<-[:MEMBER_OF]-(u) " +
- " OPTIONAL MATCH (u)<-[:SHARED_WITH]-(m)<-[:CHILD_OF*]-(rm:" + value + ") , " +
- "(r:" + value + ")-[:SHARED_WITH]->(u) where NOT r.owner = '" + val + "'" +
+ " OPTIONAL MATCH (u)<-[relRM:SHARED_WITH]-(m)<-[:CHILD_OF*]-(rm:" + value + ") , " +
+ "(r:" + value + ")-[rel:SHARED_WITH]->(u) where NOT r.owner = '" + val + "'" +
" AND NOT rm.owner='" + val + "' " +
- " OPTIONAL MATCH (g)<-[:SHARED_WITH]-(mg)<-[:CHILD_OF*]-(rmg:" + value + ")," +
- " (rg:" + value + ")-[:SHARED_WITH]->(g) where NOT rg.owner = '" + val + "'" +
+ " OPTIONAL MATCH (g)<-[relRMG:SHARED_WITH]-(mg)<-[:CHILD_OF*]-(rmg:" + value + ")," +
+ " (rg:" + value + ")-[relRG:SHARED_WITH]->(g) where NOT rg.owner = '" + val + "'" +
" AND NOT rmg.owner='" + val + "' " +
- " return distinct r, rm, rmg, rg ";
+ " return distinct r,rel, rm,relRM, rmg,relRMG, rg,relRG ";
Map<String, Object> objectMap = new HashMap<>();
objectMap.put("username", val);
objectMap.put("tenantId", callUser.getTenantId());
List<Record> records = this.neo4JConnector.searchNodes(objectMap, query);
-
- List<GenericResource> genericResourceList = GenericResourceDeserializer.deserializeList(records);
+ keyList = new ArrayList();
+ keyList.add("r:rel");
+ keyList.add("rm:relRM");
+ keyList.add("rmg:relRMG");
+ keyList.add("rg:relRG");
+ List<GenericResource> genericResourceList = GenericResourceDeserializer.deserializeList(records,keyList);
ResourceSearchResponse.Builder builder = ResourceSearchResponse.newBuilder();
builder.addAllResources(genericResourceList);
responseObserver.onNext(builder.build());
@@ -498,39 +503,55 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
if ((value.equals("FILE") || value.equals("COLLECTION")) && !storageId.isEmpty()) {
query = " MATCH (u:User) where u.username = $username AND u.tenantId = $tenantId" +
" OPTIONAL MATCH (g:Group)<-[:MEMBER_OF]-(u) " +
- " OPTIONAL MATCH (u)<-[:SHARED_WITH]-(m)<-[:CHILD_OF*]-(rm:" + value + ")-[:CHILD_OF*]->(s:Storage{entityId:'" + storageId + "'})" +
- " , (s:Storage{entityId:'" + storageId + "'})<-[:CHILD_OF*]-(r:" + value + ")-[:SHARED_WITH]->(u)" +
- " OPTIONAL MATCH (g)<-[:SHARED_WITH]-(mg)<-[:CHILD_OF*]-(rmg:" + value + ")-[:CHILD_OF*]->(s:Storage{entityId:'" + storageId + "'})" +
- " , (s:Storage{entityId:'" + storageId + "'})<-[:CHILD_OF*]-(rg:" + value + ")-[:SHARED_WITH]->(g)" +
- " return distinct rm, r,rmg,rg ";
+ " OPTIONAL MATCH (u)<-[relRM:SHARED_WITH]-(m)<-[:CHILD_OF*]-(rm:" + value + ")-[:CHILD_OF*]->(s:Storage{entityId:'" + storageId + "'})" +
+ " , (s:Storage{entityId:'" + storageId + "'})<-[:CHILD_OF*]-(r:" + value + ")-[relR:SHARED_WITH]->(u)" +
+ " OPTIONAL MATCH (g)<-[relRMG:SHARED_WITH]-(mg)<-[:CHILD_OF*]-(rmg:" + value + ")-[:CHILD_OF*]->(s:Storage{entityId:'" + storageId + "'})" +
+ " , (s:Storage{entityId:'" + storageId + "'})<-[:CHILD_OF*]-(rg:" + value + ")-[relRG:SHARED_WITH]->(g)" +
+ " return distinct rm,relRM, r,relR, rmg,relRMG, rg,relRG ";
+ keyList = new ArrayList();
+ keyList.add("rm:relRM");
+ keyList.add("r:relR");
+ keyList.add("rmg:relRMG");
+ keyList.add("rg:relRG");
if (depth == 1) {
query = " MATCH (u:User) where u.username = $username AND u.tenantId = $tenantId" +
" OPTIONAL MATCH (g:Group)<-[:MEMBER_OF]-(u) " +
- " OPTIONAL MATCH (s:Storage{entityId:'" + storageId + "'})<-[:CHILD_OF*]-(r:" + value + ")-[:SHARED_WITH]->(u)" +
- " OPTIONAL MATCH (s:Storage{entityId:'" + storageId + "'})<-[:CHILD_OF*]-(rg:" + value + ")-[:SHARED_WITH]->(g)" +
- " return distinct r, rg ";
+ " OPTIONAL MATCH (s:Storage{entityId:'" + storageId + "'})<-[:CHILD_OF*]-(r:" + value + ")-[relR:SHARED_WITH]->(u)" +
+ " OPTIONAL MATCH (s:Storage{entityId:'" + storageId + "'})<-[:CHILD_OF*]-(rg:" + value + ")-[relRG:SHARED_WITH]->(g)" +
+ " return distinct r,relR, rg, relRG ";
+ keyList = new ArrayList();
+ keyList.add("r:relR");
+ keyList.add("rg:relRG");
}
} else {
query = " MATCH (u:User) where u.username = $username AND u.tenantId = $tenantId" +
" OPTIONAL MATCH (g:Group)<-[:MEMBER_OF]-(u) " +
- " OPTIONAL MATCH (u)<-[:SHARED_WITH]-(m)<-[:CHILD_OF*]-(rm:" + value + ")" +
- " , (r:" + value + ")-[:SHARED_WITH]->(u)" +
- " OPTIONAL MATCH (g)<-[:SHARED_WITH]-(mg)<-[:CHILD_OF*]-(rmg:" + value + ")" +
- " , (rg:" + value + ")-[:SHARED_WITH]->(g)" +
- " return distinct rm, r,rmg,rg ";
+ " OPTIONAL MATCH (u)<-[relRM:SHARED_WITH]-(m)<-[:CHILD_OF*]-(rm:" + value + ")" +
+ " , (r:" + value + ")-[relR:SHARED_WITH]->(u)" +
+ " OPTIONAL MATCH (g)<-[relRMG:SHARED_WITH]-(mg)<-[:CHILD_OF*]-(rmg:" + value + ")" +
+ " , (rg:" + value + ")-[relRG:SHARED_WITH]->(g)" +
+ " return distinct rm,relRM, r,relR, rmg,relRMG, rg, relRG ";
+ keyList = new ArrayList();
+ keyList.add("rm:relRM");
+ keyList.add("r:relR");
+ keyList.add("rmg:relRMG");
+ keyList.add("rg:relRG");
if (depth == 1) {
query = " MATCH (u:User) where u.username = $username AND u.tenantId = $tenantId" +
" OPTIONAL MATCH (g:Group)<-[:MEMBER_OF]-(u) " +
- " OPTIONAL MATCH (r:" + value + ")-[:SHARED_WITH]->(u)" +
- " OPTIONAL MATCH (rg:" + value + ")-[:SHARED_WITH]->(g)" +
- " return distinct r, rg ";
+ " OPTIONAL MATCH (r:" + value + ")-[relR:SHARED_WITH]->(u)" +
+ " OPTIONAL MATCH (rg:" + value + ")-[relRG:SHARED_WITH]->(g)" +
+ " return distinct r,relR, rg, relRG ";
+ keyList = new ArrayList();
+ keyList.add("r:relR");
+ keyList.add("rg:relRG");
}
}
List<Record> records = this.neo4JConnector.searchNodes(userProps, query);
- List<GenericResource> genericResourceList = GenericResourceDeserializer.deserializeList(records);
+ List<GenericResource> genericResourceList = GenericResourceDeserializer.deserializeList(records,keyList);
ResourceSearchResponse.Builder builder = ResourceSearchResponse.newBuilder();
builder.addAllResources(genericResourceList);
responseObserver.onNext(builder.build());
diff --git a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/GenericResourceDeserializer.java b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/GenericResourceDeserializer.java
index 265f0a4..28d6177 100644
--- a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/GenericResourceDeserializer.java
+++ b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/GenericResourceDeserializer.java
@@ -21,13 +21,17 @@ import org.apache.airavata.datalake.drms.resource.GenericResource;
import org.apache.commons.collections.map.HashedMap;
import org.neo4j.driver.Record;
import org.neo4j.driver.Value;
+import org.neo4j.driver.internal.InternalNode;
import org.neo4j.driver.internal.InternalRecord;
+import org.neo4j.driver.internal.InternalRelationship;
import org.neo4j.driver.types.Node;
+import org.neo4j.driver.types.Relationship;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanWrapper;
import org.springframework.beans.PropertyAccessorFactory;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -41,7 +45,6 @@ public class GenericResourceDeserializer {
for (Record record : neo4jRecords) {
InternalRecord internalRecord = (InternalRecord) record;
List<Value> values = internalRecord.values();
-
if (values.size() > 0) {
Map<Long, Node> longNodeMap = values.stream().filter(val ->
val.toString().equals("NULL") ? false : true
@@ -54,7 +57,34 @@ public class GenericResourceDeserializer {
return deriveGenericResourceFromMap(nodeMap);
}
- public static List<GenericResource> deriveGenericResourceFromMap(Map<Long, Node> nodeMap) throws Exception {
+
+ public static List<GenericResource> deserializeList(List<Record> neo4jRecords, List<String> keyValuePairs) throws Exception {
+ Map<Long, Node> nodeMap = new HashedMap();
+ Map<Long, Relationship> relationShip = new HashMap<>();
+ for (Record record : neo4jRecords) {
+ InternalRecord internalRecord = (InternalRecord) record;
+ Map<String, Object> values = internalRecord.asMap();
+ keyValuePairs.forEach(val -> {
+ String[] keys = val.split(":");
+ String resource = keys[0];
+ String relationship = keys[1];
+
+ InternalNode resNode = (InternalNode) values.get(resource);
+ InternalRelationship relNode = (InternalRelationship) values.get(relationship);
+
+ if (resNode != null) {
+ nodeMap.put(resNode.id(), resNode);
+ relationShip.put(resNode.id(), relNode);
+
+ }
+ });
+ }
+
+ return deriveGenericResourceFromMap(nodeMap,relationShip);
+ }
+
+
+ public static List<GenericResource> deriveGenericResourceFromMap(Map<Long, Node> nodeMap, Map<Long, Relationship> relationshipMap) throws Exception {
return nodeMap.values().stream().map(node -> {
GenericResource.Builder genericResourceBuilder = GenericResource.newBuilder();
Iterator<String> iterator = node.labels().iterator();
@@ -72,6 +102,8 @@ public class GenericResourceDeserializer {
if (field.equals("resourcePath")) {
genericResourceBuilder.setResourcePath(String.valueOf(node.asMap().get(field)));
}
+ genericResourceBuilder.putProperties("permission",
+ relationshipMap.get(node.id()).asMap().get("permission").toString());
}
return genericResourceBuilder.build();
}).collect(Collectors.toList());
@@ -79,6 +111,30 @@ public class GenericResourceDeserializer {
}
+ public static List<GenericResource> deriveGenericResourceFromMap(Map<Long, Node> nodeMap) throws Exception {
+ return nodeMap.values().stream().map(node -> {
+ GenericResource.Builder genericResourceBuilder = GenericResource.newBuilder();
+ Iterator<String> iterator = node.labels().iterator();
+ while (iterator.hasNext()) {
+ genericResourceBuilder.setType(iterator.next());
+ }
+ for (String field : node.asMap().keySet()) {
+ genericResourceBuilder.putProperties(field, String.valueOf(node.asMap().get(field)));
+ if (field.equals("entityId")) {
+ genericResourceBuilder.setResourceId(String.valueOf(node.asMap().get(field)));
+ }
+ if (field.equals("resourceName") || field.equals("name")) {
+ genericResourceBuilder.setResourceName(String.valueOf(node.asMap().get(field)));
+ }
+ if (field.equals("resourcePath")) {
+ genericResourceBuilder.setResourcePath(String.valueOf(node.asMap().get(field)));
+ }
+ }
+ return genericResourceBuilder.build();
+ }).collect(Collectors.toList());
+
+ }
+
private static void setObjectFieldsUsingMap(Object target, Map<String, Object> values) {
for (String field : values.keySet()) {
BeanWrapper beanWrapper = PropertyAccessorFactory.forBeanPropertyAccess(target);