You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by is...@apache.org on 2021/06/01 17:58:55 UTC
[airavata-data-lake] 12/46: Add parser interface,
remove duplicate object creation at Neo4J
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
commit 4e9880d1d259859b05e881df39d3f73b594ca7cd
Author: Isuru Ranawaka <ir...@gmail.com>
AuthorDate: Mon Mar 22 16:03:15 2021 -0400
Add parser interface, remove duplicate object creation at Neo4J
---
.../airavata/datalake/metadata/clients/Test.java | 39 +++++-
.../airavata/datalake/metadata/AppConfig.java | 52 ++++---
.../metadata/backend/neo4j/model/nodes/Entity.java | 7 +
.../metadata/backend/neo4j/model/nodes/Group.java | 5 +
.../backend/neo4j/model/nodes/Resource.java | 5 +
.../metadata/backend/neo4j/model/nodes/Tenant.java | 5 +
.../metadata/backend/neo4j/model/nodes/User.java | 5 +
.../metadata/handlers/TenantServiceHandler.java | 3 +-
.../metadata/parsers/ExecutionContext.java | 17 +++
.../datalake/metadata/parsers/GroupParser.java | 106 ++++++++------
.../airavata/datalake/metadata/parsers/Parser.java | 13 ++
.../datalake/metadata/parsers/ResourceParser.java | 152 ++++++++++++++-------
.../datalake/metadata/parsers/TenantParser.java | 114 +++++++++-------
.../datalake/metadata/parsers/UserParser.java | 33 ++++-
.../stub/src/main/proto/resource/Resource.proto | 13 +-
15 files changed, 389 insertions(+), 180 deletions(-)
diff --git a/metadata-service/db-service/client/src/main/java/org/apache/airavata/datalake/metadata/clients/Test.java b/metadata-service/db-service/client/src/main/java/org/apache/airavata/datalake/metadata/clients/Test.java
index dd8b392..a506d75 100644
--- a/metadata-service/db-service/client/src/main/java/org/apache/airavata/datalake/metadata/clients/Test.java
+++ b/metadata-service/db-service/client/src/main/java/org/apache/airavata/datalake/metadata/clients/Test.java
@@ -9,25 +9,27 @@ public class Test {
TenantMetadataServiceGrpc.TenantMetadataServiceBlockingStub stub = serviceClient.tenant();
+
Tenant tenant = Tenant.newBuilder()
- .setTenantId("asdcfvf")
+ .setTenantId("100010402")
.setName("TenantA")
.build();
+
Group group = Group.newBuilder()
.setName("g1")
+ .setTenantId("100010402")
.build();
+
Group group2 = Group.newBuilder()
.setName("g3")
- .build();
- Group group1 = Group.newBuilder()
- .setName("g2")
- .addChildGroups(group2)
+ .setTenantId("100010402")
.build();
User user = User.newBuilder()
.setUsername("TestingUserA")
.setFirstName("Isuru")
.setLastName("Ranawaka")
+ .setTenantId("100010402")
.build();
GroupMembership groupMemberships = GroupMembership
@@ -36,26 +38,52 @@ public class Test {
.setMembershipType("ADMIN")
.build();
+ Group group1 = Group.newBuilder()
+ .setName("g2")
+ .setTenantId("100010402")
+ .addChildGroups(group2)
+ .build();
+
group1 = group1.toBuilder()
.addGroupMembership(groupMemberships)
+ .setTenantId("100010402")
.build();
+
group = group.toBuilder()
.addChildGroups(group1)
+ .setTenantId("100010402")
.build();
Resource resource = Resource.newBuilder()
.setName("R1")
+ .setTenantId("100010402")
.build();
+
Resource resource1 = Resource.newBuilder()
.setName("R2")
+ .setTenantId("100010402")
.build();
+
Resource resource2 = Resource.newBuilder()
.setName("R3")
+ .setTenantId("100010402")
.build();
resource1 = resource1.toBuilder()
.addChildResources(resource2)
+ .setTenantId("100010402")
+ .build();
+
+ ResourceSharings resourceSharings = ResourceSharings
+ .newBuilder()
+ .setPermissionType("READ")
+ .addGroups(group2)
.build();
+
+ resource1 = resource1.toBuilder()
+ .addSharings(resourceSharings)
+ .build();
+
resource = resource.toBuilder()
.addChildResources(resource1)
.build();
@@ -63,6 +91,7 @@ public class Test {
tenant = tenant.toBuilder()
.addGroups(group)
.build();
+
tenant = tenant.toBuilder()
.addResources(resource)
.build();
diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/AppConfig.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/AppConfig.java
index 2c0f692..0e03c32 100644
--- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/AppConfig.java
+++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/AppConfig.java
@@ -1,18 +1,26 @@
package org.apache.airavata.datalake.metadata;
import org.apache.airavata.datalake.metadata.backend.Connector;
+import org.apache.airavata.datalake.metadata.backend.neo4j.curd.operators.ResourceServiceImpl;
+import org.apache.airavata.datalake.metadata.backend.neo4j.curd.operators.SearchOperator;
+import org.apache.airavata.datalake.metadata.backend.neo4j.curd.operators.TenantServiceImpl;
import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group;
import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Resource;
import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Tenant;
import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User;
import org.dozer.DozerBeanMapper;
import org.dozer.loader.api.BeanMappingBuilder;
+import org.neo4j.ogm.cypher.ComparisonOperator;
+import org.neo4j.ogm.cypher.Filter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
+import java.util.ArrayList;
+import java.util.List;
+
@Configuration
public class AppConfig {
@@ -25,7 +33,9 @@ public class AppConfig {
@Bean
public DozerBeanMapper dozerBeanMapper() {
+
DozerBeanMapper mapper = new DozerBeanMapper();
+
BeanMappingBuilder tenantMapping = new BeanMappingBuilder() {
@Override
protected void configure() {
@@ -39,6 +49,7 @@ public class AppConfig {
mapping(org.apache.airavata.datalake.metadata.service.Group.class, Group.class);
}
};
+
BeanMappingBuilder resourceMapping = new BeanMappingBuilder() {
@Override
protected void configure() {
@@ -52,10 +63,12 @@ public class AppConfig {
mapping(org.apache.airavata.datalake.metadata.service.User.class, User.class);
}
};
+
mapper.addMapping(tenantMapping);
mapper.addMapping(groupMapping);
mapper.addMapping(resourceMapping);
mapper.addMapping(userMapping);
+
return mapper;
}
@@ -63,6 +76,7 @@ public class AppConfig {
@Bean
Tenant getTenant() {
LOGGER.info("Calling get tenant############");
+
Tenant tenant = new Tenant();
tenant.setTenantId("123456789");
tenant.setName("Tenant");
@@ -91,7 +105,6 @@ public class AppConfig {
group.addChildGroup(group2, 0, 0, null);
group2.addChildGroup(group3, 0, 0, null);
-
Resource resource = new Resource();
resource.setName("R1");
@@ -104,12 +117,10 @@ public class AppConfig {
Resource resource3 = new Resource();
resource3.setName("R4");
-
group.addChildUser(user, "ADMIN", 0, 0, null);
resource.addChildResource(resource1, 0, 0, null);
resource.shareWithAUser(user, "READ", 0, 0, null);
-
group2.addChildUser(user1, "ADMIN", 0, 0, null);
group3.addChildUser(user2, "ADMIN", 0, 0, null);
@@ -120,29 +131,26 @@ public class AppConfig {
tenant.add(group, 0, 0, null);
tenant.add(resource, 0, 0, null);
+ TenantServiceImpl tenantService = new TenantServiceImpl(connector);
+ tenantService.createOrUpdate(tenant);
-// TenantServiceImpl tenantService = new TenantServiceImpl(connector);
-// tenantService.createOrUpdate(tenant);
-//
-// Filter filter = new Filter("name", ComparisonOperator.EQUALS, "R3");
-//
-//
-// ResourceServiceImpl resourceService = new ResourceServiceImpl(connector);
-// SearchOperator searchOperator = new SearchOperator();
-// searchOperator.setKey("name");
-// searchOperator.setValue("R2");
-// searchOperator.setComparisonOperator(ComparisonOperator.EQUALS);
-// List searchList = new ArrayList<>();
-// searchList.add(searchOperator);
-// List<Resource> collections = (List<Resource>) resourceService.search(searchList);
-// LOGGER.info("Size", collections.size());
-// for (Resource collection : collections) {
-// LOGGER.info("#############" + collection.getName() + "Created At" + collection.getCreatedAt());
-// }
+ Filter filter = new Filter("name", ComparisonOperator.EQUALS, "R3");
+ ResourceServiceImpl resourceService = new ResourceServiceImpl(connector);
+ SearchOperator searchOperator = new SearchOperator();
+ searchOperator.setKey("name");
+ searchOperator.setValue("R2");
+ searchOperator.setComparisonOperator(ComparisonOperator.EQUALS);
+ List searchList = new ArrayList<>();
+ searchList.add(searchOperator);
+ List<Resource> collections = (List<Resource>) resourceService.search(searchList);
+ LOGGER.info("Size", collections.size());
+ for (Resource collection : collections) {
+ LOGGER.info("#############" + collection.getName() + "Created At" + collection.getCreatedAt());
+ }
- return tenant;
+ return tenant;
}
diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Entity.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Entity.java
index 69fbdca..aba5035 100644
--- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Entity.java
+++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Entity.java
@@ -35,6 +35,8 @@ public abstract class Entity {
@Property(name = "tenant_id")
private String tenantId;
+
+
public Long getId() {
return id;
}
@@ -106,4 +108,9 @@ public abstract class Entity {
public void setId(Long id) {
this.id = id;
}
+
+
+ public String getSearchableId() {
+ return primaryExternalKey+"@"+tenantId;
+ }
}
diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Group.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Group.java
index c3362c5..9b735ec 100644
--- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Group.java
+++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Group.java
@@ -142,6 +142,11 @@ public class Group extends Entity {
this.memberUsersInPointers = memberUsersInPointers;
}
+ @Override
+ public String getSearchableId() {
+ return this.getName()+"@"+this.getTenantId();
+ }
+
public void addChildUser(User user, String membership, long relationshipCreatedAt,
long relationshipModifiedAt, Map<String, String> relationshipProperties) {
MemberOf memberOf = new MemberOf();
diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Resource.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Resource.java
index b9808bf..a61848b 100644
--- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Resource.java
+++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Resource.java
@@ -108,6 +108,11 @@ public class Resource extends Entity {
}
@Override
+ public String getSearchableId() {
+ return this.getName()+"@"+this.getTenantId();
+ }
+
+ @Override
public Long getId() {
return id;
}
diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Tenant.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Tenant.java
index 7967386..5dbe1ff 100644
--- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Tenant.java
+++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Tenant.java
@@ -101,6 +101,11 @@ public class Tenant extends Entity {
this.entities.add(entity);
}
+ @Override
+ public String getSearchableId() {
+ return this.getName()+"@"+this.getTenantId();
+ }
+
public void add(Entity entity, long relationShipCreatedAt, long relationShipModifiedAt,
Map<String, String> relationshipProperties) {
diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/User.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/User.java
index d20f560..10a00ff 100644
--- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/User.java
+++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/User.java
@@ -129,4 +129,9 @@ public class User extends Entity {
public void addAccessibleResources(HasAccess accessibleResources) {
this.accessibleResources.add(accessibleResources);
}
+
+ @Override
+ public String getSearchableId() {
+ return this.getUsername() + "@" + this.getTenantId();
+ }
}
diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/handlers/TenantServiceHandler.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/handlers/TenantServiceHandler.java
index d4fce42..66a2f42 100644
--- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/handlers/TenantServiceHandler.java
+++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/handlers/TenantServiceHandler.java
@@ -33,7 +33,8 @@ public class TenantServiceHandler extends TenantMetadataServiceGrpc.TenantMetada
try {
Tenant tenant = request.getTenant();
org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Tenant parsedTenant =
- tenantParser.parseTenant(tenant);
+ (org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Tenant)
+ tenantParser.parse(tenant);
TenantServiceImpl tenantService = new TenantServiceImpl(connector);
tenantService.createOrUpdate(parsedTenant);
diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/ExecutionContext.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/ExecutionContext.java
new file mode 100644
index 0000000..0a8365a
--- /dev/null
+++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/ExecutionContext.java
@@ -0,0 +1,17 @@
+package org.apache.airavata.datalake.metadata.parsers;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class ExecutionContext {
+
+ private Map<String, Object> neo4JConvertedModels = new HashMap<>();
+
+ public Object getNeo4JConvertedModels(String key) {
+ return neo4JConvertedModels.get(key);
+ }
+
+ public void addNeo4JConvertedModels(String key, Object obj) {
+ this.neo4JConvertedModels.put(key, obj);
+ }
+}
diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/GroupParser.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/GroupParser.java
index eb31649..f486841 100644
--- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/GroupParser.java
+++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/GroupParser.java
@@ -1,5 +1,7 @@
package org.apache.airavata.datalake.metadata.parsers;
+import com.google.protobuf.GeneratedMessageV3;
+import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Entity;
import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group;
import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User;
import org.apache.airavata.datalake.metadata.service.GroupMembership;
@@ -10,59 +12,79 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.List;
-import java.util.Optional;
@Component
-public class GroupParser {
+public class GroupParser implements Parser {
private static final Logger LOGGER = LoggerFactory.getLogger(GroupParser.class);
@Autowired
private DozerBeanMapper dozerBeanMapper;
- public Group parseGroup(org.apache.airavata.datalake.metadata.service.Group group, Optional<Group> parentGroup) {
- Group newParentGroup;
- if (parentGroup.isEmpty()) {
- newParentGroup = dozerBeanMapper.map(group,
- org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group.class);
+ @Override
+ public Entity parse(GeneratedMessageV3 entity, Entity parentEntity, ExecutionContext executionContext) {
+ if (entity instanceof org.apache.airavata.datalake.metadata.service.Group) {
+ org.apache.airavata.datalake.metadata.service.Group group = (org.apache.airavata.datalake.metadata.service.Group) entity;
+ Group newParentGroup = null;
+ if (parentEntity == null) {
+ newParentGroup = dozerBeanMapper.map(group,
+ org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group.class);
+ LOGGER.info("Creating group "+ newParentGroup.getName() + " class"+ newParentGroup.toString());
+ executionContext.addNeo4JConvertedModels(newParentGroup.getSearchableId(),newParentGroup);
+ } else if (parentEntity != null){
+ newParentGroup = (Group) parentEntity;
+ Group childGroup = dozerBeanMapper.map(group,
+ org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group.class);
+ executionContext.addNeo4JConvertedModels(childGroup.getSearchableId(),childGroup);
+ LOGGER.info("Creating group "+ newParentGroup.getName() + " class"+ newParentGroup.toString());
+ newParentGroup.addChildGroup(childGroup,
+ childGroup.getCreatedAt() != 0 ? childGroup.getCreatedAt() : System.currentTimeMillis(),
+ childGroup.getLastModifiedAt() != 0 ? childGroup.getLastModifiedAt() : System.currentTimeMillis(),
+ null); // Improve this with relatioship propertie
+
+ newParentGroup = childGroup;
+ }
+
+ List<GroupMembership> groupMemberships = group.getGroupMembershipList();
+
+ if (!groupMemberships.isEmpty()) {
+ Group finalNewParentGroup1 = newParentGroup;
+ groupMemberships.forEach(mebership -> {
+ User usr = dozerBeanMapper.map(mebership.getUser(), org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User.class);
+ executionContext.addNeo4JConvertedModels(usr.getSearchableId(),usr);
+ finalNewParentGroup1.addChildUser(usr, mebership.getMembershipType(),
+ mebership.getCreatedAt() != 0 ? mebership.getCreatedAt() : System.currentTimeMillis(),
+ mebership.getLastModifiedAt() != 0 ? mebership.getLastModifiedAt() : System.currentTimeMillis(),
+ null);
+ });
+
+ }
+
+ List<org.apache.airavata.datalake.metadata.service.Group> groups = group.getChildGroupsList();
+
+ if (!groups.isEmpty()) {
+
+ Group finalNewParentGroup = newParentGroup;
+ groups.forEach(gr -> {
+ this.parse(gr, finalNewParentGroup, executionContext);
+ });
+ }
+
+ return newParentGroup;
} else {
- newParentGroup = parentGroup.get();
- Group childGroup = dozerBeanMapper.map(group,
- org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group.class);
- newParentGroup.addChildGroup(childGroup,
- childGroup.getCreatedAt() != 0 ? childGroup.getCreatedAt() : System.currentTimeMillis(),
- childGroup.getLastModifiedAt() != 0 ? childGroup.getLastModifiedAt() : System.currentTimeMillis(),
- null); // Improve this with relatioship propertie
-
- newParentGroup = childGroup;
+ String msg = "Wrong entity type detected for parser Group Parser, Expected Group";
+ LOGGER.error(msg);
+ throw new RuntimeException(msg);
}
-
- List<GroupMembership> groupMemberships = group.getGroupMembershipList();
-
- if (!groupMemberships.isEmpty()) {
- Group finalNewParentGroup1 = newParentGroup;
- groupMemberships.forEach(mebership -> {
- User usr = dozerBeanMapper.map(mebership.getUser(), org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User.class);
- finalNewParentGroup1.addChildUser(usr, mebership.getMembershipType(),
- mebership.getCreatedAt() != 0 ? mebership.getCreatedAt() : System.currentTimeMillis(),
- mebership.getLastModifiedAt() != 0 ? mebership.getLastModifiedAt() : System.currentTimeMillis(),
- null);
- });
-
- }
-
- List<org.apache.airavata.datalake.metadata.service.Group> groups = group.getChildGroupsList();
-
- if (!groups.isEmpty()) {
-
- Group finalNewParentGroup = newParentGroup;
- groups.forEach(gr -> {
- this.parseGroup(gr, Optional.of(finalNewParentGroup));
- });
- }
- return newParentGroup;
-
}
+ @Override
+ public Entity parse(GeneratedMessageV3 entity, ExecutionContext executionContext) {
+ return this.parse(entity,null, executionContext);
+ }
+ @Override
+ public Entity parse(GeneratedMessageV3 entity) {
+ return this.parse(entity,null,new ExecutionContext());
+ }
}
diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/Parser.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/Parser.java
new file mode 100644
index 0000000..9ed0683
--- /dev/null
+++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/Parser.java
@@ -0,0 +1,13 @@
+package org.apache.airavata.datalake.metadata.parsers;
+
+import com.google.protobuf.GeneratedMessageV3;
+import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Entity;
+
+public interface Parser {
+
+ public Entity parse(GeneratedMessageV3 entity, Entity parentEntity, ExecutionContext executionContext);
+
+ public Entity parse(GeneratedMessageV3 entity, ExecutionContext executionContext);
+
+ public Entity parse(GeneratedMessageV3 entity);
+}
diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/ResourceParser.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/ResourceParser.java
index 6a14446..29a7a95 100644
--- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/ResourceParser.java
+++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/ResourceParser.java
@@ -1,6 +1,10 @@
package org.apache.airavata.datalake.metadata.parsers;
+import com.google.protobuf.GeneratedMessageV3;
+import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Entity;
+import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group;
import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Resource;
+import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User;
import org.apache.airavata.datalake.metadata.service.ResourceSharings;
import org.dozer.DozerBeanMapper;
import org.slf4j.Logger;
@@ -9,69 +13,113 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.List;
-import java.util.Optional;
@Component
-public class ResourceParser {
+public class ResourceParser implements Parser {
private static final Logger LOGGER = LoggerFactory.getLogger(ResourceParser.class);
@Autowired
private DozerBeanMapper dozerBeanMapper;
+ @Autowired
+ private UserParser userParser;
- public Resource parseResource(org.apache.airavata.datalake.metadata.service.Resource resource,
- Optional<Resource> parentResource) {
- Resource newParentResource;
- if (parentResource.isEmpty()) {
- newParentResource = dozerBeanMapper.map(resource,
- org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Resource.class);
+ @Autowired
+ private GroupParser groupParser;
+
+
+ @Override
+ public Entity parse(GeneratedMessageV3 entity, Entity parentEntity, ExecutionContext executionContext) {
+ if (entity instanceof org.apache.airavata.datalake.metadata.service.Resource) {
+ org.apache.airavata.datalake.metadata.service.Resource resource =
+ (org.apache.airavata.datalake.metadata.service.Resource) entity;
+
+ Resource newParentResource = null;
+ if (parentEntity == null) {
+ newParentResource = dozerBeanMapper.map(resource,
+ org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Resource.class);
+ executionContext.addNeo4JConvertedModels(newParentResource.getSearchableId(), newParentResource);
+ } else {
+ newParentResource = (Resource) parentEntity;
+ Resource childResource = dozerBeanMapper.map(resource,
+ org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Resource.class);
+ executionContext.addNeo4JConvertedModels(newParentResource.getSearchableId(), newParentResource);
+ newParentResource.addChildResource(childResource,
+ childResource.getCreatedAt() != 0 ? childResource.getCreatedAt() : System.currentTimeMillis(),
+ childResource.getLastModifiedAt() != 0 ? childResource.getLastModifiedAt() : System.currentTimeMillis(),
+ null); // Improve this with relatioship properties
+ newParentResource = childResource;
+ }
+
+
+ List<ResourceSharings> resourceSharings = (resource).getSharingsList();
+
+ if (!resourceSharings.isEmpty()) {
+ Resource finalNewParentResource1 = newParentResource;
+ resourceSharings.forEach(reshr -> {
+ if (!reshr.getUsersList().isEmpty()) {
+ reshr.getUsersList().forEach(shr -> {
+ User user = null;
+ Object obj = executionContext.getNeo4JConvertedModels(shr.getUsername() + "@" + shr.getTenantId());
+ if (obj != null) {
+ user = (User) obj;
+ } else {
+ user = (User) userParser.parse(shr, executionContext);
+ executionContext.addNeo4JConvertedModels(user.getSearchableId(), user);
+ }
+ finalNewParentResource1.shareWithAUser(user,
+ reshr.getPermissionType(),
+ reshr.getCreatedAt() != 0 ? reshr.getCreatedAt() : System.currentTimeMillis(),
+ reshr.getLastModifiedAt() != 0 ? reshr.getLastModifiedAt() : System.currentTimeMillis(),
+ null);
+
+ });
+
+ } else if (!reshr.getGroupsList().isEmpty()) {
+ reshr.getGroupsList().forEach(gr -> {
+ Group group = null;
+ Object obj = executionContext.getNeo4JConvertedModels(gr.getName() + "@" + gr.getTenantId());
+ if (obj != null) {
+ group = (Group) obj;
+ } else {
+ group = (Group) groupParser.parse(gr, executionContext);
+ executionContext.addNeo4JConvertedModels(group.getSearchableId(), group);
+ }
+
+ LOGGER.info("Group resource " + group.toString());
+ finalNewParentResource1.shareWithAGroup(group, reshr.getPermissionType(),
+ reshr.getCreatedAt() != 0 ? reshr.getCreatedAt() : System.currentTimeMillis(),
+ reshr.getLastModifiedAt() != 0 ? reshr.getLastModifiedAt() : System.currentTimeMillis(),
+ null);
+ });
+ }
+ });
+ }
+
+
+ List<org.apache.airavata.datalake.metadata.service.Resource> resources = resource.getChildResourcesList();
+
+ if (!resources.isEmpty()) {
+ Resource finalNewParentResource = newParentResource;
+ resources.forEach(gr -> {
+ this.parse(gr, finalNewParentResource, executionContext);
+ });
+ }
+ return newParentResource;
} else {
- newParentResource = parentResource.get();
- Resource childResource = dozerBeanMapper.map(resource,
- org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Resource.class);
- newParentResource.addChildResource(childResource,
- childResource.getCreatedAt() != 0 ? childResource.getCreatedAt() : System.currentTimeMillis(),
- childResource.getLastModifiedAt() != 0 ? childResource.getLastModifiedAt() : System.currentTimeMillis(),
- null); // Improve this with relatioship properties
- newParentResource = childResource;
- }
-
-
- List<ResourceSharings> resourceSharings = resource.getSharingsList();
-
- if (! resourceSharings.isEmpty()) {
- resourceSharings.forEach(reshr-> {
- if (!reshr.getUsersList().isEmpty()) {
- reshr.getUsersList().forEach(shr-> {
- // newParentResource.shareWithAUser(shr,reshr.get);
-
- });
-
- } else if (!reshr.getGroupsList().isEmpty()) {
-
- }
-
-
- });
-
-
+ String msg = "Wrong entity type detected for parser Resource Parser, Expected Resource";
+ LOGGER.error(msg);
+ throw new RuntimeException(msg);
}
-
-
-
-
-
- List<org.apache.airavata.datalake.metadata.service.Resource> resources = resource.getChildResourcesList();
-
- if (!resources.isEmpty()) {
- Resource finalNewParentResource = newParentResource;
- resources.forEach(gr -> {
- this.parseResource(gr, Optional.of(finalNewParentResource));
- });
- }
- return newParentResource;
-
}
+ @Override
+ public Entity parse(GeneratedMessageV3 entity, ExecutionContext executionContext) {
+ return this.parse(entity, null, executionContext);
+ }
+ @Override
+ public Entity parse(GeneratedMessageV3 entity) {
+ return this.parse(entity, null, new ExecutionContext());
+ }
}
diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/TenantParser.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/TenantParser.java
index c31379d..efba859 100644
--- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/TenantParser.java
+++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/TenantParser.java
@@ -1,6 +1,7 @@
package org.apache.airavata.datalake.metadata.parsers;
-import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Tenant;
+import com.google.protobuf.GeneratedMessageV3;
+import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Entity;
import org.apache.airavata.datalake.metadata.service.Group;
import org.apache.airavata.datalake.metadata.service.Resource;
import org.apache.airavata.datalake.metadata.service.User;
@@ -11,10 +12,9 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.List;
-import java.util.Optional;
@Component
-public class TenantParser {
+public class TenantParser implements Parser {
private static Logger LOGGER = LoggerFactory.getLogger(TenantParser.class);
@Autowired
@@ -29,51 +29,73 @@ public class TenantParser {
@Autowired
private ResourceParser resourceParser;
-
- public Tenant parseTenant(org.apache.airavata.datalake.metadata.service.Tenant tenant) {
- List<Group> groups = tenant.getGroupsList();
- List<User> users = tenant.getUsersList();
- List<Resource> resources = tenant.getResourcesList();
-
-
- org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Tenant neo4JTenant =
- dozerBeanMapper.map(tenant, org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Tenant.class);
-
- if (!groups.isEmpty()) {
- groups.stream().forEach(group -> {
- org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group neo4JGr =
- groupParser.parseGroup(group, Optional.empty());
- neo4JTenant.add(neo4JGr, tenant.getCreatedAt() != 0 ? tenant.getCreatedAt() : System.currentTimeMillis(),
- tenant.getLastModifiedAt() != 0 ? tenant.getLastModifiedAt() : System.currentTimeMillis(),
- null);
-
- });
- }
-
- if (!users.isEmpty()) {
- users.stream().forEach(user -> {
- org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User usr =
- userParser.parseUser(user);
- neo4JTenant.add(usr, tenant.getCreatedAt() != 0 ? tenant.getCreatedAt() : System.currentTimeMillis(),
- tenant.getLastModifiedAt() != 0 ? tenant.getLastModifiedAt() : System.currentTimeMillis(),
- null);
-
- });
- }
-
- if (!resources.isEmpty()) {
- resources.stream().forEach(resource -> {
- org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Resource neo4JResource =
- resourceParser.parseResource(resource, Optional.empty());
- neo4JTenant.add(neo4JResource, tenant.getCreatedAt() != 0 ? tenant.getCreatedAt() : System.currentTimeMillis(),
- tenant.getLastModifiedAt() != 0 ? tenant.getLastModifiedAt() : System.currentTimeMillis(),
- null);
-
- });
+ @Override
+ public Entity parse(GeneratedMessageV3 entity, Entity parentEntity, ExecutionContext executionContext) {
+ if (entity instanceof org.apache.airavata.datalake.metadata.service.Tenant) {
+
+ org.apache.airavata.datalake.metadata.service.Tenant tenant =
+ (org.apache.airavata.datalake.metadata.service.Tenant) entity;
+
+ List<Group> groups = tenant.getGroupsList();
+ List<User> users = tenant.getUsersList();
+ List<Resource> resources = tenant.getResourcesList();
+
+ org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Tenant neo4JTenant =
+ dozerBeanMapper.map(tenant,
+ org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Tenant.class);
+ neo4JTenant.setPrimaryExternalKey(neo4JTenant.getTenantId());
+ executionContext.addNeo4JConvertedModels(neo4JTenant.getSearchableId(),neo4JTenant);
+
+ if (!groups.isEmpty()) {
+ groups.stream().forEach(group -> {
+ org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group neo4JGr =
+ (org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group)
+ groupParser.parse(group, null, executionContext);
+ executionContext.addNeo4JConvertedModels(neo4JGr.getSearchableId(),neo4JGr);
+ neo4JTenant.add(neo4JGr, tenant.getCreatedAt() != 0 ? tenant.getCreatedAt() : System.currentTimeMillis(),
+ tenant.getLastModifiedAt() != 0 ? tenant.getLastModifiedAt() : System.currentTimeMillis(),
+ null);
+ });
+ }
+
+ if (!users.isEmpty()) {
+ users.stream().forEach(user -> {
+ org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User usr =
+ (org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User)
+ userParser.parse(user, null, executionContext);
+ executionContext.addNeo4JConvertedModels(usr.getSearchableId(),usr);
+ neo4JTenant.add(usr, tenant.getCreatedAt() != 0 ? tenant.getCreatedAt() : System.currentTimeMillis(),
+ tenant.getLastModifiedAt() != 0 ? tenant.getLastModifiedAt() : System.currentTimeMillis(),
+ null);
+ });
+ }
+
+ if (!resources.isEmpty()) {
+ resources.stream().forEach(resource -> {
+ org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Resource neo4JResource =
+ (org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Resource)
+ resourceParser.parse(resource, null, executionContext);
+ executionContext.addNeo4JConvertedModels(neo4JResource.getSearchableId(),neo4JResource);
+ neo4JTenant.add(neo4JResource, tenant.getCreatedAt() != 0 ? tenant.getCreatedAt() : System.currentTimeMillis(),
+ tenant.getLastModifiedAt() != 0 ? tenant.getLastModifiedAt() : System.currentTimeMillis(),
+ null);
+ });
+ }
+ return neo4JTenant;
+ } else {
+ String msg = "Wrong entity type detected for parser Tenant Parser, Expected Tenant";
+ LOGGER.error(msg);
+ throw new RuntimeException(msg);
}
- return neo4JTenant;
-
}
+ @Override
+ public Entity parse(GeneratedMessageV3 entity, ExecutionContext executionContext) {
+ return this.parse(entity, null, executionContext);
+ }
+ @Override
+ public Entity parse(GeneratedMessageV3 entity) {
+ return this.parse(entity, null, new ExecutionContext());
+ }
}
diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/UserParser.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/UserParser.java
index 2903381..371c316 100644
--- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/UserParser.java
+++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/UserParser.java
@@ -1,22 +1,43 @@
package org.apache.airavata.datalake.metadata.parsers;
-import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User;
+import com.google.protobuf.GeneratedMessageV3;
+import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Entity;
import org.dozer.DozerBeanMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
-public class UserParser {
+public class UserParser implements Parser {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(UserParser.class);
@Autowired
private DozerBeanMapper dozerBeanMapper;
- public User parseUser(org.apache.airavata.datalake.metadata.service.User user) {
-
- return dozerBeanMapper.map(user,
- org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User.class);
+ @Override
+ public Entity parse(GeneratedMessageV3 entity, Entity parentEntity, ExecutionContext executionContext) {
+ if (entity instanceof org.apache.airavata.datalake.metadata.service.User) {
+ org.apache.airavata.datalake.metadata.service.User user =
+ (org.apache.airavata.datalake.metadata.service.User) entity;
+ return dozerBeanMapper.map(user,
+ org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User.class);
+ } else {
+ String msg = "Wrong entity type detected for parser User Parser, Expected User";
+ LOGGER.error(msg);
+ throw new RuntimeException(msg);
+ }
}
+ @Override
+ public Entity parse(GeneratedMessageV3 entity, ExecutionContext executionContext) {
+ return this.parse(entity, null, executionContext);
+ }
+ @Override
+ public Entity parse(GeneratedMessageV3 entity) {
+ return this.parse(entity, null, new ExecutionContext());
+ }
}
diff --git a/metadata-service/db-service/stub/src/main/proto/resource/Resource.proto b/metadata-service/db-service/stub/src/main/proto/resource/Resource.proto
index 81f2e3a..ce52a04 100644
--- a/metadata-service/db-service/stub/src/main/proto/resource/Resource.proto
+++ b/metadata-service/db-service/stub/src/main/proto/resource/Resource.proto
@@ -47,12 +47,13 @@ message ResourceSharings {
string shared_entity_id = 1;
string shared_entity_type = 2;
string sharing_type = 3;
- map<string, string> properties = 4;
- int64 created_at = 5;
- int64 last_modified_at = 6;
- string resource_id = 7;
- repeated User users = 8;
- repeated Group groups = 9;
+ string permission_type = 4;
+ map<string, string> properties = 5;
+ int64 created_at = 6;
+ int64 last_modified_at = 7;
+ string resource_id = 8;
+ repeated User users = 9;
+ repeated Group groups = 10;
}