You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by is...@apache.org on 2021/07/22 21:12:07 UTC
[airavata-data-lake] branch master updated: Bug fix in storage and
resource creation
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
The following commit(s) were added to refs/heads/master by this push:
new e09a57e Bug fix in storage and resource creation
new c7c7eb2 Merge pull request #30 from isururanawaka/workflow_invocation
e09a57e is described below
commit e09a57e923badf7e11b4868e0ab5d5a6dc1b6d23
Author: Isuru Ranawaka <ir...@gmail.com>
AuthorDate: Thu Jul 22 17:10:47 2021 -0400
Bug fix in storage and resource creation
---
.../orchestrator/connectors/DRMSConnector.java | 3 +-
.../processor/OutboundEventProcessor.java | 11 ++++----
.../src/main/resources/application.properties | 6 +++-
.../drms/api/handlers/ResourceServiceHandler.java | 7 +++--
.../src/main/resources/application.properties | 7 +++--
.../apache/airavata/drms/core/Neo4JConnector.java | 32 +++++++++++-----------
6 files changed, 38 insertions(+), 28 deletions(-)
diff --git a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/DRMSConnector.java b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/DRMSConnector.java
index a0b1292..d1513a7 100644
--- a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/DRMSConnector.java
+++ b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/DRMSConnector.java
@@ -90,7 +90,7 @@ public class DRMSConnector implements AbstractConnector<Configuration> {
String resourceName,
String resourcePath,
String parentId,
- String type) {
+ String type, String parentType) {
DRMSServiceAuthToken serviceAuthToken = DRMSServiceAuthToken.newBuilder()
.setAccessToken(entity.getAuthToken())
.setAuthCredentialType(AuthCredentialType.AGENT_ACCOUNT_CREDENTIAL)
@@ -106,6 +106,7 @@ public class DRMSConnector implements AbstractConnector<Configuration> {
.setResourceName(resourceName)
.setResourcePath(resourcePath)
.setType(type)
+ .putProperties("PARENT_TYPE", parentType)
.setParentId(parentId).build();
ResourceCreateRequest resourceCreateRequest = ResourceCreateRequest
.newBuilder()
diff --git a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/OutboundEventProcessor.java b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/OutboundEventProcessor.java
index ee098b5..dbf330b 100644
--- a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/OutboundEventProcessor.java
+++ b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/OutboundEventProcessor.java
@@ -102,6 +102,7 @@ public class OutboundEventProcessor implements MessageProcessor<Configuration> {
TransferMapping transferMapping = optionalStorPref.get();
String sourceStorageId = transferMapping.getSourceStorage().getSshStorage().getStorageId();
String destinationStorageId = transferMapping.getDestinationStorage().getSshStorage().getStorageId();
+ String parentType = "Storage";
String parentId = sourceStorageId;
for (int i = 1; i < collections.length - 1; i++) {
@@ -110,9 +111,10 @@ public class OutboundEventProcessor implements MessageProcessor<Configuration> {
path = path.concat(resourceName);
String entityId = Utils.getId(path);
Optional<GenericResource> optionalGenericResource =
- this.drmsConnector.createResource(repository, entity, entityId, resourceName, path, sourceStorageId, "COLLECTION");
+ this.drmsConnector.createResource(repository, entity, entityId, resourceName, path, sourceStorageId, "COLLECTION", parentType);
if (optionalGenericResource.isPresent()) {
parentId = optionalGenericResource.get().getResourceId();
+ parentType = "COLLECTION";
} else {
entity.setEventStatus(EventStatus.ERRORED.name());
entity.setError("Collection structure creation failed: " + entity.getHostName());
@@ -124,17 +126,16 @@ public class OutboundEventProcessor implements MessageProcessor<Configuration> {
Optional<GenericResource> optionalGenericResource =
this.drmsConnector.createResource(repository, entity, entity.getResourceId(),
collections[collections.length - 1], entity.getResourcePath(),
- parentId, "FILE");
+ parentId, "FILE", parentType);
- String dstResourceHost = transferMapping.getDestinationStorage().getSshStorage().getHostName();
- String destinationResourceId = dstResourceHost + ":" + entity.getResourcePath() + ":" + entity.getResourceType();
+ String destinationResourceId = destinationStorageId + ":" + entity.getResourcePath() + ":" + entity.getResourceType();
String messageId = Utils.getId(destinationResourceId);
Optional<GenericResource> destinationFile = this.drmsConnector.createResource(repository, entity, messageId,
entity.getResourceName(),
entity.getResourcePath(),
destinationStorageId,
- "FILE");
+ "FILE", "Storage");
if (optionalGenericResource.isPresent() && destinationFile.isPresent()) {
try {
diff --git a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/resources/application.properties b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/resources/application.properties
index b47d862..88dd260 100644
--- a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/resources/application.properties
+++ b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/resources/application.properties
@@ -13,4 +13,8 @@ spring.datasource.driver-class-name= com.mysql.cj.jdbc.Driver
spring.jpa.hibernate.naming.physical-strategy=org.hibernate.boot.model.naming.PhysicalNamingStrategyStandardImpl
config.path=/Users/isururanawaka/Documents/Airavata_Repository/airavata-data-lake/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/resources/config.yml
-server.port=8899
\ No newline at end of file
+server.port=8899
+mft.host=localhost
+mft.port=6565
+drms.host=localhost
+drms.port=7070
\ No newline at end of file
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
index 2cee672..6f3dda2 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
@@ -132,6 +132,7 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
String parentId = request.getResource().getParentId();
+
String entityId = request.getResource().getResourceId();
Map<String, Object> serializedMap = GenericResourceSerializer.serializeToMap(request.getResource());
Optional<Entity> exEntity = CustosUtils.mergeResourceEntity(custosClientProvider, callUser.getTenantId(),
@@ -150,9 +151,11 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
serializedMap.put("owner", exEntity.get().getOwnerId());
if (!parentId.isEmpty()) {
+ String parentLabel = request.getResource().getPropertiesMap().get("PARENT_TYPE");
+ serializedMap.remove("properties");
this.neo4JConnector.mergeNodesWithParentChildRelationShip(serializedMap, new HashMap<>(),
- request.getResource().getType(), StoragePreferenceConstants.STORAGE_PREFERENCE_LABEL,
- callUser.getUsername(), entityId, parentId, callUser.getTenantId());
+ request.getResource().getType(), parentLabel, callUser.getUsername(), entityId,
+ parentId, callUser.getTenantId());
} else {
this.neo4JConnector.mergeNode(serializedMap, request.getResource().getType(),
callUser.getUsername(), entityId, callUser.getTenantId());
diff --git a/data-resource-management-service/drms-api/src/main/resources/application.properties b/data-resource-management-service/drms-api/src/main/resources/application.properties
index 0d9237a..f19b8ca 100644
--- a/data-resource-management-service/drms-api/src/main/resources/application.properties
+++ b/data-resource-management-service/drms-api/src/main/resources/application.properties
@@ -21,9 +21,10 @@ neo4j.server.user=neo4j
neo4j.server.password=blastcovid19
group.service.host=localhost
group.service.port=6565
-custos.id=custos-whedmgamitu357p4wuke-10002708
-custos.secret=mrMdl86Ia1H94cikW7CvHoh7L0ASNXQVt2aRzSIj
+custos.id=custos-ii8g0cfwsz6ruwezykn9-10002640
+custos.secret=OxXECszt9dL4lHJQyL444UOU0lKN317D51ez067R
custos.host=custos.scigap.org
custos.port=31499
grpc.port=7070
-local.grpc.port=7070
\ No newline at end of file
+local.grpc.port=7070
+server.port=8081
\ No newline at end of file
diff --git a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/Neo4JConnector.java b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/Neo4JConnector.java
index 21088a9..9ab1a51 100644
--- a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/Neo4JConnector.java
+++ b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/Neo4JConnector.java
@@ -55,8 +55,8 @@ public class Neo4JConnector {
public List<Record> searchNodes(String query) {
Session session = driver.session();
- if(!session.isOpen()) {
- session = resume();
+ if (!session.isOpen()) {
+ session = resume();
}
Result result = session.run(query);
return result.list();
@@ -64,8 +64,8 @@ public class Neo4JConnector {
public List<Record> searchNodes(Map<String, Object> properties, String query) {
Session session = driver.session();
- if(!session.isOpen()) {
- session = resume();
+ if (!session.isOpen()) {
+ session = resume();
}
Result result = session.run(query, properties);
return result.list();
@@ -74,8 +74,8 @@ public class Neo4JConnector {
public void mergeNode(Map<String, Object> properties, String label, String userId, String entityId,
String tenantId) {
Session session = driver.session();
- if(!session.isOpen()) {
- session = resume();
+ if (!session.isOpen()) {
+ session = resume();
}
Map<String, Object> parameters = new HashMap<>();
properties.put("entityId", entityId);
@@ -97,8 +97,8 @@ public class Neo4JConnector {
String parentEntityId,
String tenantId) {
Session session = driver.session();
- if(!session.isOpen()) {
- session = resume();
+ if (!session.isOpen()) {
+ session = resume();
}
Map<String, Object> parameters = new HashMap<>();
childProperties.put("childEntityId", childEntityId);
@@ -122,11 +122,13 @@ public class Neo4JConnector {
tx.close();
}
+
+
public void deleteNode(String label, String entityId,
String tenantId) {
Session session = driver.session();
- if(!session.isOpen()) {
- session = resume();
+ if (!session.isOpen()) {
+ session = resume();
}
Map<String, Object> parameters = new HashMap<>();
parameters.put("entityId", entityId);
@@ -139,8 +141,8 @@ public class Neo4JConnector {
public void runTransactionalQuery(Map<String, Object> parameters, String query) {
Session session = driver.session();
- if(!session.isOpen()) {
- session = resume();
+ if (!session.isOpen()) {
+ session = resume();
}
Transaction tx = session.beginTransaction();
Result result = tx.run(query, parameters);
@@ -149,8 +151,6 @@ public class Neo4JConnector {
}
-
-
public void runTransactionalQuery(String query) {
Session session = driver.session();
Transaction tx = session.beginTransaction();
@@ -162,8 +162,8 @@ public class Neo4JConnector {
public void createMetadataNode(String parentLabel, String parentIdName, String parentIdValue,
String userId, String key, String value) {
Session session = driver.session();
- if(!session.isOpen()) {
- session = resume();
+ if (!session.isOpen()) {
+ session = resume();
}
Transaction tx = session.beginTransaction();