You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2021/08/18 22:57:54 UTC
[airavata-data-lake] branch master updated: Supporting admin and
owner levels
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
The following commit(s) were added to refs/heads/master by this push:
new 068f161 Supporting admin and owner levels
068f161 is described below
commit 068f1616b0319ea82c6fadfc5a7863bc3abe3597
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Wed Aug 18 18:57:45 2021 -0400
Supporting admin and owner levels
---
.../persistance/entity/DataOrchestratorEntity.java | 25 +++++-----
.../persistance/entity/OwnershipEntity.java | 56 +++++++++++++++++++++
.../orchestrator/connectors/DRMSConnector.java | 57 ++++++++++++++++++++--
.../processor/InboundEventProcessor.java | 28 ++++++++---
.../processor/OutboundEventProcessor.java | 41 ++++++++++------
5 files changed, 166 insertions(+), 41 deletions(-)
diff --git a/data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/entity/DataOrchestratorEntity.java b/data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/entity/DataOrchestratorEntity.java
index d46372e..d1843fd 100644
--- a/data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/entity/DataOrchestratorEntity.java
+++ b/data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/entity/DataOrchestratorEntity.java
@@ -51,13 +51,11 @@ public class DataOrchestratorEntity {
@Column(nullable = false)
private Date occurredTime;
-
@Column(nullable = false)
@Temporal(TemporalType.TIMESTAMP)
@CreatedDate
private Date createdAt;
-
@Column(nullable = false)
@Temporal(TemporalType.TIMESTAMP)
@LastModifiedDate
@@ -70,9 +68,6 @@ public class DataOrchestratorEntity {
private String eventType;
@Column(nullable = false)
- private String ownerId;
-
- @Column(nullable = false)
private String tenantId;
@Column(nullable = false)
@@ -90,6 +85,9 @@ public class DataOrchestratorEntity {
@OneToMany(fetch = FetchType.EAGER, mappedBy = "dataOrchestratorEntity", orphanRemoval = true, cascade = CascadeType.ALL)
private Set<WorkflowEntity> workFlowEntities;
+ @OneToMany(fetch = FetchType.EAGER, mappedBy = "dataOrchestratorEntity", orphanRemoval = true, cascade = CascadeType.ALL)
+ private Set<OwnershipEntity> ownershipEntities;
+
public String getId() {
return id;
}
@@ -170,7 +168,6 @@ public class DataOrchestratorEntity {
this.workFlowEntities = workFlowEntities;
}
-
public String getError() {
return error;
}
@@ -187,14 +184,6 @@ public class DataOrchestratorEntity {
this.eventType = eventType;
}
- public String getOwnerId() {
- return ownerId;
- }
-
- public void setOwnerId(String ownerId) {
- this.ownerId = ownerId;
- }
-
public String getTenantId() {
return tenantId;
}
@@ -226,4 +215,12 @@ public class DataOrchestratorEntity {
public void setHostName(String hostName) {
this.hostName = hostName;
}
+
+ public Set<OwnershipEntity> getOwnershipEntities() {
+ return ownershipEntities;
+ }
+
+ public void setOwnershipEntities(Set<OwnershipEntity> ownershipEntities) {
+ this.ownershipEntities = ownershipEntities;
+ }
}
diff --git a/data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/entity/OwnershipEntity.java b/data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/entity/OwnershipEntity.java
new file mode 100644
index 0000000..ec6e4eb
--- /dev/null
+++ b/data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/entity/OwnershipEntity.java
@@ -0,0 +1,56 @@
+package org.apache.airavata.datalake.orchestrator.registry.persistance.entity;
+
+import org.springframework.data.jpa.domain.support.AuditingEntityListener;
+
+import javax.persistence.*;
+
+@Entity
+@Table(name = "OWNERSHIP_ENTITY")
+@EntityListeners(AuditingEntityListener.class)
+public class OwnershipEntity {
+
+ @Id
+ private String id;
+
+ @Column(name = "USER_ID")
+ private String userId;
+
+ @Column(name = "PERMISSION_ID")
+ private String permissionId;
+
+ @ManyToOne
+ @JoinColumn(name = "DATA_ORCHESTRATOR_ENTITY_ID")
+ private DataOrchestratorEntity dataOrchestratorEntity;
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getUserId() {
+ return userId;
+ }
+
+ public void setUserId(String userId) {
+ this.userId = userId;
+ }
+
+ public String getPermissionId() {
+ return permissionId;
+ }
+
+ public void setPermissionId(String permissionId) {
+ this.permissionId = permissionId;
+ }
+
+ public DataOrchestratorEntity getDataOrchestratorEntity() {
+ return dataOrchestratorEntity;
+ }
+
+ public void setDataOrchestratorEntity(DataOrchestratorEntity dataOrchestratorEntity) {
+ this.dataOrchestratorEntity = dataOrchestratorEntity;
+ }
+}
diff --git a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/DRMSConnector.java b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/DRMSConnector.java
index 368bd3b..e24eb25 100644
--- a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/DRMSConnector.java
+++ b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/DRMSConnector.java
@@ -6,10 +6,13 @@ import org.apache.airavata.datalake.drms.AuthCredentialType;
import org.apache.airavata.datalake.drms.AuthenticatedUser;
import org.apache.airavata.datalake.drms.DRMSServiceAuthToken;
import org.apache.airavata.datalake.drms.resource.GenericResource;
+import org.apache.airavata.datalake.drms.sharing.DRMSSharingServiceGrpc;
+import org.apache.airavata.datalake.drms.sharing.ShareEntityWithUserRequest;
import org.apache.airavata.datalake.drms.storage.*;
import org.apache.airavata.datalake.orchestrator.Configuration;
import org.apache.airavata.datalake.orchestrator.core.connector.AbstractConnector;
import org.apache.airavata.datalake.orchestrator.registry.persistance.entity.DataOrchestratorEntity;
+import org.apache.airavata.datalake.orchestrator.registry.persistance.entity.OwnershipEntity;
import org.apache.airavata.datalake.orchestrator.registry.persistance.repository.DataOrchestratorEventRepository;
import org.apache.airavata.datalake.orchestrator.registry.persistance.entity.EventStatus;
import org.slf4j.Logger;
@@ -30,6 +33,7 @@ public class DRMSConnector implements AbstractConnector<Configuration> {
private ResourceServiceGrpc.ResourceServiceBlockingStub resourceServiceBlockingStub;
private StorageServiceGrpc.StorageServiceBlockingStub storageServiceBlockingStub;
private StoragePreferenceServiceGrpc.StoragePreferenceServiceBlockingStub storagePreferenceServiceBlockingStub;
+ private DRMSSharingServiceGrpc.DRMSSharingServiceBlockingStub sharingServiceBlockingStub;
public DRMSConnector(Configuration configuration) throws Exception {
this.init(configuration);
@@ -43,7 +47,7 @@ public class DRMSConnector implements AbstractConnector<Configuration> {
this.resourceServiceBlockingStub = ResourceServiceGrpc.newBlockingStub(drmsChannel);
this.storageServiceBlockingStub = StorageServiceGrpc.newBlockingStub(drmsChannel);
this.storagePreferenceServiceBlockingStub = StoragePreferenceServiceGrpc.newBlockingStub(drmsChannel);
-
+ this.sharingServiceBlockingStub = DRMSSharingServiceGrpc.newBlockingStub(drmsChannel);
}
@Override
@@ -56,12 +60,49 @@ public class DRMSConnector implements AbstractConnector<Configuration> {
return !this.drmsChannel.isShutdown();
}
- public Optional<TransferMapping> getActiveTransferMapping(DataOrchestratorEntity entity, String hostname) {
+ public void shareWithUser(DataOrchestratorEntity entity) throws Exception {
+
+ Optional<OwnershipEntity> adminOp = entity.getOwnershipEntities().stream().filter(o -> o.getPermissionId().equals("ADMIN")).findFirst();
+ if (adminOp.isEmpty()) {
+ throw new Exception("No admin user found");
+ }
+
+ DRMSServiceAuthToken serviceAuthToken = DRMSServiceAuthToken.newBuilder()
+ .setAccessToken(entity.getAuthToken())
+ .setAuthCredentialType(AuthCredentialType.AGENT_ACCOUNT_CREDENTIAL)
+ .setAuthenticatedUser(AuthenticatedUser.newBuilder()
+ .setUsername(adminOp.get().getUserId())
+ .setTenantId(entity.getTenantId())
+ .build())
+ .build();
+
+ for (OwnershipEntity ownershipEntity : entity.getOwnershipEntities()) {
+ if (ownershipEntity.getPermissionId().equals("ADMIN")) {
+ continue;
+ }
+ ShareEntityWithUserRequest.Builder shareBuilder = ShareEntityWithUserRequest.newBuilder()
+ .setAuthToken(serviceAuthToken)
+ .setEntityId(entity.getResourceId())
+ .setSharedUserId(ownershipEntity.getUserId())
+ .setPermissionId(ownershipEntity.getPermissionId());
+
+ this.sharingServiceBlockingStub.shareEntityWithUser(shareBuilder.build());
+ }
+
+ }
+
+ public Optional<TransferMapping> getActiveTransferMapping(DataOrchestratorEntity entity, String hostname) throws Exception {
+
+ Optional<OwnershipEntity> adminOp = entity.getOwnershipEntities().stream().filter(o -> o.getPermissionId().equals("ADMIN")).findFirst();
+ if (adminOp.isEmpty()) {
+ throw new Exception("No admin user found");
+ }
+
DRMSServiceAuthToken serviceAuthToken = DRMSServiceAuthToken.newBuilder()
.setAccessToken(entity.getAuthToken())
.setAuthCredentialType(AuthCredentialType.AGENT_ACCOUNT_CREDENTIAL)
.setAuthenticatedUser(AuthenticatedUser.newBuilder()
- .setUsername(entity.getOwnerId())
+ .setUsername(adminOp.get().getUserId())
.setTenantId(entity.getTenantId())
.build())
.build();
@@ -90,12 +131,18 @@ public class DRMSConnector implements AbstractConnector<Configuration> {
String resourceName,
String resourcePath,
String parentId,
- String type, String parentType) {
+ String type, String parentType) throws Exception {
+
+ Optional<OwnershipEntity> adminOp = entity.getOwnershipEntities().stream().filter(o -> o.getPermissionId().equals("ADMIN")).findFirst();
+ if (adminOp.isEmpty()) {
+ throw new Exception("No admin user found");
+ }
+
DRMSServiceAuthToken serviceAuthToken = DRMSServiceAuthToken.newBuilder()
.setAccessToken(entity.getAuthToken())
.setAuthCredentialType(AuthCredentialType.AGENT_ACCOUNT_CREDENTIAL)
.setAuthenticatedUser(AuthenticatedUser.newBuilder()
- .setUsername(entity.getOwnerId())
+ .setUsername(adminOp.get().getUserId())
.setTenantId(entity.getTenantId())
.build())
.build();
diff --git a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/InboundEventProcessor.java b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/InboundEventProcessor.java
index 83d9ada..62bd6ff 100644
--- a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/InboundEventProcessor.java
+++ b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/InboundEventProcessor.java
@@ -4,6 +4,7 @@ import org.apache.airavata.datalake.orchestrator.Configuration;
import org.apache.airavata.datalake.orchestrator.Utils;
import org.apache.airavata.datalake.orchestrator.core.processor.MessageProcessor;
import org.apache.airavata.datalake.orchestrator.registry.persistance.entity.DataOrchestratorEntity;
+import org.apache.airavata.datalake.orchestrator.registry.persistance.entity.OwnershipEntity;
import org.apache.airavata.datalake.orchestrator.registry.persistance.repository.DataOrchestratorEventRepository;
import org.apache.airavata.datalake.orchestrator.registry.persistance.entity.EventStatus;
import org.apache.airavata.dataorchestrator.messaging.model.NotificationEvent;
@@ -14,9 +15,7 @@ import org.slf4j.LoggerFactory;
import java.nio.charset.StandardCharsets;
import java.security.NoSuchAlgorithmException;
-import java.util.Arrays;
-import java.util.Base64;
-import java.util.Date;
+import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -114,8 +113,25 @@ public class InboundEventProcessor implements MessageProcessor<Configuration> {
String basePath = event.getContext().getBasePath();
String removeBasePath = resourcePath.substring(basePath.length());
String[] splitted = removeBasePath.split("/");
- String ownerId = splitted[0];
- entity.setOwnerId(ownerId);
+
+ OwnershipEntity owner1 = new OwnershipEntity();
+ owner1.setId(UUID.randomUUID().toString());
+ owner1.setUserId(splitted[1]);
+ owner1.setPermissionId("OWNER");
+ owner1.setDataOrchestratorEntity(entity);
+
+ OwnershipEntity owner2 = new OwnershipEntity();
+ owner2.setId(UUID.randomUUID().toString());
+ owner2.setUserId(splitted[0]);
+ owner2.setPermissionId("ADMIN");
+ owner2.setDataOrchestratorEntity(entity);
+
+
+ Set<OwnershipEntity> owners = new HashSet<>();
+ owners.add(owner1);
+ owners.add(owner2);
+
+ entity.setOwnershipEntities(owners);
entity.setTenantId(event.getContext().getTenantId());
@@ -126,6 +142,4 @@ public class InboundEventProcessor implements MessageProcessor<Configuration> {
entity.setResourceId(Utils.getId(event.getResourceId()));
return entity;
}
-
-
}
diff --git a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/OutboundEventProcessor.java b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/OutboundEventProcessor.java
index ba1a07c..4b04005 100644
--- a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/OutboundEventProcessor.java
+++ b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/OutboundEventProcessor.java
@@ -9,6 +9,7 @@ import org.apache.airavata.datalake.orchestrator.connectors.DRMSConnector;
import org.apache.airavata.datalake.orchestrator.connectors.WorkflowServiceConnector;
import org.apache.airavata.datalake.orchestrator.core.processor.MessageProcessor;
import org.apache.airavata.datalake.orchestrator.registry.persistance.entity.DataOrchestratorEntity;
+import org.apache.airavata.datalake.orchestrator.registry.persistance.entity.OwnershipEntity;
import org.apache.airavata.datalake.orchestrator.registry.persistance.repository.DataOrchestratorEventRepository;
import org.apache.airavata.datalake.orchestrator.registry.persistance.entity.EventStatus;
import org.apache.airavata.dataorchestrator.messaging.model.NotificationEvent;
@@ -70,15 +71,19 @@ public class OutboundEventProcessor implements MessageProcessor<Configuration> {
entityMap.computeIfAbsent(entity.getResourceId(), list -> new ArrayList()).add(entity);
});
entityMap.forEach((key, value) -> {
- DataOrchestratorEntity entity = value.remove(0);
- processEvent(entity);
- value.forEach(val -> {
- val.setEventStatus(EventStatus.DATA_ORCH_PROCESSED_AND_SKIPPED.name());
- repository.save(val);
- });
+ try {
+ DataOrchestratorEntity entity = value.remove(0);
+ processEvent(entity);
+ value.forEach(val -> {
+ val.setEventStatus(EventStatus.DATA_ORCH_PROCESSED_AND_SKIPPED.name());
+ repository.save(val);
+ });
+ } catch (Exception e) {
+ LOGGER.error("Errored while processing event", e);
+ }
});
} catch (Exception ex) {
- LOGGER.error("Error while processing events {}", ex);
+ LOGGER.error("Error while processing events", ex);
}
}
@@ -86,9 +91,15 @@ public class OutboundEventProcessor implements MessageProcessor<Configuration> {
private void processEvent(DataOrchestratorEntity entity) {
try {
- String ownerId = entity.getOwnerId();
+ // TODO move this logic to file listener as this is EMC specific
+ Optional<OwnershipEntity> adminOp = entity.getOwnershipEntities().stream().filter(o -> o.getPermissionId().equals("ADMIN")).findFirst();
+ if (adminOp.isEmpty()) {
+ throw new Exception("No admin user found");
+ }
+
String resourcePath = entity.getResourcePath();
- String tail = resourcePath.substring(resourcePath.indexOf(ownerId));
+ String tail = resourcePath.substring(resourcePath.indexOf(adminOp.get().getUserId()));
+
String[] collections = tail.split("/");
Optional<TransferMapping> optionalStorPref = drmsConnector.getActiveTransferMapping(entity, entity.getHostName());
@@ -105,13 +116,13 @@ public class OutboundEventProcessor implements MessageProcessor<Configuration> {
String parentType = "Storage";
String parentId = sourceStorageId;
- for (int i = 1; i < collections.length - 1; i++) {
+ for (int i = 0; i < collections.length - 1; i++) {
String resourceName = collections[i];
String path = entity.getResourcePath().substring(0, entity.getResourcePath().indexOf(resourceName));
path = path.concat(resourceName);
String entityId = Utils.getId(path);
Optional<GenericResource> optionalGenericResource =
- this.drmsConnector.createResource(repository, entity, entityId, resourceName, path, sourceStorageId, "COLLECTION", parentType);
+ this.drmsConnector.createResource(repository, entity, entityId, resourceName, path, parentId, "COLLECTION", parentType);
if (optionalGenericResource.isPresent()) {
parentId = optionalGenericResource.get().getResourceId();
parentType = "COLLECTION";
@@ -141,10 +152,10 @@ public class OutboundEventProcessor implements MessageProcessor<Configuration> {
try {
Optional<AnyStoragePreference> storagePreferenceOptional = this.drmsConnector
- .getStoragePreference(entity.getAuthToken(), entity.getOwnerId(), entity.getTenantId(), sourceStorageId);
+ .getStoragePreference(entity.getAuthToken(), adminOp.get().getUserId(), entity.getTenantId(), sourceStorageId);
Optional<AnyStoragePreference> destinationPreferenceOptional = this.drmsConnector
- .getStoragePreference(entity.getAuthToken(), entity.getOwnerId(), entity.getTenantId(), destinationStorageId);
+ .getStoragePreference(entity.getAuthToken(), adminOp.get().getUserId(), entity.getTenantId(), destinationStorageId);
if (storagePreferenceOptional.isPresent() && destinationPreferenceOptional.isPresent()) {
String sourceCredentialToken = storagePreferenceOptional.get()
.getSshStoragePreference()
@@ -153,13 +164,13 @@ public class OutboundEventProcessor implements MessageProcessor<Configuration> {
.getSshStoragePreference()
.getCredentialToken();
- this.workflowServiceConnector.invokeWorkflow(entity.getAuthToken(), entity.getOwnerId(),
+ this.workflowServiceConnector.invokeWorkflow(entity.getAuthToken(), adminOp.get().getUserId(),
entity.getTenantId(), entity.getResourceId(), sourceCredentialToken,
messageId, destinationCredentialToken);
entity.setEventStatus(EventStatus.DISPATCHED_TO_WORFLOW_ENGING.name());
repository.save(entity);
} else {
- String msg = "Cannot find storage preference for storage " + sourceStorageId + " for user " + entity.getOwnerId();
+ String msg = "Cannot find storage preference for storage " + sourceStorageId + " for user " + adminOp.get().getUserId();
entity.setError(msg);
entity.setEventStatus(EventStatus.ERRORED.name());
repository.save(entity);