You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2021/08/18 22:57:54 UTC

[airavata-data-lake] branch master updated: Supporting admin and owner levels

This is an automated email from the ASF dual-hosted git repository.

dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git


The following commit(s) were added to refs/heads/master by this push:
     new 068f161  Supporting admin and owner levels
068f161 is described below

commit 068f1616b0319ea82c6fadfc5a7863bc3abe3597
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Wed Aug 18 18:57:45 2021 -0400

    Supporting admin and owner levels
---
 .../persistance/entity/DataOrchestratorEntity.java | 25 +++++-----
 .../persistance/entity/OwnershipEntity.java        | 56 +++++++++++++++++++++
 .../orchestrator/connectors/DRMSConnector.java     | 57 ++++++++++++++++++++--
 .../processor/InboundEventProcessor.java           | 28 ++++++++---
 .../processor/OutboundEventProcessor.java          | 41 ++++++++++------
 5 files changed, 166 insertions(+), 41 deletions(-)

diff --git a/data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/entity/DataOrchestratorEntity.java b/data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/entity/DataOrchestratorEntity.java
index d46372e..d1843fd 100644
--- a/data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/entity/DataOrchestratorEntity.java
+++ b/data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/entity/DataOrchestratorEntity.java
@@ -51,13 +51,11 @@ public class DataOrchestratorEntity {
     @Column(nullable = false)
     private Date occurredTime;
 
-
     @Column(nullable = false)
     @Temporal(TemporalType.TIMESTAMP)
     @CreatedDate
     private Date createdAt;
 
-
     @Column(nullable = false)
     @Temporal(TemporalType.TIMESTAMP)
     @LastModifiedDate
@@ -70,9 +68,6 @@ public class DataOrchestratorEntity {
     private String eventType;
 
     @Column(nullable = false)
-    private String ownerId;
-
-    @Column(nullable = false)
     private String tenantId;
 
     @Column(nullable = false)
@@ -90,6 +85,9 @@ public class DataOrchestratorEntity {
     @OneToMany(fetch = FetchType.EAGER, mappedBy = "dataOrchestratorEntity", orphanRemoval = true, cascade = CascadeType.ALL)
     private Set<WorkflowEntity> workFlowEntities;
 
+    @OneToMany(fetch = FetchType.EAGER, mappedBy = "dataOrchestratorEntity", orphanRemoval = true, cascade = CascadeType.ALL)
+    private Set<OwnershipEntity> ownershipEntities;
+
     public String getId() {
         return id;
     }
@@ -170,7 +168,6 @@ public class DataOrchestratorEntity {
         this.workFlowEntities = workFlowEntities;
     }
 
-
     public String getError() {
         return error;
     }
@@ -187,14 +184,6 @@ public class DataOrchestratorEntity {
         this.eventType = eventType;
     }
 
-    public String getOwnerId() {
-        return ownerId;
-    }
-
-    public void setOwnerId(String ownerId) {
-        this.ownerId = ownerId;
-    }
-
     public String getTenantId() {
         return tenantId;
     }
@@ -226,4 +215,12 @@ public class DataOrchestratorEntity {
     public void setHostName(String hostName) {
         this.hostName = hostName;
     }
+
+    public Set<OwnershipEntity> getOwnershipEntities() {
+        return ownershipEntities;
+    }
+
+    public void setOwnershipEntities(Set<OwnershipEntity> ownershipEntities) {
+        this.ownershipEntities = ownershipEntities;
+    }
 }
diff --git a/data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/entity/OwnershipEntity.java b/data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/entity/OwnershipEntity.java
new file mode 100644
index 0000000..ec6e4eb
--- /dev/null
+++ b/data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/entity/OwnershipEntity.java
@@ -0,0 +1,56 @@
+package org.apache.airavata.datalake.orchestrator.registry.persistance.entity;
+
+import org.springframework.data.jpa.domain.support.AuditingEntityListener;
+
+import javax.persistence.*;
+
+@Entity
+@Table(name = "OWNERSHIP_ENTITY")
+@EntityListeners(AuditingEntityListener.class)
+public class OwnershipEntity {
+
+    @Id
+    private String id;
+
+    @Column(name = "USER_ID")
+    private String userId;
+
+    @Column(name = "PERMISSION_ID")
+    private String permissionId;
+
+    @ManyToOne
+    @JoinColumn(name = "DATA_ORCHESTRATOR_ENTITY_ID")
+    private DataOrchestratorEntity dataOrchestratorEntity;
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getUserId() {
+        return userId;
+    }
+
+    public void setUserId(String userId) {
+        this.userId = userId;
+    }
+
+    public String getPermissionId() {
+        return permissionId;
+    }
+
+    public void setPermissionId(String permissionId) {
+        this.permissionId = permissionId;
+    }
+
+    public DataOrchestratorEntity getDataOrchestratorEntity() {
+        return dataOrchestratorEntity;
+    }
+
+    public void setDataOrchestratorEntity(DataOrchestratorEntity dataOrchestratorEntity) {
+        this.dataOrchestratorEntity = dataOrchestratorEntity;
+    }
+}
diff --git a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/DRMSConnector.java b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/DRMSConnector.java
index 368bd3b..e24eb25 100644
--- a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/DRMSConnector.java
+++ b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/DRMSConnector.java
@@ -6,10 +6,13 @@ import org.apache.airavata.datalake.drms.AuthCredentialType;
 import org.apache.airavata.datalake.drms.AuthenticatedUser;
 import org.apache.airavata.datalake.drms.DRMSServiceAuthToken;
 import org.apache.airavata.datalake.drms.resource.GenericResource;
+import org.apache.airavata.datalake.drms.sharing.DRMSSharingServiceGrpc;
+import org.apache.airavata.datalake.drms.sharing.ShareEntityWithUserRequest;
 import org.apache.airavata.datalake.drms.storage.*;
 import org.apache.airavata.datalake.orchestrator.Configuration;
 import org.apache.airavata.datalake.orchestrator.core.connector.AbstractConnector;
 import org.apache.airavata.datalake.orchestrator.registry.persistance.entity.DataOrchestratorEntity;
+import org.apache.airavata.datalake.orchestrator.registry.persistance.entity.OwnershipEntity;
 import org.apache.airavata.datalake.orchestrator.registry.persistance.repository.DataOrchestratorEventRepository;
 import org.apache.airavata.datalake.orchestrator.registry.persistance.entity.EventStatus;
 import org.slf4j.Logger;
@@ -30,6 +33,7 @@ public class DRMSConnector implements AbstractConnector<Configuration> {
     private ResourceServiceGrpc.ResourceServiceBlockingStub resourceServiceBlockingStub;
     private StorageServiceGrpc.StorageServiceBlockingStub storageServiceBlockingStub;
     private StoragePreferenceServiceGrpc.StoragePreferenceServiceBlockingStub storagePreferenceServiceBlockingStub;
+    private DRMSSharingServiceGrpc.DRMSSharingServiceBlockingStub sharingServiceBlockingStub;
 
     public DRMSConnector(Configuration configuration) throws Exception {
         this.init(configuration);
@@ -43,7 +47,7 @@ public class DRMSConnector implements AbstractConnector<Configuration> {
         this.resourceServiceBlockingStub = ResourceServiceGrpc.newBlockingStub(drmsChannel);
         this.storageServiceBlockingStub = StorageServiceGrpc.newBlockingStub(drmsChannel);
         this.storagePreferenceServiceBlockingStub = StoragePreferenceServiceGrpc.newBlockingStub(drmsChannel);
-
+        this.sharingServiceBlockingStub = DRMSSharingServiceGrpc.newBlockingStub(drmsChannel);
     }
 
     @Override
@@ -56,12 +60,49 @@ public class DRMSConnector implements AbstractConnector<Configuration> {
         return !this.drmsChannel.isShutdown();
     }
 
-    public Optional<TransferMapping> getActiveTransferMapping(DataOrchestratorEntity entity, String hostname) {
+    public void shareWithUser(DataOrchestratorEntity entity) throws Exception {
+
+        Optional<OwnershipEntity> adminOp = entity.getOwnershipEntities().stream().filter(o -> o.getPermissionId().equals("ADMIN")).findFirst();
+        if (adminOp.isEmpty()) {
+            throw new Exception("No admin user found");
+        }
+
+        DRMSServiceAuthToken serviceAuthToken = DRMSServiceAuthToken.newBuilder()
+                .setAccessToken(entity.getAuthToken())
+                .setAuthCredentialType(AuthCredentialType.AGENT_ACCOUNT_CREDENTIAL)
+                .setAuthenticatedUser(AuthenticatedUser.newBuilder()
+                        .setUsername(adminOp.get().getUserId())
+                        .setTenantId(entity.getTenantId())
+                        .build())
+                .build();
+
+        for (OwnershipEntity ownershipEntity : entity.getOwnershipEntities()) {
+            if (ownershipEntity.getPermissionId().equals("ADMIN")) {
+                continue;
+            }
+            ShareEntityWithUserRequest.Builder shareBuilder = ShareEntityWithUserRequest.newBuilder()
+                    .setAuthToken(serviceAuthToken)
+                    .setEntityId(entity.getResourceId())
+                    .setSharedUserId(ownershipEntity.getUserId())
+                    .setPermissionId(ownershipEntity.getPermissionId());
+
+            this.sharingServiceBlockingStub.shareEntityWithUser(shareBuilder.build());
+        }
+
+    }
+
+    public Optional<TransferMapping> getActiveTransferMapping(DataOrchestratorEntity entity, String hostname) throws Exception {
+
+        Optional<OwnershipEntity> adminOp = entity.getOwnershipEntities().stream().filter(o -> o.getPermissionId().equals("ADMIN")).findFirst();
+        if (adminOp.isEmpty()) {
+            throw new Exception("No admin user found");
+        }
+
         DRMSServiceAuthToken serviceAuthToken = DRMSServiceAuthToken.newBuilder()
                 .setAccessToken(entity.getAuthToken())
                 .setAuthCredentialType(AuthCredentialType.AGENT_ACCOUNT_CREDENTIAL)
                 .setAuthenticatedUser(AuthenticatedUser.newBuilder()
-                        .setUsername(entity.getOwnerId())
+                        .setUsername(adminOp.get().getUserId())
                         .setTenantId(entity.getTenantId())
                         .build())
                 .build();
@@ -90,12 +131,18 @@ public class DRMSConnector implements AbstractConnector<Configuration> {
                                                     String resourceName,
                                                     String resourcePath,
                                                     String parentId,
-                                                    String type, String parentType) {
+                                                    String type, String parentType) throws Exception {
+
+        Optional<OwnershipEntity> adminOp = entity.getOwnershipEntities().stream().filter(o -> o.getPermissionId().equals("ADMIN")).findFirst();
+        if (adminOp.isEmpty()) {
+            throw new Exception("No admin user found");
+        }
+
         DRMSServiceAuthToken serviceAuthToken = DRMSServiceAuthToken.newBuilder()
                 .setAccessToken(entity.getAuthToken())
                 .setAuthCredentialType(AuthCredentialType.AGENT_ACCOUNT_CREDENTIAL)
                 .setAuthenticatedUser(AuthenticatedUser.newBuilder()
-                        .setUsername(entity.getOwnerId())
+                        .setUsername(adminOp.get().getUserId())
                         .setTenantId(entity.getTenantId())
                         .build())
                 .build();
diff --git a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/InboundEventProcessor.java b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/InboundEventProcessor.java
index 83d9ada..62bd6ff 100644
--- a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/InboundEventProcessor.java
+++ b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/InboundEventProcessor.java
@@ -4,6 +4,7 @@ import org.apache.airavata.datalake.orchestrator.Configuration;
 import org.apache.airavata.datalake.orchestrator.Utils;
 import org.apache.airavata.datalake.orchestrator.core.processor.MessageProcessor;
 import org.apache.airavata.datalake.orchestrator.registry.persistance.entity.DataOrchestratorEntity;
+import org.apache.airavata.datalake.orchestrator.registry.persistance.entity.OwnershipEntity;
 import org.apache.airavata.datalake.orchestrator.registry.persistance.repository.DataOrchestratorEventRepository;
 import org.apache.airavata.datalake.orchestrator.registry.persistance.entity.EventStatus;
 import org.apache.airavata.dataorchestrator.messaging.model.NotificationEvent;
@@ -14,9 +15,7 @@ import org.slf4j.LoggerFactory;
 
 import java.nio.charset.StandardCharsets;
 import java.security.NoSuchAlgorithmException;
-import java.util.Arrays;
-import java.util.Base64;
-import java.util.Date;
+import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -114,8 +113,25 @@ public class InboundEventProcessor implements MessageProcessor<Configuration> {
         String basePath = event.getContext().getBasePath();
         String removeBasePath = resourcePath.substring(basePath.length());
         String[] splitted = removeBasePath.split("/");
-        String ownerId = splitted[0];
-        entity.setOwnerId(ownerId);
+
+        OwnershipEntity owner1 = new OwnershipEntity();
+        owner1.setId(UUID.randomUUID().toString());
+        owner1.setUserId(splitted[1]);
+        owner1.setPermissionId("OWNER");
+        owner1.setDataOrchestratorEntity(entity);
+
+        OwnershipEntity owner2 = new OwnershipEntity();
+        owner2.setId(UUID.randomUUID().toString());
+        owner2.setUserId(splitted[0]);
+        owner2.setPermissionId("ADMIN");
+        owner2.setDataOrchestratorEntity(entity);
+
+
+        Set<OwnershipEntity> owners = new HashSet<>();
+        owners.add(owner1);
+        owners.add(owner2);
+
+        entity.setOwnershipEntities(owners);
 
         entity.setTenantId(event.getContext().getTenantId());
 
@@ -126,6 +142,4 @@ public class InboundEventProcessor implements MessageProcessor<Configuration> {
         entity.setResourceId(Utils.getId(event.getResourceId()));
         return entity;
     }
-
-
 }
diff --git a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/OutboundEventProcessor.java b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/OutboundEventProcessor.java
index ba1a07c..4b04005 100644
--- a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/OutboundEventProcessor.java
+++ b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/OutboundEventProcessor.java
@@ -9,6 +9,7 @@ import org.apache.airavata.datalake.orchestrator.connectors.DRMSConnector;
 import org.apache.airavata.datalake.orchestrator.connectors.WorkflowServiceConnector;
 import org.apache.airavata.datalake.orchestrator.core.processor.MessageProcessor;
 import org.apache.airavata.datalake.orchestrator.registry.persistance.entity.DataOrchestratorEntity;
+import org.apache.airavata.datalake.orchestrator.registry.persistance.entity.OwnershipEntity;
 import org.apache.airavata.datalake.orchestrator.registry.persistance.repository.DataOrchestratorEventRepository;
 import org.apache.airavata.datalake.orchestrator.registry.persistance.entity.EventStatus;
 import org.apache.airavata.dataorchestrator.messaging.model.NotificationEvent;
@@ -70,15 +71,19 @@ public class OutboundEventProcessor implements MessageProcessor<Configuration> {
                 entityMap.computeIfAbsent(entity.getResourceId(), list -> new ArrayList()).add(entity);
             });
             entityMap.forEach((key, value) -> {
-                DataOrchestratorEntity entity = value.remove(0);
-                processEvent(entity);
-                value.forEach(val -> {
-                    val.setEventStatus(EventStatus.DATA_ORCH_PROCESSED_AND_SKIPPED.name());
-                    repository.save(val);
-                });
+                try {
+                    DataOrchestratorEntity entity = value.remove(0);
+                    processEvent(entity);
+                    value.forEach(val -> {
+                        val.setEventStatus(EventStatus.DATA_ORCH_PROCESSED_AND_SKIPPED.name());
+                        repository.save(val);
+                    });
+                } catch (Exception e) {
+                    LOGGER.error("Errored while processing event", e);
+                }
             });
         } catch (Exception ex) {
-            LOGGER.error("Error while processing events {}", ex);
+            LOGGER.error("Error while processing events", ex);
         }
 
     }
@@ -86,9 +91,15 @@ public class OutboundEventProcessor implements MessageProcessor<Configuration> {
     private void processEvent(DataOrchestratorEntity entity) {
         try {
 
-            String ownerId = entity.getOwnerId();
+            // TODO move this logic to file listener as this is EMC specific
+            Optional<OwnershipEntity> adminOp = entity.getOwnershipEntities().stream().filter(o -> o.getPermissionId().equals("ADMIN")).findFirst();
+            if (adminOp.isEmpty()) {
+                throw new Exception("No admin user found");
+            }
+
             String resourcePath = entity.getResourcePath();
-            String tail = resourcePath.substring(resourcePath.indexOf(ownerId));
+            String tail = resourcePath.substring(resourcePath.indexOf(adminOp.get().getUserId()));
+
             String[] collections = tail.split("/");
 
             Optional<TransferMapping> optionalStorPref = drmsConnector.getActiveTransferMapping(entity, entity.getHostName());
@@ -105,13 +116,13 @@ public class OutboundEventProcessor implements MessageProcessor<Configuration> {
             String parentType = "Storage";
 
             String parentId = sourceStorageId;
-            for (int i = 1; i < collections.length - 1; i++) {
+            for (int i = 0; i < collections.length - 1; i++) {
                 String resourceName = collections[i];
                 String path = entity.getResourcePath().substring(0, entity.getResourcePath().indexOf(resourceName));
                 path = path.concat(resourceName);
                 String entityId = Utils.getId(path);
                 Optional<GenericResource> optionalGenericResource =
-                        this.drmsConnector.createResource(repository, entity, entityId, resourceName, path, sourceStorageId, "COLLECTION", parentType);
+                        this.drmsConnector.createResource(repository, entity, entityId, resourceName, path, parentId, "COLLECTION", parentType);
                 if (optionalGenericResource.isPresent()) {
                     parentId = optionalGenericResource.get().getResourceId();
                     parentType = "COLLECTION";
@@ -141,10 +152,10 @@ public class OutboundEventProcessor implements MessageProcessor<Configuration> {
                 try {
 
                     Optional<AnyStoragePreference> storagePreferenceOptional = this.drmsConnector
-                            .getStoragePreference(entity.getAuthToken(), entity.getOwnerId(), entity.getTenantId(), sourceStorageId);
+                            .getStoragePreference(entity.getAuthToken(), adminOp.get().getUserId(), entity.getTenantId(), sourceStorageId);
 
                     Optional<AnyStoragePreference> destinationPreferenceOptional = this.drmsConnector
-                            .getStoragePreference(entity.getAuthToken(), entity.getOwnerId(), entity.getTenantId(), destinationStorageId);
+                            .getStoragePreference(entity.getAuthToken(), adminOp.get().getUserId(), entity.getTenantId(), destinationStorageId);
                     if (storagePreferenceOptional.isPresent() && destinationPreferenceOptional.isPresent()) {
                         String sourceCredentialToken = storagePreferenceOptional.get()
                                 .getSshStoragePreference()
@@ -153,13 +164,13 @@ public class OutboundEventProcessor implements MessageProcessor<Configuration> {
                                 .getSshStoragePreference()
                                 .getCredentialToken();
 
-                        this.workflowServiceConnector.invokeWorkflow(entity.getAuthToken(), entity.getOwnerId(),
+                        this.workflowServiceConnector.invokeWorkflow(entity.getAuthToken(), adminOp.get().getUserId(),
                                 entity.getTenantId(), entity.getResourceId(), sourceCredentialToken,
                                 messageId, destinationCredentialToken);
                         entity.setEventStatus(EventStatus.DISPATCHED_TO_WORFLOW_ENGING.name());
                         repository.save(entity);
                     } else {
-                        String msg = "Cannot find storage preference for storage " + sourceStorageId + " for user " + entity.getOwnerId();
+                        String msg = "Cannot find storage preference for storage " + sourceStorageId + " for user " + adminOp.get().getUserId();
                         entity.setError(msg);
                         entity.setEventStatus(EventStatus.ERRORED.name());
                         repository.save(entity);