You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by is...@apache.org on 2021/07/02 18:45:59 UTC
[airavata-data-lake] branch master updated: implement metadata add,
delete and search
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
The following commit(s) were added to refs/heads/master by this push:
new a590250 implement metadata add, delete and search
new d724101 Merge pull request #10 from isururanawaka/workflow_merge
a590250 is described below
commit a59025028b3233a73d3763ee05bbaa27820ada3d
Author: Isuru Ranawaka <ir...@gmail.com>
AuthorDate: Fri Jul 2 14:43:37 2021 -0400
implement metadata add, delete and search
---
data-resource-management-service/drms-api/pom.xml | 6 +
.../java/org/apache/airavata/drms/api/Client.java | 114 ++--
.../drms/api/handlers/ResourceServiceHandler.java | 339 ++++++++++--
.../org/apache/airavata/drms/api/utils/Utils.java | 38 +-
.../drms-api/src/main/resources/sample.json | 609 +++++++++++++++++++++
.../synchronizer/handlers/SharingHandler.java | 2 +
.../drms-rest-proxy/src/main/resources/drms.pb | Bin 107086 -> 112120 bytes
.../main/proto/resource/DRMSResourceService.proto | 182 +++---
8 files changed, 1127 insertions(+), 163 deletions(-)
diff --git a/data-resource-management-service/drms-api/pom.xml b/data-resource-management-service/drms-api/pom.xml
index 6068140..8eb1192 100644
--- a/data-resource-management-service/drms-api/pom.xml
+++ b/data-resource-management-service/drms-api/pom.xml
@@ -74,6 +74,12 @@
<artifactId>spring-security-core</artifactId>
<version>${spring-security.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.json</groupId>
+ <artifactId>json</artifactId>
+ <version>${org.json.version}</version>
+ </dependency>
+
</dependencies>
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Client.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Client.java
index 4ccdc8e..ece993c 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Client.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Client.java
@@ -17,17 +17,18 @@
package org.apache.airavata.drms.api;
import com.google.protobuf.Struct;
+import com.google.protobuf.util.JsonFormat;
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
import org.apache.airavata.datalake.drms.DRMSServiceAuthToken;
import org.apache.airavata.datalake.drms.resource.GenericResource;
-import org.apache.airavata.datalake.drms.storage.ParentResourcesFetchRequest;
-import org.apache.airavata.datalake.drms.storage.ResourceServiceGrpc;
-import org.apache.airavata.datalake.drms.storage.StoragePreferenceServiceGrpc;
-import org.apache.airavata.datalake.drms.storage.StorageServiceGrpc;
+import org.apache.airavata.datalake.drms.storage.*;
import org.apache.custos.clients.CustosClientProvider;
import org.apache.custos.identity.management.client.IdentityManagementClient;
+import org.json.JSONObject;
+import org.json.JSONTokener;
+import java.io.FileInputStream;
import java.util.ArrayList;
import java.util.List;
@@ -62,12 +63,15 @@ public class Client {
//
-// StorageSearchRequest storageSearchRequest = StorageSearchRequest
-// .newBuilder()
-// .setAuthToken(authToken)
-// .addQueries(StorageSearchQuery.newBuilder().build())
-// .build();
-//
+ StorageSearchRequest storageSearchRequest = StorageSearchRequest
+ .newBuilder()
+ .setAuthToken(authToken)
+ .addQueries(StorageSearchQuery.newBuilder()
+ .setField("type")
+ .setValue("COLLECTION")
+ .build())
+ .build();
+
// resourceClient.searchStorage(storageSearchRequest);
StoragePreferenceServiceGrpc.StoragePreferenceServiceBlockingStub storagePreferenceServiceBlockingStub = StoragePreferenceServiceGrpc
@@ -150,30 +154,29 @@ public class Client {
// System.out.println(authToken.getAccessToken());
// ResourceFetchRequest resourceFetchRequest = ResourceFetchRequest.newBuilder()
// .setAuthToken(authToken)
-// .setResourceId("COLLECTION_ONE_6kKQzhIt8zxvIgn")
-// .setType("COLLECTION")
+// .setResourceId("custos-whedmgamitu357p4wuke-10002708_29186.69999998808")
// .build();
//
// resourceServiceBlockingStub.fetchResource(resourceFetchRequest);
-
- GenericResource parentResource = GenericResource.newBuilder()
- .setResourceId("56cec8a2-a2c2-4669-9274-a5b5bdd97c11")
- .setType("COLLECTION")
- .build();
-
- GenericResource childResource = GenericResource.newBuilder()
- .setResourceId("b75b4cec-8df4-4f99-9d06-818db285cf02")
- .setType("COLLECTION")
- .build();
-
- GenericResource childResource1 = GenericResource.newBuilder()
- .setResourceId("b7ee2fd5-c4b8-4bb9-896b-4cb98d91ad24")
- .setType("COLLECTION")
- .build();
-
- List<GenericResource> genericResourceList = new ArrayList<>();
- genericResourceList.add(childResource);
- genericResourceList.add(childResource1);
+//
+// GenericResource parentResource = GenericResource.newBuilder()
+// .setResourceId("56cec8a2-a2c2-4669-9274-a5b5bdd97c11")
+// .setType("COLLECTION")
+// .build();
+//
+// GenericResource childResource = GenericResource.newBuilder()
+// .setResourceId("b75b4cec-8df4-4f99-9d06-818db285cf02")
+// .setType("COLLECTION")
+// .build();
+//
+// GenericResource childResource1 = GenericResource.newBuilder()
+// .setResourceId("b7ee2fd5-c4b8-4bb9-896b-4cb98d91ad24")
+// .setType("COLLECTION")
+// .build();
+//
+// List<GenericResource> genericResourceList = new ArrayList<>();
+// genericResourceList.add(childResource);
+// genericResourceList.add(childResource1);
// AddChildResourcesMembershipRequest addChildResourcesMembershipRequest = AddChildResourcesMembershipRequest
// .newBuilder()
@@ -192,14 +195,49 @@ public class Client {
// .build();
// resourceServiceBlockingStub.deleteChildMembership(deleteChildResourcesMembershipRequest);
- ParentResourcesFetchRequest parentResourcesFetchRequest = ParentResourcesFetchRequest
+// ParentResourcesFetchRequest parentResourcesFetchRequest = ParentResourcesFetchRequest
+// .newBuilder()
+// .setAuthToken(authToken)
+// .setResourceId("FILE_ONE_bxZPopxbnPaAEq5")
+// .setType("FILE")
+// .setDepth(2)
+// .build();
+// resourceServiceBlockingStub.fetchParentResources(parentResourcesFetchRequest);
+ try {
+ FileInputStream fileInputStream =
+ new FileInputStream(
+ "/Users/isururanawaka/Documents/Airavata_Repository/airavata-data-lake" +
+ "/data-resource-management-service/drms-api/src/main/resources/sample.json");
+ JSONTokener tokener = new JSONTokener(fileInputStream);
+ JSONObject root = new JSONObject(tokener);
+//
+ Struct.Builder structBuilder = Struct.newBuilder();
+ JsonFormat.parser().merge(root.toString(), structBuilder);
+
+// AddResourceMetadataRequest addResourceMetadataRequest = AddResourceMetadataRequest
+// .newBuilder()
+// .setMetadata(structBuilder.build())
+// .setAuthToken(authToken)
+// .setResourceId("custos-whedmgamitu357p4wuke-10002708_132068.39999997616")
+// .setType("FILE")
+// .build();
+//
+// resourceServiceBlockingStub.addResourceMetadata(addResourceMetadataRequest);
+
+ FetchResourceMetadataRequest addResourceMetadataRequest = FetchResourceMetadataRequest
.newBuilder()
.setAuthToken(authToken)
- .setResourceId("FILE_ONE_bxZPopxbnPaAEq5")
+ .setResourceId("custos-whedmgamitu357p4wuke-10002708_132068.39999997616")
.setType("FILE")
- .setDepth(2)
.build();
- resourceServiceBlockingStub.fetchParentResources(parentResourcesFetchRequest);
+
+ resourceServiceBlockingStub.fetchResourceMetadata(addResourceMetadataRequest);
+
+
+
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ }
}
@@ -210,11 +248,11 @@ public class Client {
CustosClientProvider custosClientProvider = new CustosClientProvider.Builder().setServerHost("custos.scigap.org")
.setServerPort(31499)
- .setClientId("custos-cmcdclbywlxmc2ktzv0d-10000702")
- .setClientSec("1hBjD0poRNTdxwLzV0baY5mgUlogPalH9jCFz9ZG").build();
+ .setClientId("custos-whedmgamitu357p4wuke-10002708")
+ .setClientSec("mrMdl86Ia1H94cikW7CvHoh7L0ASNXQVt2aRzSIj").build();
IdentityManagementClient identityManagementClient = custosClientProvider.getIdentityManagementClient();
- Struct struct = identityManagementClient.getToken(null, null, "testuser", "testuser1234", null, "password");
+ Struct struct = identityManagementClient.getToken(null, null, "isjarana@iu.edu", "IJR@circ@1", null, "password");
return struct.getFieldsMap().get("access_token").getStringValue();
} catch (Exception ex) {
ex.printStackTrace();
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
index 9c746fd..7316168 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
@@ -17,20 +17,23 @@
package org.apache.airavata.drms.api.handlers;
import com.google.protobuf.Empty;
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.Struct;
+import com.google.protobuf.util.JsonFormat;
import io.grpc.Status;
import io.grpc.stub.StreamObserver;
import org.apache.airavata.datalake.drms.AuthenticatedUser;
import org.apache.airavata.datalake.drms.resource.GenericResource;
import org.apache.airavata.datalake.drms.storage.*;
import org.apache.airavata.drms.api.utils.CustosUtils;
+import org.apache.airavata.drms.api.utils.Utils;
import org.apache.airavata.drms.core.Neo4JConnector;
-import org.apache.airavata.drms.core.constants.ResourceConstants;
import org.apache.airavata.drms.core.constants.StoragePreferenceConstants;
import org.apache.airavata.drms.core.deserializer.GenericResourceDeserializer;
-import org.apache.airavata.drms.core.deserializer.MetadataDeserializer;
import org.apache.airavata.drms.core.serializer.GenericResourceSerializer;
import org.apache.custos.clients.CustosClientProvider;
import org.apache.custos.sharing.service.Entity;
+import org.json.JSONObject;
import org.lognet.springboot.grpc.GRpcService;
import org.neo4j.driver.Record;
import org.slf4j.Logger;
@@ -39,6 +42,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
@GRpcService
public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceImplBase {
@@ -51,6 +55,9 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
@Autowired
private CustosClientProvider custosClientProvider;
+
+ private static final String DATA_LAKE_JSON_IDENTIFIER = "DATA_LAKE_METADATA_NODE_JSON_IDENTIFIER";
+
@Override
public void fetchResource(ResourceFetchRequest request, StreamObserver<ResourceFetchResponse> responseObserver) {
@@ -59,13 +66,18 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
String resourceId = request.getResourceId();
String type = request.getType();
-
+ if (type == null || type.isEmpty()) {
+ type = "";
+ } else {
+ type = ":" + type;
+ }
Map<String, Object> userProps = new HashMap<>();
userProps.put("username", callUser.getUsername());
userProps.put("tenantId", callUser.getTenantId());
userProps.put("entityId", resourceId);
- String query = " MATCH (u:User), (r:" + type + ") where u.username = $username AND u.tenantId = $tenantId AND " +
+
+ String query = " MATCH (u:User), (r" + type + ") where u.username = $username AND u.tenantId = $tenantId AND " +
" r.entityId = $entityId AND r.tenantId = $tenantId" +
" OPTIONAL MATCH (cg:Group)-[:CHILD_OF*]->(g:Group)<-[:MEMBER_OF]-(u)" +
" return case when exists((u)<-[:SHARED_WITH]-(r)) OR exists((g)<-[:SHARED_WITH]-(r)) OR " +
@@ -88,11 +100,13 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
responseObserver.onError(Status.INTERNAL.withDescription(msg).asRuntimeException());
}
- } catch (Exception ex) {
+ } catch (
+ Exception ex) {
logger.error("Error occurred while fetching child resource {}", request.getResourceId());
String msg = "Error occurred while creating resource with id" + request.getResourceId();
responseObserver.onError(Status.INTERNAL.withDescription(msg).asRuntimeException());
}
+
}
@Override
@@ -119,7 +133,8 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
String entityId = request.getResource().getResourceId();
Map<String, Object> serializedMap = GenericResourceSerializer.serializeToMap(request.getResource());
- Optional<Entity> exEntity = CustosUtils.mergeResourceEntity(custosClientProvider, callUser.getTenantId(), storagePreferenceId, type, entityId,
+ Optional<Entity> exEntity = CustosUtils.mergeResourceEntity(custosClientProvider, callUser.getTenantId(),
+ storagePreferenceId, type, entityId,
request.getResource().getResourceName(), request.getResource().getResourceName(),
callUser.getUsername());
@@ -130,6 +145,9 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
serializedMap.put("tenantId", callUser.getTenantId());
serializedMap.put("entityId", exEntity.get().getId());
serializedMap.put("entityType", exEntity.get().getType());
+ serializedMap.put("lastUpdatedTime", exEntity.get().getCreatedAt());
+ serializedMap.put("owner", exEntity.get().getOwnerId());
+
HashMap<String, Object> hashMap = new HashMap<>();
@@ -146,7 +164,29 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
String msg = "Error occurred while creating resource entity in Custos with id"
+ request.getResource().getResourceId();
responseObserver.onError(Status.INTERNAL.withDescription(msg).asRuntimeException());
+ return;
}
+ Map<String, Object> exProps = new HashMap<>();
+ exProps.put("username", callUser.getUsername());
+ exProps.put("tenantId", callUser.getTenantId());
+ exProps.put("entityId", exEntity.get().getId());
+
+ String query = " MATCH (u:User), (r:" + type + ") where u.username = $username AND u.tenantId = $tenantId AND " +
+ " r.entityId = $entityId AND r.tenantId = $tenantId" +
+ " OPTIONAL MATCH (cg:Group)-[:CHILD_OF*]->(g:Group)<-[:MEMBER_OF]-(u)" +
+ " return case when exists((u)<-[:SHARED_WITH]-(r)) OR exists((g)<-[:SHARED_WITH]-(r)) OR " +
+ "exists((cg)<-[:SHARED_WITH]-(r)) then r else NULL end as value";
+
+
+ List<Record> records = this.neo4JConnector.searchNodes(exProps, query);
+
+ List<GenericResource> genericResourceList = GenericResourceDeserializer.deserializeList(records);
+ ResourceCreateResponse response = ResourceCreateResponse
+ .newBuilder()
+ .setResource(genericResourceList.get(0))
+ .build();
+ responseObserver.onNext(response);
+ responseObserver.onCompleted();
} catch (Exception ex) {
@@ -154,6 +194,7 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
String msg = "Error occurred while creating resource" + ex.getMessage();
responseObserver.onError(Status.INTERNAL.withDescription(msg).asRuntimeException());
}
+
}
@@ -166,13 +207,18 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
String resourceId = request.getResourceId();
String type = request.getType();
int depth = request.getDepth();
+ if (type == null || type.isEmpty()) {
+ type = "";
+ } else {
+ type = ":" + type;
+ }
Map<String, Object> userProps = new HashMap<>();
userProps.put("username", callUser.getUsername());
userProps.put("tenantId", callUser.getTenantId());
userProps.put("entityId", resourceId);
- String query = " MATCH (u:User), (r:" + type + ") where u.username = $username AND u.tenantId = $tenantId AND " +
+ String query = " MATCH (u:User), (r" + type + ") where u.username = $username AND u.tenantId = $tenantId AND " +
" r.entityId = $entityId AND r.tenantId = $tenantId" +
" OPTIONAL MATCH (cg:Group)-[:CHILD_OF*]->(g:Group)<-[:MEMBER_OF]-(u)" +
" OPTIONAL MATCH (u)<-[:SHARED_WITH]-(r)<-[:CHILD_OF*]-(cr)" +
@@ -181,7 +227,7 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
" return distinct cr, chgr, chcgr";
if (depth == 1) {
- query = " MATCH (u:User), (r:" + type + ") where u.username = $username AND u.tenantId = $tenantId AND " +
+ query = " MATCH (u:User), (r" + type + ") where u.username = $username AND u.tenantId = $tenantId AND " +
" r.entityId = $entityId AND r.tenantId = $tenantId" +
" OPTIONAL MATCH (cg:Group)-[:CHILD_OF*]->(g:Group)<-[:MEMBER_OF]-(u)" +
" OPTIONAL MATCH (u)<-[:SHARED_WITH]-(r)<-[:CHILD_OF]-(cr)" +
@@ -213,7 +259,8 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
}
@Override
- public void updateResource(ResourceUpdateRequest request, StreamObserver<ResourceUpdateResponse> responseObserver) {
+ public void updateResource(ResourceUpdateRequest
+ request, StreamObserver<ResourceUpdateResponse> responseObserver) {
super.updateResource(request, responseObserver);
}
@@ -223,17 +270,69 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
}
@Override
- public void searchResource(ResourceSearchRequest request, StreamObserver<ResourceSearchResponse> responseObserver) {
+ public void searchResource(ResourceSearchRequest
+ request, StreamObserver<ResourceSearchResponse> responseObserver) {
try {
AuthenticatedUser callUser = request.getAuthToken().getAuthenticatedUser();
List<ResourceSearchQuery> resourceSearchQueries = request.getQueriesList();
- ResourceSearchQuery searchQuery = resourceSearchQueries.get(0);
int depth = request.getDepth();
+ String value = request.getType();
+ if (value == null || value.isEmpty()) {
+ logger.error("Errored while searching generic resources");
+ responseObserver
+ .onError(Status.INTERNAL.withDescription("Errored while searching generic resources ")
+ .asRuntimeException());
+ return;
+ }
- if (searchQuery.getField().equals("type")) {
- String value = searchQuery.getValue();
+ Optional<String> metadataSearchQueryOP = Utils.getMetadataSearchQuery(resourceSearchQueries, value);
+ Optional<String> ownPropertySearchQuery = Utils.getPropertySearchQuery(resourceSearchQueries, value);
+ if (metadataSearchQueryOP.isPresent()) {
+ String query = metadataSearchQueryOP.get();
+
+ List<Record> records = this.neo4JConnector.searchNodes(query);
+ List<GenericResource> genericResourceList = GenericResourceDeserializer.deserializeList(records);
+
+ List<GenericResource> allowedResourceList = new ArrayList<>();
+
+ genericResourceList.forEach(res -> {
+ try {
+ if (hasAccessForResource(callUser.getUsername(), callUser.getTenantId(), res.getResourceId(), value)) {
+ allowedResourceList.add(res);
+ }
+ } catch (Exception exception) {
+ logger.error("Errored while searching generic resources");
+ responseObserver
+ .onError(Status.INTERNAL.withDescription("Errored while searching generic resources ")
+ .asRuntimeException());
+ return;
+ }
+ });
+
+ if (ownPropertySearchQuery.isPresent()) {
+ List<Record> ownPropertySearchRecords = this.neo4JConnector.searchNodes(ownPropertySearchQuery.get());
+ List<GenericResource> genericResources = GenericResourceDeserializer.deserializeList(ownPropertySearchRecords);
+ genericResources.forEach(res -> {
+ try {
+ if (hasAccessForResource(callUser.getUsername(), callUser.getTenantId(), res.getResourceId(), value)) {
+ allowedResourceList.add(res);
+ }
+ } catch (Exception exception) {
+ logger.error("Errored while searching generic resources");
+ responseObserver
+ .onError(Status.INTERNAL.withDescription("Errored while searching generic resources ")
+ .asRuntimeException());
+ return;
+ }
+ });
+
+ }
+ ResourceSearchResponse.Builder builder = ResourceSearchResponse.newBuilder();
+ builder.addAllResources(allowedResourceList);
+ responseObserver.onNext(builder.build());
+ } else {
Map<String, Object> userProps = new HashMap<>();
userProps.put("username", callUser.getUsername());
userProps.put("tenantId", callUser.getTenantId());
@@ -256,20 +355,18 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
List<Record> records = this.neo4JConnector.searchNodes(userProps, query);
-
List<GenericResource> genericResourceList = GenericResourceDeserializer.deserializeList(records);
ResourceSearchResponse.Builder builder = ResourceSearchResponse.newBuilder();
builder.addAllResources(genericResourceList);
responseObserver.onNext(builder.build());
- responseObserver.onCompleted();
}
+ responseObserver.onCompleted();
} catch (Exception e) {
logger.error("Errored while searching generic resources; Message: {}", e.getMessage(), e);
responseObserver.onError(Status.INTERNAL.withDescription("Errored while searching generic resources "
+ e.getMessage()).asRuntimeException());
}
-
}
@@ -381,7 +478,8 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
@Override
- public void fetchParentResources(ParentResourcesFetchRequest request, StreamObserver<ParentResourcesFetchResponse> responseObserver) {
+ public void fetchParentResources(ParentResourcesFetchRequest
+ request, StreamObserver<ParentResourcesFetchResponse> responseObserver) {
try {
AuthenticatedUser callUser = request.getAuthToken().getAuthenticatedUser();
String resourseId = request.getResourceId();
@@ -390,12 +488,17 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
if (depth == 0) {
depth = 1;
}
+ if (type == null || type.isEmpty()) {
+ type = "";
+ } else {
+ type = ":" + type;
+ }
if (hasAccessForResource(callUser.getUsername(), callUser.getTenantId(), resourseId, type)) {
Map<String, Object> userProps = new HashMap<>();
userProps.put("tenantId", callUser.getTenantId());
userProps.put("entityId", resourseId);
- String query = "MATCH (r:" + type + ") where r.entityId = $entityId AND r.tenantId = $tenantId" +
+ String query = "MATCH (r" + type + ") where r.entityId = $entityId AND r.tenantId = $tenantId" +
" MATCH (r)-[ch:CHILD_OF*1.." + depth + "]->(m) return distinct m";
List<Record> records = this.neo4JConnector.searchNodes(userProps, query);
if (!records.isEmpty()) {
@@ -426,43 +529,98 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
@Override
public void addResourceMetadata(AddResourceMetadataRequest request, StreamObserver<Empty> responseObserver) {
- AuthenticatedUser callUser = request.getAuthToken().getAuthenticatedUser();
- this.neo4JConnector.createMetadataNode(ResourceConstants.RESOURCE_LABEL, "resourceId",
- request.getResourceId(), callUser.getUsername(),
- request.getMetadata().getKey(), request.getMetadata().getValue());
- responseObserver.onNext(Empty.getDefaultInstance());
- responseObserver.onCompleted();
+ try {
+ AuthenticatedUser callUser = request.getAuthToken().getAuthenticatedUser();
+
+ String parentResourceId = request.getResourceId();
+ String type = request.getType();
+
+ Struct struct = request.getMetadata();
+ String message = JsonFormat.printer().print(struct);
+ JSONObject json = new JSONObject(message);
+
+ Map<String, Object> map = json.toMap();
+
+ mergeProperties(parentResourceId, type, callUser.getTenantId(), parentResourceId, map);
+
+ Map<String, Object> parameters = new HashMap<>();
+ Map<String, Object> properties = new HashMap<>();
+ properties.put("metadata", message);
+ parameters.put("props", properties);
+ parameters.put("parentResourceId", parentResourceId);
+ parameters.put("resourceId", UUID.randomUUID().toString());
+ parameters.put("tenantId", callUser.getTenantId());
+
+ String query = " MATCH (r:" + type + ") where r.entityId= $parentResourceId AND r.tenantId= $tenantId " +
+ " MERGE (cr:FULL_METADATA_NODE {entityId: $resourceId,tenantId: $tenantId})" +
+ " MERGE (r)-[:HAS_FULL_METADATA]->(cr) SET cr += $props return cr";
+ this.neo4JConnector.runTransactionalQuery(parameters, query);
+
+ responseObserver.onNext(Empty.getDefaultInstance());
+ responseObserver.onCompleted();
+ } catch (Exception ex) {
+ String msg = " Error occurred while adding resource metadata " + ex.getMessage();
+ logger.error(" Error occurred while adding resource metadata: Messages {} ", ex.getMessage(), ex);
+ responseObserver.onError(Status.INTERNAL.withDescription(msg).asRuntimeException());
+ }
}
@Override
- public void fetchResourceMetadata(FetchResourceMetadataRequest request, StreamObserver<FetchResourceMetadataResponse> responseObserver) {
- AuthenticatedUser callUser = request.getAuthToken().getAuthenticatedUser();
- List<Record> records = neo4JConnector.searchNodes("match (u:User)-[MEMBER_OF]->(g:Group)<-[SHARED_WITH]-(res:Resource)-[r:HAS_METADATA]->(m:Metadata) " +
- "where u.userId ='" + callUser.getUsername() + "' and res.resourceId = '" + request.getResourceId() + "' return distinct m");
+ public void fetchResourceMetadata(FetchResourceMetadataRequest
+ request, StreamObserver<FetchResourceMetadataResponse> responseObserver) {
try {
- List<MetadataNode> metadataNodes = MetadataDeserializer.deserializeList(records);
- if (metadataNodes.size() == 1) {
- responseObserver.onNext(FetchResourceMetadataResponse.newBuilder().setMetadataNode(metadataNodes.get(0)).build());
+ AuthenticatedUser callUser = request.getAuthToken().getAuthenticatedUser();
+
+ String resourceId = request.getResourceId();
+ String type = request.getType();
+ if (type == null || type.isEmpty()) {
+ type = "";
+ } else {
+ type = ":" + type;
+ }
+
+ if (hasAccessForResource(callUser.getUsername(), callUser.getTenantId(), resourceId, type)) {
+ Optional<List<String>> metadataArray = readMetadata(resourceId, type, callUser.getTenantId());
+ FetchResourceMetadataResponse.Builder builder = FetchResourceMetadataResponse.newBuilder();
+ if (metadataArray.isPresent()) {
+ metadataArray.get().forEach(val -> {
+ try {
+ Struct.Builder structBuilder = Struct.newBuilder();
+ JsonFormat.parser().merge(val, structBuilder);
+ builder.addMetadata(structBuilder.build());
+ } catch (InvalidProtocolBufferException e) {
+ String msg = " Error occurred while fetching resource metadata " + e.getMessage();
+ logger.error(" Error occurred while fetching resource metadata: Messages {} ", e.getMessage(), e);
+ responseObserver.onError(Status.INTERNAL.withDescription(msg).asRuntimeException());
+ return;
+ }
+ });
+ }
+ responseObserver.onNext(builder.build());
responseObserver.onCompleted();
} else {
- logger.error("No metadata entry for resource {}", request.getResourceId());
- responseObserver.onError(new Exception("No metadata entry for resource " + request.getResourceId()));
+ String msg = " Cannot find accessible resource ";
+ logger.error(" Cannot find accessible resource");
+ responseObserver.onError(Status.PERMISSION_DENIED.withDescription(msg).asRuntimeException());
+
}
- } catch (Exception e) {
- logger.error("Errored while fetching metadata for resource with id {}", request.getResourceId(), e);
- responseObserver.onError(new Exception("Errored while fetching metadata for resource with id "
- + request.getResourceId() + ". Msg " + e.getMessage()));
+ } catch (Exception ex) {
+ String msg = " Error occurred while fetching resource metadata " + ex.getMessage();
+ logger.error(" Error occurred while fetching resource metadata: Messages {} ", ex.getMessage(), ex);
+ responseObserver.onError(Status.INTERNAL.withDescription(msg).asRuntimeException());
}
+
}
- private boolean hasAccessForResource(String username, String tenantId, String resourceId, String type) throws Exception {
+ private boolean hasAccessForResource(String username, String tenantId, String resourceId, String type) throws
+ Exception {
Map<String, Object> userProps = new HashMap<>();
userProps.put("username", username);
userProps.put("tenantId", tenantId);
userProps.put("entityId", resourceId);
- String query = " MATCH (u:User), (r:" + type + ") where u.username = $username AND u.tenantId = $tenantId AND " +
+ String query = " MATCH (u:User), (r) where u.username = $username AND u.tenantId = $tenantId AND " +
" r.entityId = $entityId AND r.tenantId = $tenantId" +
" OPTIONAL MATCH (cg:Group)-[:CHILD_OF*]->(g:Group)<-[:MEMBER_OF]-(u)" +
" return case when exists((u)<-[:SHARED_WITH]-(r)) OR exists((g)<-[:SHARED_WITH]-(r)) OR " +
@@ -477,4 +635,107 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
return true;
}
+
+
+ private void mergeProperties(String parentResourceId, String parentType, String tenantId, String resourceId,
+ Map<String, Object> values) {
+ for (String key : values.keySet()) {
+ Map<String, Object> parameters = new HashMap<>();
+ parameters.put("parentResourceId", parentResourceId);
+ parameters.put("tenantId", tenantId);
+ if (parentResourceId.equals(resourceId) && !(values.get(key) instanceof Map) && !(values.get(key) instanceof List)) {
+ Map<String, String> props = new HashMap<>();
+ parameters.put("resourceId", resourceId);
+ String value = String.valueOf(values.get(key));
+ props.put(key, value);
+ parameters.put("props", props);
+ String query = " MATCH (r:" + parentType + ") where r.entityId= $parentResourceId " +
+ "AND r.tenantId= $tenantId" +
+ " SET r += $props return r";
+ this.neo4JConnector.runTransactionalQuery(parameters, query);
+
+ } else if (values.get(key) instanceof Map) {
+ String newResourceId = UUID.randomUUID().toString();
+ parameters.put("resourceId", newResourceId);
+ String type = "METADATA_NODE";
+ Map<String, Object> hashMap = (Map<String, Object>) values.get(key);
+ Map<String, Object> newHashMap = new HashMap<>();
+ newHashMap.put(DATA_LAKE_JSON_IDENTIFIER, key);
+ newHashMap.putAll(hashMap);
+ String query = " MATCH (r:" + parentType + ") where r.entityId= $parentResourceId AND r.tenantId= $tenantId " +
+ " MERGE (cr:" + type + " {entityId: $resourceId,tenantId: $tenantId})" +
+ " MERGE (r)-[:HAS_METADATA]->(cr) return cr";
+ this.neo4JConnector.runTransactionalQuery(parameters, query);
+
+ mergeProperties(newResourceId, type, tenantId, newResourceId, newHashMap);
+
+ } else if (values.get(key) instanceof List) {
+ ArrayList arrayList = (ArrayList) values.get(key);
+ Map<String, Object[]> props = new HashMap<>();
+ parameters.put("resourceId", resourceId);
+ props.put(key, arrayList.toArray());
+ parameters.put("props", props);
+ String query = " MATCH (r:" + parentType + ") where r.entityId= $parentResourceId " +
+ "AND r.tenantId= $tenantId" +
+ " SET r += $props return r";
+ this.neo4JConnector.runTransactionalQuery(parameters, query);
+ }
+ }
+ }
+
+ private Map<String, Object> readProperties(String resourceId, String type, String tenantId, Map<String, Object> map) throws Exception {
+
+ Map<String, Object> parameters = new HashMap<>();
+ parameters.put("entityId", resourceId);
+ parameters.put("tenantId", tenantId);
+ String query = " Match (r" + type + ") where r.entityId=$entityId and r.tenantId=$tenantId " +
+ " Match (r)-[:HAS_METADATA*]->(m) return m";
+ List<Record> records = this.neo4JConnector.searchNodes(parameters, query);
+
+ List<GenericResource> genericResourceList = GenericResourceDeserializer.deserializeList(records);
+ if (!genericResourceList.isEmpty()) {
+ genericResourceList.forEach(res -> {
+ String resId = res.getResourceId();
+ Map<String, String> propertiesMap = res.getPropertiesMap();
+ String value = propertiesMap.get(DATA_LAKE_JSON_IDENTIFIER);
+ ((Map<String, Object>) map.computeIfAbsent(value, val -> new HashMap<String, Object>())).putAll(propertiesMap);
+ ((Map<String, Object>) map.get(value)).remove(DATA_LAKE_JSON_IDENTIFIER);
+// propertiesMap.forEach((key, val) -> {
+// if (!key.equals(DATA_LAKE_JSON_IDENTIFIER)) {
+// ((Map<String, Object>) map.get(value)).put(key, val);
+//
+// }
+// });
+
+ try {
+ readProperties(resId, ":METADATA_NODE", tenantId, (Map<String, Object>) map.get(value));
+ } catch (Exception exception) {
+ exception.printStackTrace();
+ }
+
+ });
+
+ }
+ return map;
+ }
+
+
+ private Optional<List<String>> readMetadata(String resourceId, String type, String tenantId) throws Exception {
+ Map<String, Object> parameters = new HashMap<>();
+ parameters.put("entityId", resourceId);
+ parameters.put("tenantId", tenantId);
+ String query = " Match (r" + type + ") where r.entityId=$entityId and r.tenantId=$tenantId " +
+ " Match (r)-[:HAS_FULL_METADATA]->(m) return m";
+ List<Record> records = this.neo4JConnector.searchNodes(parameters, query);
+
+ List<GenericResource> genericResourceList = GenericResourceDeserializer.deserializeList(records);
+ if (!genericResourceList.isEmpty()) {
+ return Optional.ofNullable(genericResourceList.stream().map(val -> {
+ Map<String, String> proprties = val.getPropertiesMap();
+ return proprties.get("metadata");
+ }).collect(Collectors.toList()));
+ }
+ return Optional.empty();
+ }
+
}
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/utils/Utils.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/utils/Utils.java
index b373ce4..5d4725f 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/utils/Utils.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/utils/Utils.java
@@ -1,14 +1,17 @@
package org.apache.airavata.drms.api.utils;
import io.grpc.Context;
+import org.apache.airavata.datalake.drms.storage.ResourceSearchQuery;
+import java.util.List;
+import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
public class Utils {
private static ConcurrentHashMap<String, Context.Key<Object>> keyMap = new ConcurrentHashMap<String, Context.Key<Object>>();
- public static final String CONTEXT_HOLDER = "CONTEXT_HOLDER";
+ public static final String CONTEXT_HOLDER = "CONTEXT_HOLDER";
public static Context.Key<Object> getUserContextKey() {
if (keyMap.containsKey("AUTHORIZED_USER")) {
@@ -19,7 +22,40 @@ public class Utils {
}
+ public static Optional<String> getMetadataSearchQuery(List<ResourceSearchQuery> resourceSearchQueries, String type) {
+ if (!resourceSearchQueries.isEmpty()) {
+ String preRegex = "'(?i).*";
+ String postRegex = ".*'";
+ String query = " MATCH (r:" + type + ")-[:HAS_METADATA*]->(m) WHERE ";
+ //TODO: works only for one property
+ for (ResourceSearchQuery qry : resourceSearchQueries) {
+ if (qry.getField().contains(" ")) {
+ query = " MATCH (r:" + type + ")-[:HAS_METADATA*]->(m:METADATA_NODE{`" + qry.getField() + "`: '" + qry.getValue() + "'})" +
+ " Return r ";
+ return Optional.ofNullable(query);
+ } else {
+ String finalSearchStr = preRegex + qry.getValue() + postRegex;
+ query = query + " m." + qry.getField() + "=~ " + finalSearchStr + " AND ";
+ }
+ }
+ query = query.substring(0, query.length() - 5);
+ query = query + " RETURN r";
+ return Optional.ofNullable(query);
+ }
+ return Optional.empty();
+ }
+
+ public static Optional<String> getPropertySearchQuery(List<ResourceSearchQuery> resourceSearchQueries, String type) {
+ if (!resourceSearchQueries.isEmpty()) {
+ for (ResourceSearchQuery qry : resourceSearchQueries) {
+ String query = " MATCH (r:" + type + "{`" + qry.getField() + "`: '" + qry.getValue() + "'})" +
+ " Return r ";
+ return Optional.ofNullable(query);
+ }
+ }
+ return Optional.empty();
+ }
}
diff --git a/data-resource-management-service/drms-api/src/main/resources/sample.json b/data-resource-management-service/drms-api/src/main/resources/sample.json
new file mode 100644
index 0000000..0b69950
--- /dev/null
+++ b/data-resource-management-service/drms-api/src/main/resources/sample.json
@@ -0,0 +1,609 @@
+{
+ "resourcePath":"/N/u/mcmullen/Carbonate/",
+ "fileinfo":{
+ "comp":1,
+ "offset":598726,
+ "ranges":"null",
+ "format":1,
+ "nImages":1,
+ "dir":"",
+ "type":"float",
+ "samples":1,
+ "url":"",
+ "gap":0,
+ "name":"testForClass_0006.dm4",
+ "width":4096,
+ "byteOrder":"little",
+ "lutSize":0,
+ "whiteIsZero":"f",
+ "height":4096
+ },
+ "RPIDhandle":"4263537/4000",
+ "dm4":{
+ "root":{
+ "InImageMode":"true",
+ "MinVersionList":{
+ "0":{
+ "RequiredVersion":50659328
+ }
+ },
+ "WindowPosition":[
+ 32,
+ 3,
+ 1622,
+ 1593
+ ],
+ "Thumbnails":{
+ "0":{
+ "SourceSize_Pixels":[
+ 1590,
+ 1590
+ ],
+ "ImageIndex":0
+ }
+ },
+ "DocumentObjectList":{
+ "0":{
+ "UniqueID":8,
+ "IsMoveable":"true",
+ "FillMode":1,
+ "HasBackground":"false",
+ "BackgroundMode":2,
+ "IsSelectable":"true",
+ "Rectangle":[
+ 0,
+ 0,
+ 1590,
+ 1590
+ ],
+ "IsTranslatable":"true",
+ "RestrictionStyle":0,
+ "AnnotationType":20,
+ "AnnotationGroupList":{
+ "0":{
+ "UniqueID":9,
+ "IsMoveable":"true",
+ "FillMode":2,
+ "TextOffsetH":1,
+ "HasBackground":"false",
+ "BackgroundMode":2,
+ "IsSelectable":"true",
+ "Rectangle":[
+ 3616,
+ 256,
+ 3840,
+ 1408
+ ],
+ "IsTranslatable":"true",
+ "RestrictionStyle":0,
+ "AnnotationType":31,
+ "Font":{
+ "FamilyName":"Lucida Console",
+ "Size":80,
+ "Attributes":7
+ },
+ "IsResizable":"true",
+ "TextOffsetV":1,
+ "ForegroundColor":[
+ -1,
+ -1,
+ -1
+ ],
+ "IsTransferrable":"true",
+ "IsVisible":"true",
+ "TextWidth":741.917,
+ "BackgroundColor":[
+ 16448,
+ 0,
+ 16448
+ ]
+ }
+ },
+ "IsResizable":"true",
+ "ImageDisplayType":1,
+ "ImageDisplayInfo":{
+ "ComplexMode":4,
+ "Brightness":0.5,
+ "RangeAdjust":1,
+ "SparseSurvey_UseNumberPixels":"true",
+ "SparseSurvey_GridSize":16,
+ "HighLimit":91607.03,
+ "EstimatedMaxTrimPercentage":0.001,
+ "Gamma":0.5,
+ "CaptionSize":12,
+ "MinimumContrast":0,
+ "DiffractionMode":"false",
+ "EstimatedMin":91607.03,
+ "SparseSurvey_NumberPixels":32,
+ "CLUT":{
+ "Size":1536,
+ "Offset":2383
+ },
+ "BrightColor":[
+ -1,
+ -1,
+ -1
+ ],
+ "ContrastMode":1,
+ "SurveyTechique":2,
+ "DoAutoSurvey":"true",
+ "LowLimitContrastDeltaTriggerPercentage":0,
+ "EstimatedMinTrimPercentage":0.001,
+ "HiLimitContrastDeltaTriggerPercentage":0,
+ "MainSliceId":{
+ "0":0
+ },
+ "ComplexRange":1000,
+ "CaptionOn":"false",
+ "CLUTName":"Greyscale",
+ "EstimatedMax":76706.74,
+ "IsInverted":"false",
+ "Contrast":0.5,
+ "IsIgnoreGirdBar":"false",
+ "LowLimit":76706.74
+ },
+ "ImageSource":0,
+ "ForegroundColor":[
+ -1,
+ 0,
+ 0
+ ],
+ "IsTransferrable":"true",
+ "IsVisible":"true",
+ "BackgroundColor":[
+ -1,
+ -1,
+ -1
+ ]
+ }
+ },
+ "HasWindowPosition":"true",
+ "LayoutType":"Camera",
+ "ImageList":{
+ "0":{
+ "UniqueID":{
+ "0":1986283998,
+ "1":1970537810,
+ "2":2010261246,
+ "3":851384688
+ },
+ "ImageData":{
+ "DataType":23,
+ "Dimensions":{
+ "0":384,
+ "1":384
+ },
+ "Data":{
+ "Size":589824,
+ "Offset":7604
+ },
+ "PixelDepth":4,
+ "Calibrations":{
+ "Brightness":{
+ "Origin":0,
+ "Scale":1
+ },
+ "DisplayCalibratedUnits":"true",
+ "Dimension":{
+ "0":{
+ "Origin":0,
+ "Scale":1
+ },
+ "1":{
+ "Origin":0,
+ "Scale":1
+ }
+ }
+ }
+ },
+ "ImageTags":{
+ "GMS Version":{
+ "Created":"3.21.1374.0"
+ }
+ },
+ "Name":"Image Of testForClass_0006"
+ },
+ "1":{
+ "UniqueID":{
+ "0":649227372,
+ "1":2077039872,
+ "2":1831281967,
+ "3":1840526002
+ },
+ "ImageData":{
+ "DataType":2,
+ "Dimensions":{
+ "0":4096,
+ "1":4096
+ },
+ "Data":{
+ "Size":67108864,
+ "Offset":598726
+ },
+ "PixelDepth":4,
+ "Calibrations":{
+ "Brightness":{
+ "Origin":0,
+ "Scale":1
+ },
+ "DisplayCalibratedUnits":"true",
+ "Dimension":{
+ "0":{
+ "Origin":0,
+ "Scale":0.45125848,
+ "Units":"nm"
+ },
+ "1":{
+ "Origin":0,
+ "Scale":0.45125848,
+ "Units":"nm"
+ }
+ }
+ }
+ },
+ "ImageTags":{
+ "Session Info":{
+ "Microscope":"1400plus",
+ "Items":{
+ "0":{
+ "Tag path":"Session Info:Specimen",
+ "Data Type":20,
+ "Precision":0,
+ "Label":"Specimen"
+ },
+ "1":{
+ "Tag path":"Session Info:Operator",
+ "Data Type":20,
+ "Precision":0,
+ "Label":"Operator"
+ },
+ "2":{
+ "Tag path":"Session Info:Microscope",
+ "Data Type":20,
+ "Precision":0,
+ "Label":"Microscope",
+ "Value":"1400plus"
+ }
+ }
+ },
+ "GMS Version":{
+ "Saved":"3.21.1374.0",
+ "Created":"3.21.1374.0"
+ },
+ "DataBar":{
+ "Acquisition Time (OS)":1.31661214205959776E17,
+ "Binning":1,
+ "Exposure Number":10720,
+ "Acquisition Time":"11:57:00 AM",
+ "Collection Number":6,
+ "Exposure Time (s)":1,
+ "Acquisition Date":"3/21/2018",
+ "Device Name":"OneView",
+ "Signal Name":"OneView"
+ },
+ "Microscope Info":{
+ "Illumination Mode":"TEM",
+ "Emission Current (�A)":0,
+ "Stage Position":{
+ "Stage Beta":0,
+ "Stage Z":-17.987630859375,
+ "Stage Alpha":-0.08524005115032196,
+ "Stage X":98.873078125,
+ "Stage Y":225.2194375
+ },
+ "Formatted Indicated Mag":"25kx",
+ "STEM Camera Length":0,
+ "Formatted Actual Mag":"33.2kx",
+ "HT Extrapolated":"false",
+ "Cs(mm)":0,
+ "Magnification Interpolated":"true",
+ "Name":"JEOL COM",
+ "Operation Mode":"IMAGING",
+ "Imaging Mode":"MAG1",
+ "Probe Size (nm)":0,
+ "Actual Magnification":33240.371968932224,
+ "Indicated Magnification":25000,
+ "Voltage":120000,
+ "Formatted Voltage":"120kV",
+ "Operation Mode Type":1,
+ "Probe Current (nA)":0
+ },
+ "Acquisition":{
+ "Frame":{
+ "Area":{
+ "Transform":{
+ "Transform List":{
+ "0":{
+ "Binning":[
+ 1,
+ 1
+ ],
+ "Class Name":"cm_acquisitiontransform",
+ "Sub Area Adjust":[
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ "Transpose":{
+ "Diagonal Flip":"false",
+ "Vertical Flip":"false",
+ "Horizontal Flip":"true"
+ }
+ }
+ },
+ "Class Name":"cm_acquisitiontransform_list"
+ }
+ },
+ "CCD":{
+ "Pixel Size (um)":[
+ 15,
+ 15
+ ]
+ },
+ "Intensity":{
+ "Transform":{
+ "Transform List":{
+ "0":{
+ "Class Name":"cm_valuetransform_affine",
+ "Scale":1,
+ "Offset":0
+ },
+ "1":{
+ "Class Name":"cm_valuetransform_adc",
+ "ADC Max":65535,
+ "ADC Min":0
+ }
+ },
+ "Class Name":"cm_valuetransform_list"
+ }
+ },
+ "Reference Images":{
+ "Dark":{
+ "Mean (counts)":130.79538156255063,
+ "Standard Deviation (counts)":68.57993297466311
+ }
+ }
+ },
+ "Parameters":{
+ "High Level":{
+ "Acquisition Buffer Size":0,
+ "Number Of Frame Shutters":1,
+ "Secondary Shutter Post Exposure Compensation (s)":0,
+ "Antiblooming":"false",
+ "Continuous Readout":"false",
+ "Class Name":"cm_camera_highlevelparameters",
+ "Shutter":{
+ "Primary Shutter States Mask":0,
+ "Secondary Shutter States Mask":0,
+ "Primary Shutter States":0,
+ "Shutter Exposure":"false",
+ "Secondary Shutter States":0,
+ "Shutter Index":0
+ },
+ "Capture Duration":280.00638168800003,
+ "Quality Level":0,
+ "Processing":"Gain Normalized",
+ "Binning":[
+ 1,
+ 1
+ ],
+ "Corrections Mask":817,
+ "Frame Exposure":0.04,
+ "CCD Read Ports":1,
+ "Async Readout":0,
+ "Stack Format":1,
+ "Lookback Duration":0,
+ "Shutter Pre Exposure Compensation (s)":0,
+ "CCD Read Area":[
+ 0,
+ 0,
+ 4096,
+ 4096
+ ],
+ "Read Mode":0,
+ "Read Frame Style":0,
+ "Exposure (s)":1,
+ "Shutter Post Exposure Compensation (s)":0,
+ "Complex Exposure":{
+ "Sub-Exposure Count":0,
+ "Frame Count":0
+ },
+ "Acquire Stack":"true",
+ "Exposure Live Time":"Infinity",
+ "Async Processing":"false",
+ "HardwareCorrections":0,
+ "Choose Number Of Frame Shutters Automatically":"true",
+ "Transform":{
+ "Diagonal Flip":"false",
+ "Vertical Flip":"false",
+ "Horizontal Flip":"false"
+ },
+ "Exposure Priority":1,
+ "Auto Save":"false",
+ "Corrections":801,
+ "Secondary Shutter Pre Exposure Compensation (s)":0
+ },
+ "Detector":{
+ "top":0,
+ "left":0,
+ "vbin":1,
+ "continuous":"false",
+ "width":4096,
+ "exposure (s)":1,
+ "hbin":1,
+ "height":4096
+ },
+ "Parameter Set Tag Path":"Imaging:Acquire:Record",
+ "Base Detector":{
+ "Class Name":"cm_namedcameradetectorparameterset",
+ "Name":"default"
+ },
+ "Version":34013184,
+ "Objects":{
+ "0":{
+ "Parameter 1":1,
+ "Frame Combine Style":"Copy",
+ "Class Name":"cm_imgproc_finalcombine"
+ },
+ "1":{
+ "View Name":"Frame",
+ "Screen Relative Position":[
+ 1,
+ 0
+ ],
+ "Do Auto Zoom":"false",
+ "Viewer Class":"acquire",
+ "Version":34013184,
+ "Window Relative Position":[
+ 1,
+ 0
+ ],
+ "Class Name":"cm_stdviewerimagedisplayer",
+ "Zoom":1
+ }
+ },
+ "Parameter Set Name":"Record",
+ "Environment":{
+ "Mode Name":"Imaging"
+ },
+ "Acquisition Write Flags":-1
+ },
+ "Device":{
+ "Camera Number":0,
+ "Configuration":{
+ "Transpose":{
+ "Diagonal Flip":"false",
+ "Vertical Flip":"false",
+ "Horizontal Flip":"true"
+ },
+ "Location":"high_resolution"
+ },
+ "Temperature (C)":19.99609375,
+ "CCD":{
+ "Pixel Size (um)":[
+ 15,
+ 15
+ ]
+ },
+ "Source ID":"OneView-46129640005",
+ "Source Model":"OneView",
+ "Active Size (pixels)":[
+ 4096,
+ 4096
+ ],
+ "Source":"OneView",
+ "Name":"OneView"
+ }
+ }
+ },
+ "Name":"testForClass_0006"
+ }
+ },
+ "ApplicationBounds":[
+ 0,
+ 0,
+ 1626,
+ 3011
+ ],
+ "NextDocumentObjectID":11,
+ "Page Behavior":{
+ "RestoreImageDisplayBounds":[
+ 0,
+ 0,
+ 4096,
+ 4096
+ ],
+ "PageTransform":{
+ "Scale":[
+ 1,
+ 1
+ ],
+ "Offset":[
+ 0,
+ 0
+ ]
+ },
+ "IsZoomedToWindow":"true",
+ "RestoreImageDisplayID":8,
+ "LayedOut":"false",
+ "DrawMargins":"true",
+ "TargetDisplayID":-1,
+ "DrawPaper":"true",
+ "IsFixedInPageMode":"false"
+ },
+ "Image Behavior":{
+ "WindowRect":[
+ 0,
+ 0,
+ 1590,
+ 1590
+ ],
+ "IsZoomedToWindow":"true",
+ "ImageDisplayBounds":[
+ 0,
+ 0,
+ 1590,
+ 1590
+ ],
+ "UnscaledTransform":{
+ "Scale":[
+ 1,
+ 1
+ ],
+ "Offset":[
+ 0,
+ 0
+ ]
+ },
+ "ZoomAndMoveTransform":{
+ "Scale":[
+ 1,
+ 1
+ ],
+ "Offset":[
+ 0,
+ 0
+ ]
+ },
+ "ViewDisplayID":8
+ },
+ "ImageSourceList":{
+ "0":{
+ "ImageRef":1,
+ "ClassName":"ImageSource:Simple",
+ "Id":{
+ "0":0
+ }
+ }
+ },
+ "PageSetup":{
+ "Win32":{
+ "Size":60,
+ "Offset":67720571
+ },
+ "Win32_DevModeW":{
+ "Size":1140,
+ "Offset":67720692
+ },
+ "General":[
+ 1,
+ 1000,
+ 8500,
+ 11000,
+ 1000,
+ 1000,
+ -1000,
+ -1000
+ ],
+ "Win32_DevNamesW":{
+ "Size":158,
+ "Offset":67721894
+ }
+ }
+ }
+ },
+ "resourceName":"testForClass_0006.dm4",
+ "Content-Length":67722459,
+ "Content-Type":"application/dm4"
+}
\ No newline at end of file
diff --git a/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/handlers/SharingHandler.java b/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/handlers/SharingHandler.java
index 59f4c26..75a739f 100644
--- a/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/handlers/SharingHandler.java
+++ b/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/handlers/SharingHandler.java
@@ -75,6 +75,8 @@ public class SharingHandler {
map.put("tenantId", clientId);
map.put("entityId", entity.getId());
map.put("entityType", entity.getType());
+ map.put("lastModifiedTime", entity.getUpdatedAt());
+ map.put("owner", entity.getOwnerId());
Map<String, Object> parameters = new HashMap<>();
parameters.put("props", map);
parameters.put("tenantId", clientId);
diff --git a/data-resource-management-service/drms-rest-proxy/src/main/resources/drms.pb b/data-resource-management-service/drms-rest-proxy/src/main/resources/drms.pb
index e195bc9..3542897 100644
Binary files a/data-resource-management-service/drms-rest-proxy/src/main/resources/drms.pb and b/data-resource-management-service/drms-rest-proxy/src/main/resources/drms.pb differ
diff --git a/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResourceService.proto b/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResourceService.proto
index f73804e..c9357d5 100644
--- a/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResourceService.proto
+++ b/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResourceService.proto
@@ -24,115 +24,121 @@ import "google/api/annotations.proto";
import "Common.proto";
import "resource/DRMSResource.proto";
import "google/protobuf/empty.proto";
+import "google/protobuf/struct.proto";
message ResourceFetchRequest {
- org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
- string resource_id = 2;
- string type = 3;
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ string resource_id = 2;
+ string type = 3;
}
message ChildResourceFetchRequest {
- org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
- string resource_id = 2;
- string type = 3;
- int32 depth = 4;
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ string resource_id = 2;
+ string type = 3;
+ int32 depth = 4;
}
message ChildResourceFetchResponse {
- repeated org.apache.airavata.datalake.drms.resource.GenericResource resources = 1;
+ repeated org.apache.airavata.datalake.drms.resource.GenericResource resources = 1;
}
message ResourceFetchResponse {
- org.apache.airavata.datalake.drms.resource.GenericResource resource = 1;
+ org.apache.airavata.datalake.drms.resource.GenericResource resource = 1;
}
message ResourceCreateRequest {
- org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
- org.apache.airavata.datalake.drms.resource.GenericResource resource = 2;
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ org.apache.airavata.datalake.drms.resource.GenericResource resource = 2;
}
message ResourceCreateResponse {
- org.apache.airavata.datalake.drms.resource.GenericResource resource = 1;
+ org.apache.airavata.datalake.drms.resource.GenericResource resource = 1;
}
message ResourceUpdateRequest {
- org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
- string resourceId = 2;
- org.apache.airavata.datalake.drms.resource.GenericResource resource = 3;
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ string resourceId = 2;
+ org.apache.airavata.datalake.drms.resource.GenericResource resource = 3;
}
message ResourceUpdateResponse {
- org.apache.airavata.datalake.drms.resource.GenericResource resource = 1;
+ org.apache.airavata.datalake.drms.resource.GenericResource resource = 1;
}
message ResourceDeleteRequest {
- org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
- string resourceId = 2;
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ string resourceId = 2;
}
message ResourceSearchQuery {
- string field = 1;
- string value = 2;
- string options = 3; // LIKE, NOT, EQUALS, GREATER, LESS
+ string field = 1;
+ string value = 2;
+ string options = 3; // LIKE, NOT, EQUALS, GREATER, LESS
}
message ResourceSearchRequest {
- org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
- repeated ResourceSearchQuery queries = 2;
- int32 depth = 4;
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ repeated ResourceSearchQuery queries = 2;
+ int32 depth = 4;
+ string type = 5;
}
message ResourceSearchResponse {
- org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
- repeated org.apache.airavata.datalake.drms.resource.GenericResource resources = 2;
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ repeated org.apache.airavata.datalake.drms.resource.GenericResource resources = 2;
}
message Metadata {
- string key = 1;
- string value = 2;
+ string key = 1;
+ string value = 2;
}
message AddResourceMetadataRequest {
- org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
- string resourceId = 2;
- Metadata metadata = 3;
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ string resourceId = 2;
+ string type = 3;
+ google.protobuf.Struct metadata = 4;
+
}
message FetchResourceMetadataRequest {
- org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
- string resourceId = 2;
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ string resourceId = 2;
+ string type = 3;
}
-message MetadataNode {
- repeated Metadata metadata = 1;
+message FetchResourceMetadataResponse {
+ repeated google.protobuf.Struct metadata = 1;
+
}
-message FetchResourceMetadataResponse {
- MetadataNode metadataNode = 1;
+message MetadataNode {
+ repeated Metadata metadata = 1;
}
-message AddChildResourcesMembershipRequest{
- org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
- org.apache.airavata.datalake.drms.resource.GenericResource parent_resource = 2;
- repeated org.apache.airavata.datalake.drms.resource.GenericResource child_resources = 3;
+message AddChildResourcesMembershipRequest {
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ org.apache.airavata.datalake.drms.resource.GenericResource parent_resource = 2;
+ repeated org.apache.airavata.datalake.drms.resource.GenericResource child_resources = 3;
}
-message DeleteChildResourcesMembershipRequest{
- org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
- org.apache.airavata.datalake.drms.resource.GenericResource parent_resource = 2;
- repeated org.apache.airavata.datalake.drms.resource.GenericResource child_resources = 3;
+message DeleteChildResourcesMembershipRequest {
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ org.apache.airavata.datalake.drms.resource.GenericResource parent_resource = 2;
+ repeated org.apache.airavata.datalake.drms.resource.GenericResource child_resources = 3;
}
-message ParentResourcesFetchRequest{
- org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
- string resource_id = 2;
- string type =3;
- int32 depth =4;
+message ParentResourcesFetchRequest {
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ string resource_id = 2;
+ string type = 3;
+ int32 depth = 4;
}
@@ -140,77 +146,83 @@ message ParentResourcesFetchResponse {
map<string, org.apache.airavata.datalake.drms.resource.GenericResource> properties = 1;
}
-message OperationStatusResponse{
- bool status = 1;
+message OperationStatusResponse {
+ bool status = 1;
}
service ResourceService {
- rpc fetchResource (ResourceFetchRequest) returns (ResourceFetchResponse) {
- option (google.api.http) = {
+ rpc fetchResource (ResourceFetchRequest) returns (ResourceFetchResponse) {
+ option (google.api.http) = {
get: "/v1.0/api/drms/resource"
};
- }
+ }
- rpc createResource (ResourceCreateRequest) returns (ResourceCreateResponse) {
- option (google.api.http) = {
+ rpc createResource (ResourceCreateRequest) returns (ResourceCreateResponse) {
+ option (google.api.http) = {
post: "/v1.0/api/drms/resource"
};
- }
+ }
- rpc updateResource (ResourceUpdateRequest) returns (ResourceUpdateResponse) {
- option (google.api.http) = {
+ rpc updateResource (ResourceUpdateRequest) returns (ResourceUpdateResponse) {
+ option (google.api.http) = {
put: "/v1.0/api/drms/resource"
};
- }
+ }
- rpc deletePreferenceStorage (ResourceDeleteRequest) returns (google.protobuf.Empty) {
- option (google.api.http) = {
+ rpc deletePreferenceStorage (ResourceDeleteRequest) returns (google.protobuf.Empty) {
+ option (google.api.http) = {
delete: "/v1.0/api/drms/resource"
};
- }
+ }
- rpc searchResource (ResourceSearchRequest) returns (ResourceSearchResponse) {
- option (google.api.http) = {
+ rpc searchResource (ResourceSearchRequest) returns (ResourceSearchResponse) {
+ option (google.api.http) = {
post: "/v1.0/api/drms/resource/searchResource"
};
- }
+ }
- rpc fetchChildResources (ChildResourceFetchRequest) returns (ChildResourceFetchResponse) {
- option (google.api.http) = {
+ rpc fetchChildResources (ChildResourceFetchRequest) returns (ChildResourceFetchResponse) {
+ option (google.api.http) = {
get: "/v1.0/api/drms/resource/child"
};
- }
+ }
- rpc fetchParentResources (ParentResourcesFetchRequest) returns (ParentResourcesFetchResponse) {
- option (google.api.http) = {
+ rpc fetchParentResources (ParentResourcesFetchRequest) returns (ParentResourcesFetchResponse) {
+ option (google.api.http) = {
get: "/v1.0/api/drms/resource/parent"
};
- }
+ }
- rpc addChildMembership (AddChildResourcesMembershipRequest) returns (OperationStatusResponse) {
- option (google.api.http) = {
+ rpc addChildMembership (AddChildResourcesMembershipRequest) returns (OperationStatusResponse) {
+ option (google.api.http) = {
post: "/v1.0/api/drms/resource/child"
};
- }
+ }
- rpc deleteChildMembership (DeleteChildResourcesMembershipRequest) returns (OperationStatusResponse) {
- option (google.api.http) = {
+ rpc deleteChildMembership (DeleteChildResourcesMembershipRequest) returns (OperationStatusResponse) {
+ option (google.api.http) = {
delete: "/v1.0/api/drms/resource/child"
};
- }
+ }
- rpc addResourceMetadata (AddResourceMetadataRequest) returns (google.protobuf.Empty) {
- option (google.api.http) = {
+ rpc addResourceMetadata (AddResourceMetadataRequest) returns (google.protobuf.Empty) {
+ option (google.api.http) = {
post: "/v1.0/api/drms/resource/metadata"
};
- }
+ }
+
+ rpc fetchResourceMetadata (FetchResourceMetadataRequest) returns (FetchResourceMetadataResponse) {
+ option (google.api.http) = {
+ get: "/v1.0/api/drms/resource/metadata"
+ };
+ }
- rpc fetchResourceMetadata (FetchResourceMetadataRequest) returns (FetchResourceMetadataResponse) {
- option (google.api.http) = {
+ rpc SearchResourceThroughMetadata (FetchResourceMetadataRequest) returns (FetchResourceMetadataResponse) {
+ option (google.api.http) = {
get: "/v1.0/api/drms/resource/metadata"
};
- }
+ }
}
\ No newline at end of file