You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by is...@apache.org on 2021/09/13 13:25:55 UTC
[airavata-data-lake] branch master updated: Bug fix in fetching
COLLECTION_GROUP in resource queries
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
The following commit(s) were added to refs/heads/master by this push:
new 37e1b4f Bug fix in fetching COLLECTION_GROUP in resource queries
new ebdec66 Merge pull request #73 from isururanawaka/sharing_service_impl
37e1b4f is described below
commit 37e1b4fd6aac5fc0e04773f9e0922e6ade1eb4c4
Author: Isuru Ranawaka <ir...@gmail.com>
AuthorDate: Mon Sep 13 09:24:27 2021 -0400
Bug fix in fetching COLLECTION_GROUP in resource queries
---
.../drms/api/handlers/ResourceServiceHandler.java | 72 ++++++++++++----------
.../custos/synchronizer/CustosSynchronizer.java | 2 +-
.../datafetcher/CustosDataFetchingJob.java | 14 ++---
.../handlers/events/CustosEventListener.java | 2 +-
.../src/main/resources/config.yml | 12 ++--
5 files changed, 53 insertions(+), 49 deletions(-)
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
index 304618e..63b70eb 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
@@ -529,14 +529,13 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
responseObserver.onCompleted();
return;
} else {
+ String query = "";
+ Map<String, Object> userProps = new HashMap<>();
+ userProps.put("username", callUser.getUsername());
+ userProps.put("tenantId", callUser.getTenantId());
+ if ((value.equals("FILE") || value.equals("COLLECTION"))) {
+ for (String storageId : storageList) {
- for (String storageId : storageList) {
- Map<String, Object> userProps = new HashMap<>();
- userProps.put("username", callUser.getUsername());
- userProps.put("tenantId", callUser.getTenantId());
-
- String query = "";
- if ((value.equals("FILE") || value.equals("COLLECTION")) && !storageId.isEmpty()) {
query = " MATCH (u:User) where u.username = $username AND u.tenantId = $tenantId" +
" OPTIONAL MATCH (g:Group)<-[:MEMBER_OF]-(u) " +
" OPTIONAL MATCH (u)<-[relRM:SHARED_WITH]-(m)<-[:CHILD_OF*]-(rm:" + value + ")-[:CHILD_OF*]->(s:Storage{entityId:'" + storageId + "'})" +
@@ -563,51 +562,56 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
keyList.add("r2:relR2");
keyList.add("r3:relR3");
}
+ logger.debug("Search query {}", query);
+
+ List<Record> records = this.neo4JConnector.searchNodes(userProps, query);
- } else {
+ List<GenericResource> genericResourceList = GenericResourceDeserializer.deserializeList(records, keyList);
+ allowedResourceList.addAll(genericResourceList);
+ }
+ } else {
+ query = " MATCH (u:User) where u.username = $username AND u.tenantId = $tenantId" +
+ " OPTIONAL MATCH (g:Group)<-[:MEMBER_OF]-(u) " +
+ " OPTIONAL MATCH (u)<-[relRM:SHARED_WITH]-(m)<-[:CHILD_OF*]-(rm:" + value + ")" +
+ " , (r:" + value + ")-[relR:SHARED_WITH]->(u)" +
+ " OPTIONAL MATCH (g)<-[relRMG:SHARED_WITH]-(mg)<-[:CHILD_OF*]-(rmg:" + value + ")" +
+ " , (rg:" + value + ")-[relRG:SHARED_WITH]->(g)" +
+ " return distinct rm,relRM, r,relR, rmg,relRMG, rg, relRG ";
+ keyList = new ArrayList();
+ keyList.add("rm:relRM");
+ keyList.add("r:relR");
+ keyList.add("rmg:relRMG");
+ keyList.add("rg:relRG");
+ if (depth == 1) {
query = " MATCH (u:User) where u.username = $username AND u.tenantId = $tenantId" +
" OPTIONAL MATCH (g:Group)<-[:MEMBER_OF]-(u) " +
- " OPTIONAL MATCH (u)<-[relRM:SHARED_WITH]-(m)<-[:CHILD_OF*]-(rm:" + value + ")" +
- " , (r:" + value + ")-[relR:SHARED_WITH]->(u)" +
- " OPTIONAL MATCH (g)<-[relRMG:SHARED_WITH]-(mg)<-[:CHILD_OF*]-(rmg:" + value + ")" +
- " , (rg:" + value + ")-[relRG:SHARED_WITH]->(g)" +
- " return distinct rm,relRM, r,relR, rmg,relRMG, rg, relRG ";
+ " OPTIONAL MATCH (r:" + value + ")-[relR:SHARED_WITH]->(u)" +
+ " OPTIONAL MATCH (rg:" + value + ")-[relRG:SHARED_WITH]->(g)" +
+ " return distinct r,relR, rg, relRG ";
keyList = new ArrayList();
- keyList.add("rm:relRM");
keyList.add("r:relR");
- keyList.add("rmg:relRMG");
keyList.add("rg:relRG");
- if (depth == 1) {
- query = " MATCH (u:User) where u.username = $username AND u.tenantId = $tenantId" +
- " OPTIONAL MATCH (g:Group)<-[:MEMBER_OF]-(u) " +
- " OPTIONAL MATCH (r:" + value + ")-[relR:SHARED_WITH]->(u)" +
- " OPTIONAL MATCH (rg:" + value + ")-[relRG:SHARED_WITH]->(g)" +
- " return distinct r,relR, rg, relRG ";
- keyList = new ArrayList();
- keyList.add("r:relR");
- keyList.add("rg:relRG");
- }
}
-
- logger.debug("Search query {}", query);
-
List<Record> records = this.neo4JConnector.searchNodes(userProps, query);
List<GenericResource> genericResourceList = GenericResourceDeserializer.deserializeList(records, keyList);
allowedResourceList.addAll(genericResourceList);
}
- ResourceSearchResponse.Builder builder = ResourceSearchResponse.newBuilder();
- builder.addAllResources(allowedResourceList);
- responseObserver.onNext(builder.build());
- responseObserver.onCompleted();
- }
- } catch (Exception e) {
+ }
+ ResourceSearchResponse.Builder builder = ResourceSearchResponse.newBuilder();
+ builder.addAllResources(allowedResourceList);
+ responseObserver.onNext(builder.build());
+ responseObserver.onCompleted();
+
+ } catch (
+ Exception e) {
logger.error("Errored while searching generic resources; Message: {}", e.getMessage(), e);
responseObserver.onError(Status.INTERNAL.withDescription("Errored while searching generic resources "
+ e.getMessage()).asRuntimeException());
}
+
}
diff --git a/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/CustosSynchronizer.java b/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/CustosSynchronizer.java
index a12a7d8..223d6ab 100644
--- a/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/CustosSynchronizer.java
+++ b/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/CustosSynchronizer.java
@@ -67,7 +67,7 @@ public class CustosSynchronizer implements CommandLineRunner {
custosEventListener.consume(new ConsumerCallback() {
@Override
public void process(Message notificationEvent) throws Exception {
- LOGGER.debug("Message Id" + notificationEvent.getMessageId());
+ LOGGER.info("Process from kafka message Id" + notificationEvent.getMessageId());
EventDemux.delegateEvents(notificationEvent);
}
});
diff --git a/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/datafetcher/CustosDataFetchingJob.java b/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/datafetcher/CustosDataFetchingJob.java
index b6960b7..407ea9d 100644
--- a/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/datafetcher/CustosDataFetchingJob.java
+++ b/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/datafetcher/CustosDataFetchingJob.java
@@ -24,13 +24,13 @@ public class CustosDataFetchingJob implements Job {
public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
try {
LOGGER.debug("Executing CustosDataFetchingJob ....... ");
- JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap();
- String path = jobDataMap.getString("configurationPath");
- Configuration configuration = loadConfiguration(path);
- UserAndGroupHandler userAndGroupHandler = new UserAndGroupHandler();
- userAndGroupHandler.mergeUserAndGroups(configuration);
- SharingHandler sharingHandler = new SharingHandler();
- sharingHandler.mergeSharings(configuration);
+// JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap();
+// String path = jobDataMap.getString("configurationPath");
+// Configuration configuration = loadConfiguration(path);
+// UserAndGroupHandler userAndGroupHandler = new UserAndGroupHandler();
+// userAndGroupHandler.mergeUserAndGroups(configuration);
+// SharingHandler sharingHandler = new SharingHandler();
+// sharingHandler.mergeSharings(configuration);
} catch (Exception ex) {
String msg = "Error occurred while executing job" + ex.getMessage();
LOGGER.error(msg, ex);
diff --git a/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/handlers/events/CustosEventListener.java b/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/handlers/events/CustosEventListener.java
index 18a3d59..b1b789b 100644
--- a/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/handlers/events/CustosEventListener.java
+++ b/data-resource-management-service/drms-custos-synchronizer/src/main/java/org/apache/airavata/drms/custos/synchronizer/handlers/events/CustosEventListener.java
@@ -51,7 +51,7 @@ public class CustosEventListener {
try {
callback.process(record.value());
} catch (Exception exception) {
- exception.printStackTrace();
+ LOGGER.info("Exception occurred in kafka listener ",exception.getMessage());
} finally {
consumer.commitSync(Collections.singletonMap(partition, new OffsetAndMetadata(record.offset() + 1)));
}
diff --git a/data-resource-management-service/drms-custos-synchronizer/src/main/resources/config.yml b/data-resource-management-service/drms-custos-synchronizer/src/main/resources/config.yml
index 621ecc6..b03d4dc 100644
--- a/data-resource-management-service/drms-custos-synchronizer/src/main/resources/config.yml
+++ b/data-resource-management-service/drms-custos-synchronizer/src/main/resources/config.yml
@@ -1,17 +1,17 @@
pollingInterval: 60
dataResourceManagementService:
- dbURI: "bolt://192.168.0.14:7687"
+ dbURI: "bolt://149.165.156.173:7687"
dbUser: "neo4j"
- dbPassword: "123456"
+ dbPassword: "blastcovid19"
custos:
host: "custos.scigap.org"
port: 31499
- custosId: "custos-whedmgamitu357p4wuke-10002708"
- custosSec: "mrMdl86Ia1H94cikW7CvHoh7L0ASNXQVt2aRzSIj"
+ custosId: "custos-ii8g0cfwsz6ruwezykn9-10002640"
+ custosSec: "OxXECszt9dL4lHJQyL444UOU0lKN317D51ez067R"
custosBrokerURL: "149.165.156.200:9092"
consumerGroup: "custosEventsGroup"
maxPollRecordsConfig: 10
topics:
- - "10002708-754b89b5-3a57-496c-aa34-8e2b4916fbc"
+ - "10002640-561b67b4-fa7d-4368-8643-dcd9d0159c1d"
tenantsToBeSynced:
- - "custos-whedmgamitu357p4wuke-10002708"
+ - "custos-ii8g0cfwsz6ruwezykn9-10002640"