You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2021/06/01 03:03:21 UTC
[airavata-data-lake] 14/42: Implementing the storage layer with
metadata and sharing support
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
commit 2056a2079209548cb09bcdf2bf72c5c163183286
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Wed Mar 24 09:37:30 2021 -0400
Implementing the storage layer with metadata and sharing support
---
.gitignore | 3 +
data-resource-management-service/drms-api/pom.xml | 29 ++++--
.../java/org/apache/airavata/drms/api/Client.java | 70 +++++++++++++
.../java/org/apache/airavata/drms/api/Config.java | 32 ++++++
.../apache/airavata/drms/api/DRMSApiRunner.java | 16 +++
.../drms/api/handlers/GroupServiceHandler.java | 95 ++++++++++++++++++
.../drms/api/handlers/ResourceServiceHandler.java | 16 +++
.../handlers/StoragePreferenceServiceHandler.java | 16 +++
.../drms/api/handlers/StorageServiceHandler.java | 109 ++++++++++++++++++++-
.../src/main/resources/application.properties | 23 +++++
.../drms-api/src/main/resources/logback.xml | 45 +++++++++
data-resource-management-service/drms-core/pom.xml | 40 +++++++-
.../apache/airavata/drms/core/Neo4JConnector.java | 66 +++++++++++++
.../drms/core/constants/StorageConstants.java | 24 +++++
.../storage/AnyStorageDeserializer.java | 81 +++++++++++++++
.../serializer/storage/AnyStorageSerializer.java | 58 +++++++++++
.../drms-stubs/pom.xml | 35 +++----
.../drms-stubs/src/main/proto/Common.proto | 17 ++++
.../drms-stubs/src/main/proto/group/Group.proto | 46 +++++++++
.../src/main/proto/group/GroupService.proto | 86 ++++++++++++++++
.../src/main/proto/preference/S3Preference.proto | 17 ++++
.../src/main/proto/preference/SSHPreference.proto | 17 ++++
.../preference/StoragePreferenceService.proto | 17 ++++
.../src/main/proto/resource/DRMSResource.proto | 17 ++++
.../main/proto/resource/DRMSResourceService.proto | 17 ++++
.../src/main/proto/sharing/Sharing.proto | 28 ++++++
.../src/main/proto/storage/S3Storage.proto | 17 ++++
.../src/main/proto/storage/SSHStorage.proto | 17 ++++
.../src/main/proto/storage/StorageService.proto | 30 ++++++
data-resource-management-service/pom.xml | 28 +++++-
pom.xml | 4 +
31 files changed, 1083 insertions(+), 33 deletions(-)
diff --git a/.gitignore b/.gitignore
index cb794d5..f07bdb5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,6 @@ data-orchestrator/data-orchestrator-api/target
/metadata-service/metadata-service.iml
/metadata-service/data-builders/data-builders.iml
/metadata-service/db-service/db-service.iml
+/metadata-service/db-service/client/client.iml
+/metadata-service/db-service/server/server.iml
+/metadata-service/db-service/stub/stub.iml
diff --git a/data-resource-management-service/drms-api/pom.xml b/data-resource-management-service/drms-api/pom.xml
index a6d5805..92e265b 100644
--- a/data-resource-management-service/drms-api/pom.xml
+++ b/data-resource-management-service/drms-api/pom.xml
@@ -1,4 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@@ -15,11 +31,6 @@
<dependency>
<groupId>org.apache.airavata.data.lake</groupId>
<artifactId>drms-stubs</artifactId>
- <version>0.01-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata.data.lake</groupId>
- <artifactId>stub</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
@@ -41,7 +52,7 @@
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
- <version>2.4.2</version>
+ <version>${spring.boot.version}</version>
</dependency>
<dependency>
<groupId>net.sf.dozer</groupId>
@@ -49,9 +60,9 @@
<version>${dozer}</version>
</dependency>
<dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-services</artifactId>
- <version>1.25.0</version>
+ <groupId>org.apache.airavata.data.lake</groupId>
+ <artifactId>drms-core</artifactId>
+ <version>${project.version}</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Client.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Client.java
new file mode 100644
index 0000000..8e589ef
--- /dev/null
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Client.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.airavata.drms.api;
+
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import org.apache.airavata.datalake.drms.DRMSServiceAuthToken;
+import org.apache.airavata.datalake.drms.groups.*;
+import org.apache.airavata.datalake.drms.storage.*;
+
+public class Client {
+ public static void main(String ar[]) {
+
+ DRMSServiceAuthToken token1 = DRMSServiceAuthToken.newBuilder().setAccessToken("Token-1").build();
+ DRMSServiceAuthToken token2 = DRMSServiceAuthToken.newBuilder().setAccessToken("Token-2").build();
+
+ ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 6565).usePlaintext().build();
+ StorageServiceGrpc.StorageServiceBlockingStub storageClient = StorageServiceGrpc.newBlockingStub(channel);
+
+ System.out.println("List for user 1");
+ StorageSearchResponse storages = storageClient.searchStorage(
+ StorageSearchRequest.newBuilder().setAuthToken(token1).build());
+ System.out.println(storages);
+
+ System.out.println("List for user 2");
+ storages = storageClient.searchStorage(
+ StorageSearchRequest.newBuilder().setAuthToken(token2).build());
+ System.out.println(storages);
+
+ System.out.println("Fetch");
+ StorageFetchResponse fetchResponse = storageClient.fetchStorage(
+ StorageFetchRequest.newBuilder().setAuthToken(token1).setStorageId("staging_pga_storage").buildPartial());
+
+ System.out.println(fetchResponse);
+
+ GroupServiceGrpc.GroupServiceBlockingStub groupClient = GroupServiceGrpc.newBlockingStub(channel);
+
+ System.out.println("User");
+ FetchCurrentUserResponse currentUser = groupClient.fetchCurrentUser(
+ FetchCurrentUserRequest.newBuilder().setAuthToken(token1).build());
+ System.out.println(currentUser);
+
+ System.out.println("Groups");
+ FetchCurrentUserGroupsResponse currentGroups = groupClient.fetchCurrentUserGroups(
+ FetchCurrentUserGroupsRequest.newBuilder().setAuthToken(token2).build());
+ System.out.println(currentGroups);
+
+ System.out.println("Adding metadata");
+ storageClient.addStorageMetadata(AddStorageMetadataRequest.newBuilder()
+ .setAuthToken(token1)
+ .setStorageId("prod_pga")
+ .setKey("createdOn")
+ .setValue("02/15/2021")
+ .build());
+ }
+}
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Config.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Config.java
index 494a21e..3ec0ab5 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Config.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Config.java
@@ -1,7 +1,39 @@
package org.apache.airavata.drms.api;
+import org.apache.airavata.drms.core.Neo4JConnector;
+import org.springframework.context.annotation.Bean;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
import org.springframework.context.annotation.Configuration;
@Configuration
public class Config {
+
+ @org.springframework.beans.factory.annotation.Value("${neo4j.server.uri}")
+ public String neo4jServerUri;
+
+ @org.springframework.beans.factory.annotation.Value("${neo4j.server.user}")
+ public String neo4jServerUser;
+
+ @org.springframework.beans.factory.annotation.Value("${neo4j.server.password}")
+ public String neo4jServerPassword;
+
+ @Bean
+ public Neo4JConnector neo4JConnector() {
+ return new Neo4JConnector(neo4jServerUri, neo4jServerUser, neo4jServerPassword);
+ }
}
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/DRMSApiRunner.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/DRMSApiRunner.java
index 69a2203..21df7ec 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/DRMSApiRunner.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/DRMSApiRunner.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.airavata.drms.api;
import org.springframework.boot.SpringApplication;
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/GroupServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/GroupServiceHandler.java
new file mode 100644
index 0000000..5ad81d1
--- /dev/null
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/GroupServiceHandler.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.airavata.drms.api.handlers;
+
+import io.grpc.stub.StreamObserver;
+import org.apache.airavata.datalake.drms.groups.*;
+import org.lognet.springboot.grpc.GRpcService;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@GRpcService
+public class GroupServiceHandler extends GroupServiceGrpc.GroupServiceImplBase {
+
+ @Override
+ public void fetchCurrentUser(FetchCurrentUserRequest request, StreamObserver<FetchCurrentUserResponse> responseObserver) {
+ try {
+ User user = getUserByAccessToken(request.getAuthToken().getAccessToken());
+ responseObserver.onNext(FetchCurrentUserResponse.newBuilder().setUser(user).build());
+ responseObserver.onCompleted();
+ } catch (Exception e) {
+ responseObserver.onError(e);
+ }
+ }
+
+ @Override
+ public void fetchUser(FetchUserRequest request, StreamObserver<FetchUserResponse> responseObserver) {
+ super.fetchUser(request, responseObserver);
+ }
+
+ @Override
+ public void fetchUserGroups(FetchUserGroupsRequest request, StreamObserver<FetchUserGroupsResponse> responseObserver) {
+ super.fetchUserGroups(request, responseObserver);
+ }
+
+ @Override
+ public void fetchCurrentUserGroups(FetchCurrentUserGroupsRequest request,
+ StreamObserver<FetchCurrentUserGroupsResponse> responseObserver) {
+ try {
+ User user = getUserByAccessToken(request.getAuthToken().getAccessToken());
+ List<Group> userGroups = getUserGroups(user.getUserId());
+ responseObserver.onNext(FetchCurrentUserGroupsResponse.newBuilder().addAllGroups(userGroups).build());
+ responseObserver.onCompleted();
+
+ } catch (Exception e) {
+ responseObserver.onError(e);
+ }
+ }
+
+ private User getUserByAccessToken(String accessToken) throws Exception {
+ switch (accessToken) {
+ case "Token-1":
+ return User.newBuilder().setUserId("user-1").setUsername("bob").setFirstName("Bob").setLastName("Leech")
+ .setEmailAddress("bob@gmail.com").build();
+ case "Token-2":
+ return User.newBuilder().setUserId("user-2").setUsername("alice").setFirstName("Alice").setLastName("Ann")
+ .setEmailAddress("alice@gmail.com").build();
+ }
+ throw new Exception("No user for given access token");
+ }
+
+ private List<Group> getUserGroups(String userId) {
+ Map<String, Group> groups = new HashMap<>();
+ List<Group> groupList = new ArrayList<>();
+ groups.put("group-1", Group.newBuilder().setGroupId("group-1").setName("Teachers").setDescription("Teachers").build());
+ groups.put("group-2", Group.newBuilder().setGroupId("group-2").setName("Students").setDescription("Students").build());
+
+ switch (userId) {
+ case "user-1":
+ groupList.add(groups.get("group-1"));
+ groupList.add(groups.get("group-2"));
+ break;
+ case "user-2":
+ groupList.add(groups.get("group-2"));
+ break;
+ }
+ return groupList;
+ }
+}
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
index ecc26a4..4e34e7a 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.airavata.drms.api.handlers;
import com.google.protobuf.Empty;
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java
index 739921d..a010253 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.airavata.drms.api.handlers;
import com.google.protobuf.Empty;
diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java
index 47c6511..29e34e7 100644
--- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java
+++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java
@@ -1,21 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.airavata.drms.api.handlers;
import com.google.protobuf.Empty;
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
import io.grpc.stub.StreamObserver;
+import org.apache.airavata.datalake.drms.DRMSServiceAuthToken;
+import org.apache.airavata.datalake.drms.groups.FetchCurrentUserRequest;
+import org.apache.airavata.datalake.drms.groups.FetchCurrentUserResponse;
+import org.apache.airavata.datalake.drms.groups.GroupServiceGrpc;
+import org.apache.airavata.datalake.drms.groups.User;
import org.apache.airavata.datalake.drms.storage.*;
+import org.apache.airavata.drms.core.Neo4JConnector;
+import org.apache.airavata.drms.core.deserializer.storage.AnyStorageDeserializer;
+import org.apache.airavata.drms.core.constants.StorageConstants;
+import org.apache.airavata.drms.core.serializer.storage.AnyStorageSerializer;
import org.lognet.springboot.grpc.GRpcService;
+import org.neo4j.driver.Record;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
@GRpcService
public class StorageServiceHandler extends StorageServiceGrpc.StorageServiceImplBase {
+ private static final Logger logger = LoggerFactory.getLogger(StorageServiceHandler.class);
+
+ @Autowired
+ private Neo4JConnector neo4JConnector;
+
+ @org.springframework.beans.factory.annotation.Value("${group.service.host}")
+ private String groupServiceHost;
+
+ @org.springframework.beans.factory.annotation.Value("${group.service.port}")
+ private int groupServicePort;
+
+
+ private User getUser(DRMSServiceAuthToken authToken) {
+ ManagedChannel channel = ManagedChannelBuilder.forAddress(groupServiceHost, groupServicePort).usePlaintext().build();
+ GroupServiceGrpc.GroupServiceBlockingStub groupClient = GroupServiceGrpc.newBlockingStub(channel);
+ FetchCurrentUserResponse userResponse = groupClient.fetchCurrentUser(
+ FetchCurrentUserRequest.newBuilder().setAuthToken(authToken).build());
+ return userResponse.getUser();
+ }
+
@Override
public void fetchStorage(StorageFetchRequest request, StreamObserver<StorageFetchResponse> responseObserver) {
- super.fetchStorage(request, responseObserver);
+
+ User callUser = getUser(request.getAuthToken());
+
+ List<Record> records = this.neo4JConnector.searchNodes(
+ "MATCH (u:User)-[r1:MEMBER_OF]->(g:Group)<-[r2:SHARED_WITH]-(s:Storage) where " +
+ "s.storageId = '" + request.getStorageId() + "' and u.userId = '" + callUser.getUserId() +
+ "' return distinct s");
+
+ if (!records.isEmpty()) {
+ try {
+ List<AnyStorage> storageList = AnyStorageDeserializer.deserializeList(records);
+ responseObserver.onNext(StorageFetchResponse.newBuilder().setStorage(storageList.get(0)).build());
+ responseObserver.onCompleted();
+ } catch (Exception e) {
+
+ logger.error("Errored while fetching storage with id {}", request.getStorageId(), e);
+ responseObserver.onError(new Exception("Errored while fetching storage with id " + request.getStorageId()
+ + ". Msg " + e.getMessage()));
+ }
+ } else {
+ logger.error("Could not find a storage with id {}", request.getStorageId());
+ responseObserver.onError(new Exception("Could not find a storage with id " + request.getStorageId()));
+ }
}
@Override
public void createStorage(StorageCreateRequest request, StreamObserver<StorageCreateResponse> responseObserver) {
- super.createStorage(request, responseObserver);
+ User user = getUser(request.getAuthToken());
+ AnyStorage storage = request.getStorage();
+ Map<String, Object> serializedMap = AnyStorageSerializer.serializeToMap(storage);
+ this.neo4JConnector.createNode(serializedMap, StorageConstants.STORAGE_LABEL, user.getUserId());
}
@Override
@@ -29,7 +109,30 @@ public class StorageServiceHandler extends StorageServiceGrpc.StorageServiceImpl
}
@Override
+ public void addStorageMetadata(AddStorageMetadataRequest request, StreamObserver<Empty> responseObserver) {
+ User callUser = getUser(request.getAuthToken());
+ this.neo4JConnector.createMetadataNode(StorageConstants.STORAGE_LABEL, "storageId",
+ request.getStorageId(), callUser.getUserId(),
+ request.getKey(), request.getValue());
+ }
+
+ @Override
public void searchStorage(StorageSearchRequest request, StreamObserver<StorageSearchResponse> responseObserver) {
- super.searchStorage(request, responseObserver);
+ User callUser = getUser(request.getAuthToken());
+
+ List<Record> records = this.neo4JConnector.searchNodes(
+ "MATCH (u:User)-[r1:MEMBER_OF]->(g:Group)<-[r2:SHARED_WITH]-(s:Storage) where u.userId ='" +
+ callUser.getUserId() + "' return distinct s");
+ try {
+ List<AnyStorage> storageList = AnyStorageDeserializer.deserializeList(records);
+ StorageSearchResponse.Builder builder = StorageSearchResponse.newBuilder();
+ builder.addAllStorages(storageList);
+ responseObserver.onNext(builder.build());
+ responseObserver.onCompleted();
+
+ } catch (Exception e) {
+ logger.error("Errored while searching storages; Message: {}", e.getMessage(), e);
+ responseObserver.onError(e);
+ }
}
}
diff --git a/data-resource-management-service/drms-api/src/main/resources/application.properties b/data-resource-management-service/drms-api/src/main/resources/application.properties
new file mode 100644
index 0000000..d674f86
--- /dev/null
+++ b/data-resource-management-service/drms-api/src/main/resources/application.properties
@@ -0,0 +1,23 @@
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+neo4j.server.uri=bolt://192.168.0.14:7687
+neo4j.server.user=neo4j
+neo4j.server.password=123456
+group.service.host=localhost
+group.service.port=6565
\ No newline at end of file
diff --git a/data-resource-management-service/drms-api/src/main/resources/logback.xml b/data-resource-management-service/drms-api/src/main/resources/logback.xml
new file mode 100644
index 0000000..dcf6d23
--- /dev/null
+++ b/data-resource-management-service/drms-api/src/main/resources/logback.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<configuration>
+
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d [%t] %-5p %c{30} %m [%X]%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="LOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <File>../logs/airavata.log</File>
+ <Append>true</Append>
+ <encoder>
+ <pattern>%d [%t] %-5p %c{30} %m [%X]%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>../logs/airavata.log.%d{yyyy-MM-dd}</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ <totalSizeCap>1GB</totalSizeCap>
+ </rollingPolicy>
+ </appender>
+
+ <logger name="ch.qos.logback" level="WARN"/>
+ <logger name="org.apache.airavata" level="INFO"/>
+ <root level="INFO">
+ <appender-ref ref="CONSOLE"/>
+ <appender-ref ref="LOGFILE"/>
+ </root>
+</configuration>
\ No newline at end of file
diff --git a/data-resource-management-service/drms-core/pom.xml b/data-resource-management-service/drms-core/pom.xml
index 497ac63..a32e81f 100644
--- a/data-resource-management-service/drms-core/pom.xml
+++ b/data-resource-management-service/drms-core/pom.xml
@@ -1,4 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@@ -15,12 +31,32 @@
<dependency>
<groupId>org.neo4j</groupId>
<artifactId>neo4j-ogm-core</artifactId>
- <version>3.2.20</version>
+ <version>${neo4j.ogm.version}</version>
</dependency>
<dependency>
<groupId>org.neo4j</groupId>
<artifactId>neo4j-ogm-bolt-driver</artifactId>
- <version>3.2.20</version>
+ <version>${neo4j.ogm.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.neo4j</groupId>
+ <artifactId>neo4j</artifactId>
+ <version>${neo4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata.data.lake</groupId>
+ <artifactId>drms-stubs</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>net.sf.dozer</groupId>
+ <artifactId>dozer</artifactId>
+ <version>${dozer}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ <version>${spring.boot.version}</version>
</dependency>
</dependencies>
diff --git a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/Neo4JConnector.java b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/Neo4JConnector.java
new file mode 100644
index 0000000..1d39ce4
--- /dev/null
+++ b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/Neo4JConnector.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.airavata.drms.core;
+
+import org.neo4j.driver.*;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class Neo4JConnector {
+
+ private String uri;
+ private String userName;
+ private String password;
+
+ public Neo4JConnector(String uri, String userName, String password) {
+ this.uri = uri;
+ this.userName = userName;
+ this.password = password;
+ }
+
+ public List<Record> searchNodes(String query) {
+ Driver driver = GraphDatabase.driver(uri, AuthTokens.basic(userName, password));
+ Session session = driver.session();
+ Result result = session.run(query);
+ return result.list();
+ }
+
+ public void createNode(Map<String, Object> properties, String label, String userId) {
+ Driver driver = GraphDatabase.driver(uri, AuthTokens.basic(userName, password));
+ Session session = driver.session();
+ Map<String, Object> parameters = new HashMap<>();
+ parameters.put("props", properties);
+ Transaction tx = session.beginTransaction();
+ Result result = tx.run("match (u:User)-[r1:MEMBER_OF {membershipType:'USER_GROUP'}]->(g:Group) where u.userId = '" + userId + "' CREATE (n:" + label + ")-[r2:SHARED_WITH {permission:'WRITE'}]->(g) SET n = $props return n", parameters);
+ tx.commit();
+ tx.close();
+ }
+
+ public void createMetadataNode(String parentLabel, String parentIdName, String parentIdValue,
+ String userId, String key, String value) {
+ Driver driver = GraphDatabase.driver(uri, AuthTokens.basic(userName, password));
+ Session session = driver.session();
+ Transaction tx = session.beginTransaction();
+ tx.run("match (u:User)-[r1:MEMBER_OF]->(g:Group)<-[r2:SHARED_WITH]-(s:" + parentLabel + ") where u.userId='" + userId +
+ "' and s." + parentIdName + "='" + parentIdValue +
+ "' merge (m:Metadata)<-[r3:HAS_METADATA]-(s) set m." + key + "='" + value + "' return m");
+ tx.commit();
+ tx.close();
+ }
+}
diff --git a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/constants/StorageConstants.java b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/constants/StorageConstants.java
new file mode 100644
index 0000000..f7d6a9d
--- /dev/null
+++ b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/constants/StorageConstants.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.airavata.drms.core.constants;
+
+public final class StorageConstants {
+ public static final String STORAGE_LABEL = "Storage";
+ public static final String STORAGE_TYPE_LABEL = "type";
+ public static final String SSH_STORAGE_TYPE_LABEL = "SSH";
+ public static final String S3_STORAGE_TYPE_LABEL = "S3";
+}
diff --git a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/storage/AnyStorageDeserializer.java b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/storage/AnyStorageDeserializer.java
new file mode 100644
index 0000000..8d0fb95
--- /dev/null
+++ b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/storage/AnyStorageDeserializer.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.airavata.drms.core.deserializer.storage;
+
+import org.apache.airavata.datalake.drms.storage.AnyStorage;
+import org.apache.airavata.datalake.drms.storage.s3.S3Storage;
+import org.apache.airavata.datalake.drms.storage.ssh.SSHStorage;
+import org.apache.airavata.drms.core.constants.StorageConstants;
+import org.neo4j.driver.Record;
+import org.neo4j.driver.Value;
+import org.neo4j.driver.internal.InternalRecord;
+import org.neo4j.driver.types.Node;
+import org.springframework.beans.BeanWrapper;
+import org.springframework.beans.PropertyAccessorFactory;
+
+import java.util.*;
+
+public class AnyStorageDeserializer {
+
+ public static List<AnyStorage> deserializeList(List<Record> neo4jRecords) throws Exception {
+ List<AnyStorage> storageList = new ArrayList<>();
+ for (Record record : neo4jRecords) {
+ InternalRecord internalRecord = (InternalRecord) record;
+ List<Value> values = internalRecord.values();
+ for (Value value : values) {
+ Node node = value.asNode();
+ if (node.hasLabel(StorageConstants.STORAGE_LABEL)) {
+ storageList.add(deriveStorageFromMap(node.asMap()));
+ }
+ }
+ }
+ return storageList;
+ }
+
+ private static AnyStorage deriveStorageFromMap(Map<String, Object> fixedMap) throws Exception {
+
+ Map<String, Object> asMap = new HashMap<>(fixedMap);
+ AnyStorage.Builder anyStorageBuilder = AnyStorage.newBuilder();
+ String type = (String)asMap.get(StorageConstants.STORAGE_TYPE_LABEL);
+ asMap.remove(StorageConstants.STORAGE_TYPE_LABEL);
+
+ switch (type) {
+ case StorageConstants.SSH_STORAGE_TYPE_LABEL:
+ SSHStorage.Builder builder = SSHStorage.newBuilder();
+ setObjectFieldsUsingMap(builder, asMap);
+ SSHStorage sshStorage = builder.build();
+ anyStorageBuilder.setSshStorage(sshStorage);
+ break;
+ case StorageConstants.S3_STORAGE_TYPE_LABEL:
+ S3Storage.Builder s3Builder = S3Storage.newBuilder();
+ setObjectFieldsUsingMap(s3Builder, asMap);
+ anyStorageBuilder.setS3Storage(s3Builder.build());
+ break;
+ default:
+ throw new Exception("Unsupported storage type for deserializing : " + type);
+ }
+
+ return anyStorageBuilder.build();
+ }
+
+ private static void setObjectFieldsUsingMap(Object target, Map<String, Object> values) {
+ for (String field :values.keySet()) {
+ BeanWrapper beanWrapper = PropertyAccessorFactory.forBeanPropertyAccess(target);
+ beanWrapper.setPropertyValue(field, values.get(field));
+ }
+ }
+}
diff --git a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/serializer/storage/AnyStorageSerializer.java b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/serializer/storage/AnyStorageSerializer.java
new file mode 100644
index 0000000..d286798
--- /dev/null
+++ b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/serializer/storage/AnyStorageSerializer.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.airavata.drms.core.serializer.storage;
+
+import com.google.protobuf.Descriptors;
+import org.apache.airavata.datalake.drms.storage.AnyStorage;
+import org.apache.airavata.datalake.drms.storage.s3.S3Storage;
+import org.apache.airavata.datalake.drms.storage.ssh.SSHStorage;
+import org.apache.airavata.drms.core.constants.StorageConstants;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class AnyStorageSerializer {
+
+ public static Map<String, Object> serializeToMap(AnyStorage anyStorage) {
+
+ Map<String, Object> fields = new HashMap<>();
+ Map<Descriptors.FieldDescriptor, Object> allFields = null;
+ switch (anyStorage.getStorageCase()) {
+ case SSHSTORAGE:
+ SSHStorage sshStorage = anyStorage.getSshStorage();
+ allFields = sshStorage.getAllFields();
+ fields.put(StorageConstants.STORAGE_TYPE_LABEL, StorageConstants.SSH_STORAGE_TYPE_LABEL);
+ break;
+ case S3STORAGE:
+ S3Storage s3Storage = anyStorage.getS3Storage();
+ allFields = s3Storage.getAllFields();
+ fields.put(StorageConstants.STORAGE_TYPE_LABEL, StorageConstants.S3_STORAGE_TYPE_LABEL);
+ break;
+ case STORAGE_NOT_SET:
+ break;
+ }
+
+ if (allFields != null) {
+ allFields.forEach((descriptor, value) -> {
+ String fieldName = descriptor.getJsonName();
+ fields.put(fieldName, value);
+ });
+ }
+
+ return fields;
+ }
+}
diff --git a/data-resource-management-service/drms-stubs/pom.xml b/data-resource-management-service/drms-stubs/pom.xml
index 915db71..abb8a3b 100644
--- a/data-resource-management-service/drms-stubs/pom.xml
+++ b/data-resource-management-service/drms-stubs/pom.xml
@@ -1,4 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@@ -21,7 +37,7 @@
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-services</artifactId>
- <version>1.35.0</version>
+ <version>${grpc.services}</version>
<exclusions>
<exclusion>
<groupId>com.google.code.gson</groupId>
@@ -32,7 +48,7 @@
<dependency>
<groupId>javax.annotation</groupId>
<artifactId>javax.annotation-api</artifactId>
- <version>${javax.annotation}</version>
+ <version>${javax.annotation.version}</version>
</dependency>
</dependencies>
<build>
@@ -64,19 +80,4 @@
</plugin>
</plugins>
</build>
-
- <properties>
- <maven.compiler.source>11</maven.compiler.source>
- <maven.compiler.target>11</maven.compiler.target>
- <protobuf.maven.plugin>0.5.1</protobuf.maven.plugin>
- <os.maven.plugin>1.5.0.Final</os.maven.plugin>
- <javax.annotation>1.3.2</javax.annotation>
- <h2>1.4.191</h2>
- <protobuf.java>3.10.0</protobuf.java>
- <grpc.spring.boot>4.4.3</grpc.spring.boot>
- <spring.boot.data.jpa>2.4.2</spring.boot.data.jpa>
- <log4j.over.slf4j>1.7.26</log4j.over.slf4j>
- <dozer>5.5.1</dozer>
- <mariadb.jdbc>2.5.1</mariadb.jdbc>
- </properties>
</project>
\ No newline at end of file
diff --git a/data-resource-management-service/drms-stubs/src/main/proto/Common.proto b/data-resource-management-service/drms-stubs/src/main/proto/Common.proto
index 3334881..8511c74 100644
--- a/data-resource-management-service/drms-stubs/src/main/proto/Common.proto
+++ b/data-resource-management-service/drms-stubs/src/main/proto/Common.proto
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
syntax = "proto3";
option java_multiple_files = true;
diff --git a/data-resource-management-service/drms-stubs/src/main/proto/group/Group.proto b/data-resource-management-service/drms-stubs/src/main/proto/group/Group.proto
new file mode 100644
index 0000000..4515c21
--- /dev/null
+++ b/data-resource-management-service/drms-stubs/src/main/proto/group/Group.proto
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = "proto3";
+
+option java_multiple_files = true;
+package org.apache.airavata.datalake.drms.groups;
+
+import "google/api/annotations.proto";
+
+message User {
+ string userId = 1;
+ string username = 2;
+ string firstName = 3;
+ string lastName = 4;
+ string emailAddress = 5;
+}
+
+message Group {
+ string groupId = 1;
+ string name = 2;
+ string description = 3;
+ repeated Group childGroups = 4;
+ string parentGroupId = 5;
+ repeated GroupMembership groupMembership = 6;
+}
+
+message GroupMembership {
+ User user = 1;
+ string groupId = 2;
+ string membershipType = 3;
+}
\ No newline at end of file
diff --git a/data-resource-management-service/drms-stubs/src/main/proto/group/GroupService.proto b/data-resource-management-service/drms-stubs/src/main/proto/group/GroupService.proto
new file mode 100644
index 0000000..389d4cb
--- /dev/null
+++ b/data-resource-management-service/drms-stubs/src/main/proto/group/GroupService.proto
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = "proto3";
+
+option java_multiple_files = true;
+package org.apache.airavata.datalake.drms.groups;
+
+import "google/api/annotations.proto";
+import "group/Group.proto";
+import "Common.proto";
+
+message FetchCurrentUserRequest {
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken authToken = 1;
+}
+
+message FetchCurrentUserResponse {
+ User user = 1;
+}
+
+message FetchUserRequest {
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken authToken = 1;
+ string userId = 2;
+}
+
+message FetchUserResponse {
+ User user = 1;
+}
+
+message FetchCurrentUserGroupsRequest {
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken authToken = 1;
+}
+
+message FetchCurrentUserGroupsResponse {
+ repeated Group groups = 1;
+}
+
+message FetchUserGroupsRequest {
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken authToken = 1;
+ string userId = 2;
+}
+
+message FetchUserGroupsResponse {
+ repeated Group groups = 1;
+}
+
+service GroupService {
+
+ rpc fetchCurrentUser(FetchCurrentUserRequest) returns (FetchCurrentUserResponse) {
+ option (google.api.http) = {
+ get: "/v1.0/api/currentuser"
+ };
+ }
+
+ rpc fetchUser (FetchUserRequest) returns (FetchUserResponse) {
+ option (google.api.http) = {
+ get: "/v1.0/api/user"
+ };
+ }
+
+ rpc fetchUserGroups (FetchUserGroupsRequest) returns (FetchUserGroupsResponse) {
+ option (google.api.http) = {
+ get: "/v1.0/api/usergroups"
+ };
+ }
+
+ rpc fetchCurrentUserGroups (FetchCurrentUserGroupsRequest) returns (FetchCurrentUserGroupsResponse) {
+ option (google.api.http) = {
+ get: "/v1.0/api/currentusergroups"
+ };
+ }
+}
\ No newline at end of file
diff --git a/data-resource-management-service/drms-stubs/src/main/proto/preference/S3Preference.proto b/data-resource-management-service/drms-stubs/src/main/proto/preference/S3Preference.proto
index b7eac3e..5717be6 100644
--- a/data-resource-management-service/drms-stubs/src/main/proto/preference/S3Preference.proto
+++ b/data-resource-management-service/drms-stubs/src/main/proto/preference/S3Preference.proto
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
syntax = "proto3";
option java_multiple_files = true;
diff --git a/data-resource-management-service/drms-stubs/src/main/proto/preference/SSHPreference.proto b/data-resource-management-service/drms-stubs/src/main/proto/preference/SSHPreference.proto
index 8cf20c6..08f2b3f 100644
--- a/data-resource-management-service/drms-stubs/src/main/proto/preference/SSHPreference.proto
+++ b/data-resource-management-service/drms-stubs/src/main/proto/preference/SSHPreference.proto
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
syntax = "proto3";
option java_multiple_files = true;
diff --git a/data-resource-management-service/drms-stubs/src/main/proto/preference/StoragePreferenceService.proto b/data-resource-management-service/drms-stubs/src/main/proto/preference/StoragePreferenceService.proto
index f8e296a..29e07c6 100644
--- a/data-resource-management-service/drms-stubs/src/main/proto/preference/StoragePreferenceService.proto
+++ b/data-resource-management-service/drms-stubs/src/main/proto/preference/StoragePreferenceService.proto
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
syntax = "proto3";
option java_multiple_files = true;
diff --git a/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResource.proto b/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResource.proto
index 914e2ec..adf1231 100644
--- a/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResource.proto
+++ b/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResource.proto
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
syntax = "proto3";
option java_multiple_files = true;
diff --git a/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResourceService.proto b/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResourceService.proto
index 6bd1fc7..600974a 100644
--- a/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResourceService.proto
+++ b/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResourceService.proto
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
syntax = "proto3";
option java_multiple_files = true;
diff --git a/data-resource-management-service/drms-stubs/src/main/proto/sharing/Sharing.proto b/data-resource-management-service/drms-stubs/src/main/proto/sharing/Sharing.proto
new file mode 100644
index 0000000..17517f3
--- /dev/null
+++ b/data-resource-management-service/drms-stubs/src/main/proto/sharing/Sharing.proto
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = "proto3";
+
+option java_multiple_files = true;
+package org.apache.airavata.datalake.drms.sharing;
+
+import "google/api/annotations.proto";
+
+message ShareStorageRequest {
+ string storageId = 1;
+ string groupId = 2;
+}
\ No newline at end of file
diff --git a/data-resource-management-service/drms-stubs/src/main/proto/storage/S3Storage.proto b/data-resource-management-service/drms-stubs/src/main/proto/storage/S3Storage.proto
index 58d4768..93b2a00 100644
--- a/data-resource-management-service/drms-stubs/src/main/proto/storage/S3Storage.proto
+++ b/data-resource-management-service/drms-stubs/src/main/proto/storage/S3Storage.proto
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
syntax = "proto3";
option java_multiple_files = true;
diff --git a/data-resource-management-service/drms-stubs/src/main/proto/storage/SSHStorage.proto b/data-resource-management-service/drms-stubs/src/main/proto/storage/SSHStorage.proto
index d77e026..d7d8bb4 100644
--- a/data-resource-management-service/drms-stubs/src/main/proto/storage/SSHStorage.proto
+++ b/data-resource-management-service/drms-stubs/src/main/proto/storage/SSHStorage.proto
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
syntax = "proto3";
option java_multiple_files = true;
diff --git a/data-resource-management-service/drms-stubs/src/main/proto/storage/StorageService.proto b/data-resource-management-service/drms-stubs/src/main/proto/storage/StorageService.proto
index 891962a..fca430a 100644
--- a/data-resource-management-service/drms-stubs/src/main/proto/storage/StorageService.proto
+++ b/data-resource-management-service/drms-stubs/src/main/proto/storage/StorageService.proto
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
syntax = "proto3";
option java_multiple_files = true;
@@ -64,6 +81,13 @@ message StorageSearchResponse {
repeated AnyStorage storages = 2;
}
+message AddStorageMetadataRequest {
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken authToken = 1;
+ string storageId = 2;
+ string key = 3;
+ string value = 4;
+}
+
service StorageService {
rpc fetchStorage (StorageFetchRequest) returns (StorageFetchResponse) {
@@ -95,4 +119,10 @@ service StorageService {
post: "/v1.0/api/drms/storage/search"
};
}
+
+ rpc addStorageMetadata(AddStorageMetadataRequest) returns (google.protobuf.Empty) {
+ option (google.api.http) = {
+ post: "/v1.0/api/drms/storage/metadata"
+ };
+ }
}
\ No newline at end of file
diff --git a/data-resource-management-service/pom.xml b/data-resource-management-service/pom.xml
index d05865a..d1c7685 100644
--- a/data-resource-management-service/pom.xml
+++ b/data-resource-management-service/pom.xml
@@ -1,4 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@@ -12,10 +28,18 @@
<artifactId>data-resource-management-service</artifactId>
<packaging>pom</packaging>
<modules>
- <module>drms-api</module>
- <module>drms-core</module>
<module>drms-stubs</module>
+ <module>drms-core</module>
+ <module>drms-api</module>
</modules>
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <version>${log4j.over.slf4j}</version>
+ </dependency>
+ </dependencies>
+
</project>
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index bb2d693..96bf525 100644
--- a/pom.xml
+++ b/pom.xml
@@ -94,6 +94,7 @@
<grpc.spring.boot>3.5.1</grpc.spring.boot>
<spring.boot.data.jpa>2.2.1.RELEASE</spring.boot.data.jpa>
<dozer>5.5.1</dozer>
+ <grpc.services>1.25.0</grpc.services>
<!-- Spring dependencies -->
<spring.boot.version>2.4.2</spring.boot.version>
@@ -146,7 +147,10 @@
<com.codahale.version>0.7.0</com.codahale.version>
+ <neo4j.ogm.version>3.2.20</neo4j.ogm.version>
+ <neo4j.version>3.4.6</neo4j.version>
+ <log4j.over.slf4j>1.7.26</log4j.over.slf4j>
</properties>
</project>