You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2020/08/16 07:44:33 UTC
[airavata-mft] branch master updated: Fetching child files and
directories of a parent resource through metadata api
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
The following commit(s) were added to refs/heads/master by this push:
new 2b4539d Fetching child files and directories of a parent resource through metadata api
2b4539d is described below
commit 2b4539dfc5799523112c33b3977635a31c23a08c
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Sun Aug 16 03:44:20 2020 -0400
Fetching child files and directories of a parent resource through metadata api
---
.../mft/admin/models/rpc/SyncRPCRequest.java | 7 ----
.../apache/airavata/mft/agent/rpc/RPCParser.java | 30 +++++++++++++++
.../airavata/mft/api/handler/MFTApiHandler.java | 45 ++++++++++++++++++----
api/stub/src/main/proto/MFTApi.proto | 3 +-
.../airavata/mft/examples/metadata/SCPExample.java | 32 +++++++++++++++
5 files changed, 101 insertions(+), 16 deletions(-)
diff --git a/admin/src/main/java/org/apache/airavata/mft/admin/models/rpc/SyncRPCRequest.java b/admin/src/main/java/org/apache/airavata/mft/admin/models/rpc/SyncRPCRequest.java
index d7933e4..1aadc83 100644
--- a/admin/src/main/java/org/apache/airavata/mft/admin/models/rpc/SyncRPCRequest.java
+++ b/admin/src/main/java/org/apache/airavata/mft/admin/models/rpc/SyncRPCRequest.java
@@ -76,7 +76,6 @@ public class SyncRPCRequest {
private String agentId;
private String method;
private Map<String, String> parameters = new HashMap<>();
- private String returnAddress;
private String messageId;
private SyncRPCRequestBuilder() {
@@ -101,11 +100,6 @@ public class SyncRPCRequest {
return this;
}
- public SyncRPCRequestBuilder withReturnAddress(String returnAddress) {
- this.returnAddress = returnAddress;
- return this;
- }
-
public SyncRPCRequestBuilder withMessageId(String messageId) {
this.messageId = messageId;
return this;
@@ -116,7 +110,6 @@ public class SyncRPCRequest {
syncRPCRequest.setAgentId(agentId);
syncRPCRequest.setMethod(method);
syncRPCRequest.setParameters(parameters);
- syncRPCRequest.setReturnAddress(returnAddress);
syncRPCRequest.setMessageId(messageId);
return syncRPCRequest;
}
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java b/agent/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java
index e4d5deb..f136704 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java
@@ -64,6 +64,21 @@ public class RPCParser {
return mapper.writeValueAsString(fileResourceMetadata);
}
break;
+ case "getChildFileResourceMetadata":
+ resourceId = request.getParameters().get("resourceId");
+ resourceType = request.getParameters().get("resourceType");
+ resourceToken = request.getParameters().get("resourceToken");
+ String childPath = request.getParameters().get("childPath");
+ mftAuthorizationToken = request.getParameters().get("mftAuthorizationToken");
+
+ metadataCollectorOp = MetadataCollectorResolver.resolveMetadataCollector(resourceType);
+ if (metadataCollectorOp.isPresent()) {
+ MetadataCollector metadataCollector = metadataCollectorOp.get();
+ metadataCollector.init(resourceServiceHost, resourceServicePort, secretServiceHost, secretServicePort);
+ FileResourceMetadata fileResourceMetadata = metadataCollector.getFileResourceMetadata(resourceId, childPath, resourceToken);
+ return mapper.writeValueAsString(fileResourceMetadata);
+ }
+ break;
case "getDirectoryResourceMetadata":
resourceId = request.getParameters().get("resourceId");
resourceType = request.getParameters().get("resourceType");
@@ -78,6 +93,21 @@ public class RPCParser {
return mapper.writeValueAsString(dirResourceMetadata);
}
break;
+ case "getChildDirectoryResourceMetadata":
+ resourceId = request.getParameters().get("resourceId");
+ resourceType = request.getParameters().get("resourceType");
+ resourceToken = request.getParameters().get("resourceToken");
+ childPath = request.getParameters().get("childPath");
+ mftAuthorizationToken = request.getParameters().get("mftAuthorizationToken");
+
+ metadataCollectorOp = MetadataCollectorResolver.resolveMetadataCollector(resourceType);
+ if (metadataCollectorOp.isPresent()) {
+ MetadataCollector metadataCollector = metadataCollectorOp.get();
+ metadataCollector.init(resourceServiceHost, resourceServicePort, secretServiceHost, secretServicePort);
+ DirectoryResourceMetadata dirResourceMetadata = metadataCollector.getDirectoryResourceMetadata(resourceId, childPath, resourceToken);
+ return mapper.writeValueAsString(dirResourceMetadata);
+ }
+ break;
}
logger.error("Unknown method type specified {}", request.getMethod());
throw new Exception("Unknown method " + request.getMethod());
diff --git a/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java b/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java
index 21f85bb..108bd30 100644
--- a/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java
+++ b/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java
@@ -144,19 +144,33 @@ public class MFTApiHandler extends MFTApiServiceGrpc.MFTApiServiceImplBase {
}
}
+ /**
+ * Fetches metadata for a specified file resource. This has 2 modes
+ * 1. Fetch the metadata of the exact file pointed in the resourceId. This assumes resourceId is an id of a file resource
+ * 2. Fetch the metadata of a child directory in the parent directory. To do this, childPath should be provided explicitly
+ * and resourceId is the parent resource id. Here resource id should be an id of directory resource
+ */
@Override
public void getFileResourceMetadata(FetchResourceMetadataRequest request, StreamObserver<FileMetadataResponse> responseObserver) {
try {
- SyncRPCResponse rpcResponse = agentRPCClient.sendSyncRequest(SyncRPCRequest.SyncRPCRequestBuilder.builder()
+ SyncRPCRequest.SyncRPCRequestBuilder requestBuilder = SyncRPCRequest.SyncRPCRequestBuilder.builder()
.withAgentId(request.getTargetAgentId())
.withMessageId(UUID.randomUUID().toString())
- .withMethod("getFileResourceMetadata")
.withParameter("resourceId", request.getResourceId())
.withParameter("resourceType", request.getResourceType())
.withParameter("resourceToken", request.getResourceToken())
- .withParameter("mftAuthorizationToken", request.getMftAuthorizationToken())
- .build());
+ .withParameter("mftAuthorizationToken", request.getMftAuthorizationToken());
+
+ if (request.getChildPath().isEmpty()) {
+ requestBuilder.withMethod("getFileResourceMetadata");
+ } else {
+ // If a childPath is specified, look for child directories in the given parent resource id
+ requestBuilder.withMethod("getChildFileResourceMetadata");
+ requestBuilder.withParameter("childPath", request.getChildPath());
+ }
+
+ SyncRPCResponse rpcResponse = agentRPCClient.sendSyncRequest(requestBuilder.build());
switch (rpcResponse.getResponseStatus()) {
case SUCCESS:
@@ -178,18 +192,33 @@ public class MFTApiHandler extends MFTApiServiceGrpc.MFTApiServiceImplBase {
}
}
+ /**
+ * Fetches metadata for a specified directory resource. This method assumes that the resourceId is an id of
+ * a directory resource. This has 2 modes
+ * 1. Fetch the metadata of the exact directory pointed in the resourceId
+ * 2. Fetch the metadata of a child directory in the parent directory. To do this, childPath should be provided explicitly
+ * and resourceId is the parent resource id.
+ */
@Override
public void getDirectoryResourceMetadata(FetchResourceMetadataRequest request, StreamObserver<DirectoryMetadataResponse> responseObserver) {
try {
- SyncRPCResponse rpcResponse = agentRPCClient.sendSyncRequest(SyncRPCRequest.SyncRPCRequestBuilder.builder()
+ SyncRPCRequest.SyncRPCRequestBuilder requestBuilder = SyncRPCRequest.SyncRPCRequestBuilder.builder()
.withAgentId(request.getTargetAgentId())
.withMessageId(UUID.randomUUID().toString())
- .withMethod("getDirectoryResourceMetadata")
.withParameter("resourceId", request.getResourceId())
.withParameter("resourceType", request.getResourceType())
.withParameter("resourceToken", request.getResourceToken())
- .withParameter("mftAuthorizationToken", request.getMftAuthorizationToken())
- .build());
+ .withParameter("mftAuthorizationToken", request.getMftAuthorizationToken());
+
+ if (request.getChildPath().isEmpty()) {
+ requestBuilder.withMethod("getDirectoryResourceMetadata");
+ } else {
+ // If a childPath is specified, look for child directories in the given parent resource id
+ requestBuilder.withMethod("getChildDirectoryResourceMetadata");
+ requestBuilder.withParameter("childPath", request.getChildPath());
+ }
+
+ SyncRPCResponse rpcResponse = agentRPCClient.sendSyncRequest(requestBuilder.build());
switch (rpcResponse.getResponseStatus()) {
case SUCCESS:
diff --git a/api/stub/src/main/proto/MFTApi.proto b/api/stub/src/main/proto/MFTApi.proto
index c41bc4b..dc003ef 100644
--- a/api/stub/src/main/proto/MFTApi.proto
+++ b/api/stub/src/main/proto/MFTApi.proto
@@ -81,7 +81,8 @@ message FetchResourceMetadataRequest {
string resourceBackend = 4;
string resourceCredentialBackend = 5;
string targetAgentId = 7;
- string mftAuthorizationToken = 8;
+ string childPath = 8; // if the child entities of the parent resource are required, set this field
+ string mftAuthorizationToken = 9;
}
service MFTApiService {
diff --git a/examples/src/main/java/org/apache/airavata/mft/examples/metadata/SCPExample.java b/examples/src/main/java/org/apache/airavata/mft/examples/metadata/SCPExample.java
index 4d4e0a7..3cf2ab0 100644
--- a/examples/src/main/java/org/apache/airavata/mft/examples/metadata/SCPExample.java
+++ b/examples/src/main/java/org/apache/airavata/mft/examples/metadata/SCPExample.java
@@ -10,6 +10,7 @@ public class SCPExample {
public static void main(String args[]) throws Exception {
MFTApiServiceGrpc.MFTApiServiceBlockingStub client = MFTApiClient.buildClient("localhost", 7004);
+ // File metadata
long startTime = System.currentTimeMillis();
FileMetadataResponse fileResourceMetadata = client.getFileResourceMetadata(FetchResourceMetadataRequest.newBuilder()
.setResourceId("remote-ssh-resource2")
@@ -22,6 +23,7 @@ public class SCPExample {
System.out.println(fileResourceMetadata);
System.out.println("Time for processing : " + (endTime - startTime) + " ms");
+ // Directory metadata
startTime = System.currentTimeMillis();
DirectoryMetadataResponse directoryMetadataResponse = client.getDirectoryResourceMetadata(FetchResourceMetadataRequest.newBuilder()
.setResourceId("remote-ssh-dir-resource")
@@ -34,5 +36,35 @@ public class SCPExample {
System.out.println("Directory metadata response ");
System.out.println(directoryMetadataResponse);
System.out.println("Time for processing : " + (endTime - startTime) + " ms");
+
+ // Child file inside parent directoru
+ startTime = System.currentTimeMillis();
+ fileResourceMetadata = client.getFileResourceMetadata(FetchResourceMetadataRequest.newBuilder()
+ .setResourceId("remote-ssh-dir-resource") // Parent directory resource id
+ .setResourceType("SCP")
+ .setResourceToken("local-ssh-cred")
+ .setTargetAgentId("agent0")
+ .setChildPath("/tmp/10mb.txt") // Child file path
+ .build());
+ endTime = System.currentTimeMillis();
+ System.out.println("Child file metadata response ");
+ System.out.println(fileResourceMetadata);
+ System.out.println("Time for processing : " + (endTime - startTime) + " ms");
+
+ // Child directory inside parent directoru
+ startTime = System.currentTimeMillis();
+ directoryMetadataResponse = client.getDirectoryResourceMetadata(FetchResourceMetadataRequest.newBuilder()
+ .setResourceId("remote-ssh-dir-resource") // Parent directory resource id
+ .setResourceType("SCP")
+ .setResourceToken("local-ssh-cred")
+ .setTargetAgentId("agent0")
+ .setChildPath("/tmp/hsperfdata_root") // Child directory path
+ .build());
+ endTime = System.currentTimeMillis();
+
+ System.out.println("Child directory metadata response ");
+ System.out.println(directoryMetadataResponse);
+ System.out.println("Time for processing : " + (endTime - startTime) + " ms");
+
}
}