You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2023/09/01 18:11:00 UTC
[airavata-mft] branch master updated (54be23e -> d8cf348)
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
from 54be23e Supporting storage removal from cli
new f4acebe Avoiding writing additional byte in SwiftIncomingConnector
new d8cf348 Minor optimizations
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/airavata/mft/agent/TransportMediator.java | 9 ++++++---
.../apache/airavata/mft/transport/s3/S3OutgoingConnector.java | 8 ++++----
.../airavata/mft/transport/swift/SwiftIncomingConnector.java | 4 ++--
3 files changed, 12 insertions(+), 9 deletions(-)
[airavata-mft] 01/02: Avoiding writing additional byte in SwiftIncomingConnector
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit f4acebe3e6ab0b7a4987f33fe84345b0867c7aef
Author: DImuthuUpe <di...@gmail.com>
AuthorDate: Fri Sep 1 23:39:40 2023 +0530
Avoiding writing additional byte in SwiftIncomingConnector
---
.../apache/airavata/mft/transport/swift/SwiftIncomingConnector.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java
index 4c9ab88..aa2631f 100644
--- a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java
+++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java
@@ -66,7 +66,7 @@ public class SwiftIncomingConnector implements IncomingChunkedConnector {
public void downloadChunk(int chunkId, long startByte, long endByte, String downloadFile) throws Exception {
SwiftObject swiftObject = objectApi.get(
resourcePath,
- GetOptions.Builder.range(startByte, endByte));
+ GetOptions.Builder.range(startByte, endByte - 1));
InputStream inputStream = swiftObject.getPayload().openStream();
@@ -85,7 +85,7 @@ public class SwiftIncomingConnector implements IncomingChunkedConnector {
SwiftObject swiftObject = objectApi.get(
resourcePath,
- GetOptions.Builder.range(startByte, endByte));
+ GetOptions.Builder.range(startByte, endByte - 1));
return swiftObject.getPayload().openStream();
}
[airavata-mft] 02/02: Minor optimizations
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit d8cf348886735fe92d16f1c0b2c5d83e7741ce5a
Author: DImuthuUpe <di...@gmail.com>
AuthorDate: Fri Sep 1 23:40:51 2023 +0530
Minor optimizations
---
.../java/org/apache/airavata/mft/agent/TransportMediator.java | 9 ++++++---
.../apache/airavata/mft/transport/s3/S3OutgoingConnector.java | 8 ++++----
2 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/agent/service/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java b/agent/service/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java
index 42c0630..0f55b4a 100644
--- a/agent/service/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java
+++ b/agent/service/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java
@@ -280,9 +280,12 @@ public class TransportMediator {
uploader.uploadChunk(chunkIdx, startPos, endPos, inputStream);
} else {
String tempFile = tempDataDir + File.separator + UUID.randomUUID().toString() + "-" + chunkIdx;
- downloader.downloadChunk(chunkIdx, startPos, endPos, tempFile);
- uploader.uploadChunk(chunkIdx, startPos, endPos, tempFile);
- new File(tempFile).delete();
+ try {
+ downloader.downloadChunk(chunkIdx, startPos, endPos, tempFile);
+ uploader.uploadChunk(chunkIdx, startPos, endPos, tempFile);
+ } finally {
+ new File(tempFile).delete();
+ }
}
return chunkIdx;
} catch (Exception e) {
diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java
index 6cc17d4..cc3d6d2 100644
--- a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java
+++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java
@@ -92,10 +92,10 @@ public class S3OutgoingConnector implements OutgoingChunkedConnector {
.withUploadId(initResponse.getUploadId())
.withPartNumber(chunkId + 1)
.withFileOffset(0)
- //.withMD5Digest(Md5Utils.md5AsBase64(new File(uploadFile)))
-// .withFile(file)
- .withInputStream(new BufferedInputStream(new FileInputStream(file), Math.min(16 * 1024 * 1024, (int) ( endByte - startByte))))
- .withPartSize(file.length());
+ .withFile(file)
+ .withMD5Digest(Md5Utils.md5AsBase64(new File(uploadFile)))
+ //.withInputStream(new BufferedInputStream(new FileInputStream(file), Math.min(16 * 1024 * 1024, (int) ( endByte - startByte))))
+ .withPartSize(endByte - startByte);
UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest);
this.partETags.add(uploadResult.getPartETag());