You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2023/03/18 20:58:48 UTC
[airavata-mft] 04/05: S3 Outgoing bufferStreaming update
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit 29c05e15b5f2c9c57ad916a1de214c2a7d99f9fc
Author: Praneeth Chityala <pr...@gmail.com>
AuthorDate: Sat Mar 18 01:36:59 2023 -0400
S3 Outgoing bufferStreaming update
---
.../org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java
index 4803b32..6dfa022 100644
--- a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java
+++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java
@@ -36,7 +36,9 @@ import org.apache.airavata.mft.resource.stubs.s3.storage.S3StorageGetRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.BufferedInputStream;
import java.io.File;
+import java.io.FileInputStream;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
@@ -61,7 +63,6 @@ public class S3OutgoingConnector implements OutgoingChunkedConnector {
public void init(ConnectorConfig cc) throws Exception {
this.resourcePath = cc.getResourcePath();
- this.resourceLength = cc.getMetadata().getFile().getResourceSize();
s3Storage = cc.getStorage().getS3();
@@ -92,7 +93,8 @@ public class S3OutgoingConnector implements OutgoingChunkedConnector {
.withPartNumber(chunkId + 1)
.withFileOffset(0)
//.withMD5Digest(Md5Utils.md5AsBase64(new File(uploadFile)))
- .withFile(file)
+// .withFile(file)
+ .withInputStream(new BufferedInputStream(new FileInputStream(file), Math.min(16 * 1024 * 1024, (int) ( endByte - startByte))))
.withPartSize(file.length());
UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest);
@@ -137,6 +139,6 @@ public class S3OutgoingConnector implements OutgoingChunkedConnector {
@Override
public void failed() throws Exception {
-
+ logger.error("S3 failed to upload chunk to bucket {} for resource path {}", s3Storage.getBucketName(), resourcePath);
}
}