You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by ac...@apache.org on 2020/03/18 11:30:05 UTC
[camel] branch master updated (8123df2 -> 3bb388a)
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git.
from 8123df2 Upgrade apache solr
new 2846fc0 CAMEL-14663 - Camel-AWS2 S3: Add support for multipart upload
new 4bbc8a2 CAMEL-14663 - Camel-AWS2 S3: Add support for multipart upload, set correct logging level
new 0f3c279 CAMEL-14663 - Camel-AWS2 S3: Add support for multipart upload, removed useless println
new bd7dad1 CAMEL-14663 - Camel-AWS2 S3: Add support for multipart upload, fixed CS
new 3bb388a CAMEL-14663 - Camel-AWS2 S3: Add support for multipart upload, added an integration test
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../camel/component/aws2/s3/AWS2S3Producer.java | 111 ++++++++++++++++++++-
...S3MultipartUploadOperationIntegrationTest.java} | 25 ++---
.../camel-aws2-s3/src/test/resources/empty.txt | Bin 0 -> 5242880 bytes
3 files changed, 118 insertions(+), 18 deletions(-)
copy components/camel-aws2-s3/src/test/java/org/apache/camel/component/aws2/s3/integration/{S3DeleteBucketOperationIntegrationTest.java => S3MultipartUploadOperationIntegrationTest.java} (73%)
create mode 100644 components/camel-aws2-s3/src/test/resources/empty.txt
[camel] 04/05: CAMEL-14663 - Camel-AWS2 S3: Add support for
multipart upload, fixed CS
Posted by ac...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git
commit bd7dad1441e7a32396580ea29e5d2363f1e1904b
Author: Andrea Cosentino <an...@gmail.com>
AuthorDate: Wed Mar 18 12:23:48 2020 +0100
CAMEL-14663 - Camel-AWS2 S3: Add support for multipart upload, fixed CS
---
.../java/org/apache/camel/component/aws2/s3/AWS2S3Producer.java | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/components/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Producer.java b/components/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Producer.java
index d02135c..3f05e19 100644
--- a/components/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Producer.java
+++ b/components/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Producer.java
@@ -38,7 +38,6 @@ import org.apache.camel.util.ObjectHelper;
import org.apache.camel.util.URISupport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.core.sync.RequestBody;
@@ -134,9 +133,9 @@ public class AWS2S3Producer extends DefaultProducer {
Map<String, String> objectMetadata = determineMetadata(exchange);
if (objectMetadata.containsKey("Content-Length")) {
- if (objectMetadata.get("Content-Length").equalsIgnoreCase("0")) {
- objectMetadata.put("Content-Length", String.valueOf(filePayload.length()));
- }
+ if (objectMetadata.get("Content-Length").equalsIgnoreCase("0")) {
+ objectMetadata.put("Content-Length", String.valueOf(filePayload.length()));
+ }
} else {
objectMetadata.put("Content-Length", String.valueOf(filePayload.length()));
}
[camel] 02/05: CAMEL-14663 - Camel-AWS2 S3: Add support for
multipart upload, set correct logging level
Posted by ac...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 4bbc8a2117e0e18775ac9da7aa962fce4e61386a
Author: Andrea Cosentino <an...@gmail.com>
AuthorDate: Wed Mar 18 12:14:11 2020 +0100
CAMEL-14663 - Camel-AWS2 S3: Add support for multipart upload, set correct logging level
---
components/camel-aws2-s3/src/test/resources/log4j2.properties | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/components/camel-aws2-s3/src/test/resources/log4j2.properties b/components/camel-aws2-s3/src/test/resources/log4j2.properties
index e33abee..a287c66 100644
--- a/components/camel-aws2-s3/src/test/resources/log4j2.properties
+++ b/components/camel-aws2-s3/src/test/resources/log4j2.properties
@@ -24,5 +24,5 @@ appender.out.type = Console
appender.out.name = out
appender.out.layout.type = PatternLayout
appender.out.layout.pattern = %d [%-15.15t] %-5p %-30.30c{1} - %m%n
-rootLogger.level = DEBUG
+rootLogger.level = INFO
rootLogger.appenderRef.file.ref = file
[camel] 01/05: CAMEL-14663 - Camel-AWS2 S3: Add support for
multipart upload
Posted by ac...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 2846fc08880c1316acc20e8b3df9f2e1019bd54d
Author: Andrea Cosentino <an...@gmail.com>
AuthorDate: Wed Mar 18 12:13:41 2020 +0100
CAMEL-14663 - Camel-AWS2 S3: Add support for multipart upload
---
.../camel/component/aws2/s3/AWS2S3Producer.java | 114 ++++++++++++++++++++-
.../src/test/resources/log4j2.properties | 2 +-
2 files changed, 114 insertions(+), 2 deletions(-)
diff --git a/components/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Producer.java b/components/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Producer.java
index a0d1fa2..71d2640 100644
--- a/components/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Producer.java
+++ b/components/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Producer.java
@@ -22,7 +22,9 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.camel.Endpoint;
@@ -36,14 +38,22 @@ import org.apache.camel.util.ObjectHelper;
import org.apache.camel.util.URISupport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.core.sync.ResponseTransformer;
import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.BucketCannedACL;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
import software.amazon.awssdk.services.s3.model.CopyObjectResponse;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
import software.amazon.awssdk.services.s3.model.DeleteBucketResponse;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
@@ -55,6 +65,7 @@ import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
import software.amazon.awssdk.services.s3.model.ObjectCannedACL;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
/**
* A Producer which sends messages to the Amazon Web Service Simple Storage
@@ -74,7 +85,11 @@ public class AWS2S3Producer extends DefaultProducer {
public void process(final Exchange exchange) throws Exception {
AWS2S3Operations operation = determineOperation(exchange);
if (ObjectHelper.isEmpty(operation)) {
- processSingleOp(exchange);
+ if (getConfiguration().isMultiPartUpload()) {
+ processMultiPart(exchange);
+ } else {
+ processSingleOp(exchange);
+ }
} else {
switch (operation) {
case copyObject:
@@ -103,6 +118,103 @@ public class AWS2S3Producer extends DefaultProducer {
}
}
}
+
+ public void processMultiPart(final Exchange exchange) throws Exception {
+ File filePayload = null;
+ Object obj = exchange.getIn().getMandatoryBody();
+ // Need to check if the message body is WrappedFile
+ if (obj instanceof WrappedFile) {
+ obj = ((WrappedFile<?>)obj).getFile();
+ }
+ if (obj instanceof File) {
+ filePayload = (File)obj;
+ } else {
+ throw new IllegalArgumentException("aws-s3: MultiPart upload requires a File input.");
+ }
+
+ Map<String, String> objectMetadata = determineMetadata(exchange);
+ if (objectMetadata.containsKey("Content-Length")) {
+ if (objectMetadata.get("Content-Length").equalsIgnoreCase("0")) {
+ objectMetadata.put("Content-Length", String.valueOf(filePayload.length()));
+ }
+ } else {
+ objectMetadata.put("Content-Length", String.valueOf(filePayload.length()));
+ }
+
+ final String keyName = determineKey(exchange);
+ CreateMultipartUploadRequest.Builder createMultipartUploadRequest = CreateMultipartUploadRequest.builder()
+ .bucket(getConfiguration().getBucketName()).key(keyName);
+
+ String storageClass = determineStorageClass(exchange);
+ if (storageClass != null) {
+ createMultipartUploadRequest.storageClass(storageClass);
+ }
+
+ String cannedAcl = exchange.getIn().getHeader(AWS2S3Constants.CANNED_ACL, String.class);
+ if (cannedAcl != null) {
+ ObjectCannedACL objectAcl = ObjectCannedACL.valueOf(cannedAcl);
+ createMultipartUploadRequest.acl(objectAcl);
+ }
+
+ BucketCannedACL acl = exchange.getIn().getHeader(AWS2S3Constants.ACL, BucketCannedACL.class);
+ if (acl != null) {
+ // note: if cannedacl and acl are both specified the last one will
+ // be used. refer to
+ // PutObjectRequest#setAccessControlList for more details
+ createMultipartUploadRequest.acl(acl.toString());
+ }
+
+ if (getConfiguration().isUseAwsKMS()) {
+ createMultipartUploadRequest.ssekmsKeyId(getConfiguration().getAwsKMSKeyId());
+ }
+
+ LOG.trace("Initiating multipart upload [{}] from exchange [{}]...", createMultipartUploadRequest, exchange);
+
+ CreateMultipartUploadResponse initResponse = getEndpoint().getS3Client().createMultipartUpload(createMultipartUploadRequest.build());
+ final long contentLength = Long.valueOf(objectMetadata.get("Content-Length"));
+ final List<String> partETags = new ArrayList<>();
+ List<CompletedPart> completedParts = new ArrayList<CompletedPart>();
+ long partSize = getConfiguration().getPartSize();
+ CompleteMultipartUploadResponse uploadResult = null;
+
+ long filePosition = 0;
+
+ try {
+ for (int part = 1; filePosition < contentLength; part++) {
+ System.err.println("PART! " + part);
+ partSize = Math.min(partSize, contentLength - filePosition);
+
+ UploadPartRequest uploadRequest = UploadPartRequest.builder().bucket(getConfiguration().getBucketName()).key(keyName)
+ .uploadId(initResponse.uploadId()).partNumber(part).build();
+
+ LOG.trace("Uploading part [{}] for {}", part, keyName);
+ String etag = getEndpoint().getS3Client().uploadPart(uploadRequest, RequestBody.fromFile(filePayload)).eTag();
+ partETags.add(etag);
+ CompletedPart partUpload = CompletedPart.builder().partNumber(part).eTag(etag).build();
+ completedParts.add(partUpload);
+ filePosition += partSize;
+ System.err.println(filePosition);
+ }
+ CompletedMultipartUpload completeMultipartUpload = CompletedMultipartUpload.builder().parts(completedParts).build();
+ CompleteMultipartUploadRequest compRequest = CompleteMultipartUploadRequest.builder().multipartUpload(completeMultipartUpload).bucket(getConfiguration().getBucketName()).key(keyName).uploadId(initResponse.uploadId()).build();
+
+ uploadResult = getEndpoint().getS3Client().completeMultipartUpload(compRequest);
+
+ } catch (Exception e) {
+ getEndpoint().getS3Client().abortMultipartUpload(AbortMultipartUploadRequest.builder().bucket(getConfiguration().getBucketName()).key(keyName).uploadId(initResponse.uploadId()).build());
+ throw e;
+ }
+
+ Message message = getMessageForResponse(exchange);
+ message.setHeader(AWS2S3Constants.E_TAG, uploadResult.eTag());
+ if (uploadResult.versionId() != null) {
+ message.setHeader(AWS2S3Constants.VERSION_ID, uploadResult.versionId());
+ }
+
+ if (getConfiguration().isDeleteAfterWrite()) {
+ FileUtil.deleteFile(filePayload);
+ }
+ }
public void processSingleOp(final Exchange exchange) throws Exception {
diff --git a/components/camel-aws2-s3/src/test/resources/log4j2.properties b/components/camel-aws2-s3/src/test/resources/log4j2.properties
index a287c66..e33abee 100644
--- a/components/camel-aws2-s3/src/test/resources/log4j2.properties
+++ b/components/camel-aws2-s3/src/test/resources/log4j2.properties
@@ -24,5 +24,5 @@ appender.out.type = Console
appender.out.name = out
appender.out.layout.type = PatternLayout
appender.out.layout.pattern = %d [%-15.15t] %-5p %-30.30c{1} - %m%n
-rootLogger.level = INFO
+rootLogger.level = DEBUG
rootLogger.appenderRef.file.ref = file
[camel] 03/05: CAMEL-14663 - Camel-AWS2 S3: Add support for
multipart upload, removed useless println
Posted by ac...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 0f3c2792d369c9eed788f87693ac1f9cb50e1a41
Author: Andrea Cosentino <an...@gmail.com>
AuthorDate: Wed Mar 18 12:21:33 2020 +0100
CAMEL-14663 - Camel-AWS2 S3: Add support for multipart upload, removed useless println
---
.../main/java/org/apache/camel/component/aws2/s3/AWS2S3Producer.java | 2 --
1 file changed, 2 deletions(-)
diff --git a/components/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Producer.java b/components/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Producer.java
index 71d2640..d02135c 100644
--- a/components/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Producer.java
+++ b/components/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Producer.java
@@ -181,7 +181,6 @@ public class AWS2S3Producer extends DefaultProducer {
try {
for (int part = 1; filePosition < contentLength; part++) {
- System.err.println("PART! " + part);
partSize = Math.min(partSize, contentLength - filePosition);
UploadPartRequest uploadRequest = UploadPartRequest.builder().bucket(getConfiguration().getBucketName()).key(keyName)
@@ -193,7 +192,6 @@ public class AWS2S3Producer extends DefaultProducer {
CompletedPart partUpload = CompletedPart.builder().partNumber(part).eTag(etag).build();
completedParts.add(partUpload);
filePosition += partSize;
- System.err.println(filePosition);
}
CompletedMultipartUpload completeMultipartUpload = CompletedMultipartUpload.builder().parts(completedParts).build();
CompleteMultipartUploadRequest compRequest = CompleteMultipartUploadRequest.builder().multipartUpload(completeMultipartUpload).bucket(getConfiguration().getBucketName()).key(keyName).uploadId(initResponse.uploadId()).build();
[camel] 05/05: CAMEL-14663 - Camel-AWS2 S3: Add support for
multipart upload, added an integration test
Posted by ac...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 3bb388a9c3fc3a114dc309e5183cb291b422e95c
Author: Andrea Cosentino <an...@gmail.com>
AuthorDate: Wed Mar 18 12:27:21 2020 +0100
CAMEL-14663 - Camel-AWS2 S3: Add support for multipart upload, added an integration test
---
.../S3MultipartUploadOperationIntegrationTest.java | 77 +++++++++++++++++++++
.../camel-aws2-s3/src/test/resources/empty.txt | Bin 0 -> 5242880 bytes
2 files changed, 77 insertions(+)
diff --git a/components/camel-aws2-s3/src/test/java/org/apache/camel/component/aws2/s3/integration/S3MultipartUploadOperationIntegrationTest.java b/components/camel-aws2-s3/src/test/java/org/apache/camel/component/aws2/s3/integration/S3MultipartUploadOperationIntegrationTest.java
new file mode 100644
index 0000000..aed2710
--- /dev/null
+++ b/components/camel-aws2-s3/src/test/java/org/apache/camel/component/aws2/s3/integration/S3MultipartUploadOperationIntegrationTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.aws2.s3.integration;
+
+import java.io.File;
+
+import org.apache.camel.BindToRegistry;
+import org.apache.camel.EndpointInject;
+import org.apache.camel.Exchange;
+import org.apache.camel.Processor;
+import org.apache.camel.ProducerTemplate;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.aws2.s3.AWS2S3Constants;
+import org.apache.camel.component.mock.MockEndpoint;
+import org.apache.camel.test.junit4.CamelTestSupport;
+import org.junit.Ignore;
+import org.junit.Test;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+
+@Ignore("Must be manually tested. Provide your own accessKey and secretKey!")
+public class S3MultipartUploadOperationIntegrationTest extends CamelTestSupport {
+
+ @BindToRegistry("amazonS3Client")
+ S3Client client = S3Client.builder().credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("xxx", "yyy"))).region(Region.EU_WEST_1).build();
+
+ @EndpointInject
+ private ProducerTemplate template;
+
+ @EndpointInject("mock:result")
+ private MockEndpoint result;
+
+ @Test
+ public void sendIn() throws Exception {
+ result.expectedMessageCount(1);
+
+ template.send("direct:putObject", new Processor() {
+
+ @Override
+ public void process(Exchange exchange) throws Exception {
+ exchange.getIn().setHeader(AWS2S3Constants.KEY, "empty.txt");
+ exchange.getIn().setBody(new File("src/test/resources/empty.txt"));
+ }
+ });
+
+ assertMockEndpointsSatisfied();
+ }
+
+ @Override
+ protected RouteBuilder createRouteBuilder() throws Exception {
+ return new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ String awsEndpoint = "aws2-s3://mycamel?multiPartUpload=true&autoCreateBucket=false&partSize=1048576";
+
+ from("direct:putObject").to(awsEndpoint).to("mock:result");
+
+ }
+ };
+ }
+}
diff --git a/components/camel-aws2-s3/src/test/resources/empty.txt b/components/camel-aws2-s3/src/test/resources/empty.txt
new file mode 100644
index 0000000..3995316
Binary files /dev/null and b/components/camel-aws2-s3/src/test/resources/empty.txt differ