You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2017/09/12 07:51:00 UTC
[1/3] camel git commit: CAMEL-11698: Make s3 consumer auto close
s3objects if an exception was thrown during creating exchanges/batch to avoid
leaking resources. Thanks to Mykhailo Vlakh for reporting.
Repository: camel
Updated Branches:
refs/heads/camel-2.18.x fd6fb1996 -> ccb8d4e60
refs/heads/camel-2.19.x f45e90b43 -> 577869637
refs/heads/master 91bf7bf1e -> a8bbcd65a
CAMEL-11698: Make s3 consumer auto close s3objects if an exception was thrown during creating exchanges/batch to avoid leaking resources. Thanks to Mykhailo Vlakh for reporting.
Project: http://git-wip-us.apache.org/repos/asf/camel/repo
Commit: http://git-wip-us.apache.org/repos/asf/camel/commit/a8bbcd65
Tree: http://git-wip-us.apache.org/repos/asf/camel/tree/a8bbcd65
Diff: http://git-wip-us.apache.org/repos/asf/camel/diff/a8bbcd65
Branch: refs/heads/master
Commit: a8bbcd65a59178c9c893cf2fbc2720ea7e7c2b7b
Parents: 91bf7bf
Author: Claus Ibsen <da...@apache.org>
Authored: Tue Sep 12 09:39:55 2017 +0200
Committer: Claus Ibsen <da...@apache.org>
Committed: Tue Sep 12 09:39:55 2017 +0200
----------------------------------------------------------------------
.../src/main/docs/aws-s3-component.adoc | 22 ++++++++---------
.../camel/component/aws/s3/S3Configuration.java | 26 +++++++++-----------
.../camel/component/aws/s3/S3Consumer.java | 24 ++++++++++++++----
.../camel/component/aws/s3/S3Endpoint.java | 11 +++------
4 files changed, 45 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/camel/blob/a8bbcd65/components/camel-aws/src/main/docs/aws-s3-component.adoc
----------------------------------------------------------------------
diff --git a/components/camel-aws/src/main/docs/aws-s3-component.adoc b/components/camel-aws/src/main/docs/aws-s3-component.adoc
index 2612603..101906b 100644
--- a/components/camel-aws/src/main/docs/aws-s3-component.adoc
+++ b/components/camel-aws/src/main/docs/aws-s3-component.adoc
@@ -73,30 +73,30 @@ with the following path and query parameters:
| **amazonS3Endpoint** (common) | The region with which the AWS-S3 client wants to work with. | | String
| **encryptionMaterials** (common) | The encryption materials to use in case of Symmetric/Asymmetric client usage | | EncryptionMaterials
| **pathStyleAccess** (common) | Whether or not the S3 client should use path style access | false | boolean
-| **policy** (common) | Camel 2.8.4: The policy for this queue to set in the com.amazonaws.services.s3.AmazonS3setBucketPolicy() method. | | String
-| **proxyHost** (common) | Camel 2.16: To define a proxy host when instantiating the SQS client | | String
-| **proxyPort** (common) | Camel 2.16: Specify a proxy port to be used inside the client definition. | | Integer
+| **policy** (common) | The policy for this queue to set in the com.amazonaws.services.s3.AmazonS3setBucketPolicy() method. | | String
+| **proxyHost** (common) | To define a proxy host when instantiating the SQS client | | String
+| **proxyPort** (common) | Specify a proxy port to be used inside the client definition. | | Integer
| **secretKey** (common) | Amazon AWS Secret Key | | String
| **useEncryption** (common) | Define if encryption must be used or not | false | boolean
-| **autocloseBody** (consumer) | If this option is true and includeBody is true then the S3Object.close() method will be called on exchange completion This option is strongly related to includeBody option. In case of setting includeBody to true and autocloseBody to false it will be up to the caller to close the S3Object stream. Setting autocloseBody to true will close the S3Object stream automatically. | true | boolean
| **bridgeErrorHandler** (consumer) | Allows for bridging the consumer to the Camel routing Error Handler which mean any exceptions occurred while the consumer is trying to pickup incoming messages or the likes will now be processed as a message and handled by the routing Error Handler. By default the consumer will use the org.apache.camel.spi.ExceptionHandler to deal with exceptions that will be logged at WARN or ERROR level and ignored. | false | boolean
| **deleteAfterRead** (consumer) | Delete objects from S3 after they have been retrieved. The delete is only performed if the Exchange is committed. If a rollback occurs the object is not deleted. If this option is false then the same objects will be retrieve over and over again on the polls. Therefore you need to use the Idempotent Consumer EIP in the route to filter out duplicates. You can filter using the link S3ConstantsBUCKET_NAME and link S3ConstantsKEY headers or only the link S3ConstantsKEY header. | true | boolean
| **fileName** (consumer) | To get the object from the bucket with the given file name | | String
| **includeBody** (consumer) | If it is true the exchange body will be set to a stream to the contents of the file. If false the headers will be set with the S3 object metadata but the body will be null. This option is strongly related to autocloseBody option. In case of setting includeBody to true and autocloseBody to false it will be up to the caller to close the S3Object stream. Setting autocloseBody to true will close the S3Object stream automatically. | true | boolean
| **maxConnections** (consumer) | Set the maxConnections parameter in the S3 client configuration | 60 | int
| **maxMessagesPerPoll** (consumer) | Gets the maximum number of messages as a limit to poll at each polling. Is default unlimited but use 0 or negative number to disable it as unlimited. | 10 | int
-| **prefix** (consumer) | Camel 2.10.1: The prefix which is used in the com.amazonaws.services.s3.model.ListObjectsRequest to only consume objects we are interested in. | | String
+| **prefix** (consumer) | The prefix which is used in the com.amazonaws.services.s3.model.ListObjectsRequest to only consume objects we are interested in. | | String
| **sendEmptyMessageWhenIdle** (consumer) | If the polling consumer did not poll any files you can enable this option to send an empty message (no body) instead. | false | boolean
+| **autocloseBody** (consumer) | If this option is true and includeBody is true then the S3Object.close() method will be called on exchange completion. This option is strongly related to includeBody option. In case of setting includeBody to true and autocloseBody to false it will be up to the caller to close the S3Object stream. Setting autocloseBody to true will close the S3Object stream automatically. | true | boolean
| **exceptionHandler** (consumer) | To let the consumer use a custom ExceptionHandler. Notice if the option bridgeErrorHandler is enabled then this options is not in use. By default the consumer will deal with exceptions that will be logged at WARN or ERROR level and ignored. | | ExceptionHandler
| **exchangePattern** (consumer) | Sets the exchange pattern when the consumer creates an exchange. | | ExchangePattern
| **pollStrategy** (consumer) | A pluggable org.apache.camel.PollingConsumerPollingStrategy allowing you to provide your custom implementation to control error handling usually occurred during the poll operation before an Exchange have been created and being routed in Camel. | | PollingConsumerPoll Strategy
-| **deleteAfterWrite** (producer) | Camel 2.11.0: Delete file object after the S3 file has been uploaded | false | boolean
-| **multiPartUpload** (producer) | Camel 2.15.0: If it is true camel will upload the file with multi part format the part size is decided by the option of partSize | false | boolean
-| **operation** (producer) | Camel 2.18: The operation to do in case the user don't want to do only an upload | | S3Operations
-| **partSize** (producer) | Camel 2.15.0: Setup the partSize which is used in multi part upload the default size is 25M. | 26214400 | long
+| **deleteAfterWrite** (producer) | Delete file object after the S3 file has been uploaded | false | boolean
+| **multiPartUpload** (producer) | If it is true camel will upload the file with multi part format the part size is decided by the option of partSize | false | boolean
+| **operation** (producer) | The operation to do in case the user don't want to do only an upload | | S3Operations
+| **partSize** (producer) | Setup the partSize which is used in multi part upload the default size is 25M. | 26214400 | long
| **region** (producer) | The region where the bucket is located. This option is used in the com.amazonaws.services.s3.model.CreateBucketRequest. | | String
-| **serverSideEncryption** (producer) | Camel 2.16: Sets the server-side encryption algorithm when encrypting the object using AWS-managed keys. For example use AES256. | | String
-| **storageClass** (producer) | Camel 2.8.4: The storage class to set in the com.amazonaws.services.s3.model.PutObjectRequest request. | | String
+| **serverSideEncryption** (producer) | Sets the server-side encryption algorithm when encrypting the object using AWS-managed keys. For example use AES256. | | String
+| **storageClass** (producer) | The storage class to set in the com.amazonaws.services.s3.model.PutObjectRequest request. | | String
| **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean
| **backoffErrorThreshold** (scheduler) | The number of subsequent error polls (failed due some error) that should happen before the backoffMultipler should kick-in. | | int
| **backoffIdleThreshold** (scheduler) | The number of subsequent idle polls that should happen before the backoffMultipler should kick-in. | | int
http://git-wip-us.apache.org/repos/asf/camel/blob/a8bbcd65/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Configuration.java
----------------------------------------------------------------------
diff --git a/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Configuration.java b/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Configuration.java
index ff4184a..655e9da 100644
--- a/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Configuration.java
+++ b/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Configuration.java
@@ -65,7 +65,7 @@ public class S3Configuration implements Cloneable {
private boolean pathStyleAccess;
@UriParam(label = "producer", enums = "copyObject,deleteBucket,listBuckets")
private S3Operations operation;
- @UriParam(label = "consumer", defaultValue = "true")
+ @UriParam(label = "consumer,advanced", defaultValue = "true")
private boolean autocloseBody = true;
@UriParam(label = "common")
private EncryptionMaterials encryptionMaterials;
@@ -77,7 +77,7 @@ public class S3Configuration implements Cloneable {
}
/**
- * *Camel 2.15.0*: Setup the partSize which is used in multi part upload,
+ * Setup the partSize which is used in multi part upload,
* the default size is 25M.
*/
public void setPartSize(long partSize) {
@@ -89,7 +89,7 @@ public class S3Configuration implements Cloneable {
}
/**
- * *Camel 2.15.0*: If it is true, camel will upload the file with multi part
+ * If it is true, camel will upload the file with multi part
* format, the part size is decided by the option of `partSize`
*/
public void setMultiPartUpload(boolean multiPartUpload) {
@@ -146,7 +146,7 @@ public class S3Configuration implements Cloneable {
}
/**
- * *Camel 2.10.1*: The prefix which is used in the
+ * The prefix which is used in the
* com.amazonaws.services.s3.model.ListObjectsRequest to only consume
* objects we are interested in.
*/
@@ -230,7 +230,7 @@ public class S3Configuration implements Cloneable {
}
/**
- * *Camel 2.11.0*: Delete file object after the S3 file has been uploaded
+ * Delete file object after the S3 file has been uploaded
*/
public void setDeleteAfterWrite(boolean deleteAfterWrite) {
this.deleteAfterWrite = deleteAfterWrite;
@@ -241,7 +241,7 @@ public class S3Configuration implements Cloneable {
}
/**
- * *Camel 2.8.4*: The policy for this queue to set in the
+ * The policy for this queue to set in the
* `com.amazonaws.services.s3.AmazonS3#setBucketPolicy()` method.
*/
public void setPolicy(String policy) {
@@ -253,7 +253,7 @@ public class S3Configuration implements Cloneable {
}
/**
- * *Camel 2.8.4*: The storage class to set in the
+ * The storage class to set in the
* `com.amazonaws.services.s3.model.PutObjectRequest` request.
*/
public void setStorageClass(String storageClass) {
@@ -265,7 +265,7 @@ public class S3Configuration implements Cloneable {
}
/**
- * *Camel 2.16*: Sets the server-side encryption algorithm when encrypting
+ * Sets the server-side encryption algorithm when encrypting
* the object using AWS-managed keys. For example use <tt>AES256</tt>.
*/
public void setServerSideEncryption(String serverSideEncryption) {
@@ -277,7 +277,7 @@ public class S3Configuration implements Cloneable {
}
/**
- * *Camel 2.16*: To define a proxy host when instantiating the SQS client
+ * To define a proxy host when instantiating the SQS client
*/
public void setProxyHost(String proxyHost) {
this.proxyHost = proxyHost;
@@ -288,8 +288,7 @@ public class S3Configuration implements Cloneable {
}
/**
- * *Camel 2.16*: Specify a proxy port to be used inside the client
- * definition.
+ * Specify a proxy port to be used inside the client definition.
*/
public void setProxyPort(Integer proxyPort) {
this.proxyPort = proxyPort;
@@ -311,8 +310,7 @@ public class S3Configuration implements Cloneable {
}
/**
- * *Camel 2.18*: The operation to do in case the user don't want to do only
- * an upload
+ * The operation to do in case the user don't want to do only an upload
*/
public void setOperation(S3Operations operation) {
this.operation = operation;
@@ -324,7 +322,7 @@ public class S3Configuration implements Cloneable {
/**
* If this option is true and includeBody is true, then the S3Object.close()
- * method will be called on exchange completion This option is strongly
+ * method will be called on exchange completion. This option is strongly
* related to includeBody option. In case of setting includeBody to true and
* autocloseBody to false, it will be up to the caller to close the S3Object
* stream. Setting autocloseBody to true, will close the S3Object stream
http://git-wip-us.apache.org/repos/asf/camel/blob/a8bbcd65/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Consumer.java
----------------------------------------------------------------------
diff --git a/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Consumer.java b/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Consumer.java
index 5fb4936..7483ecd 100644
--- a/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Consumer.java
+++ b/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Consumer.java
@@ -16,6 +16,8 @@
*/
package org.apache.camel.component.aws.s3;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
@@ -35,6 +37,7 @@ import org.apache.camel.Processor;
import org.apache.camel.impl.ScheduledBatchPollingConsumer;
import org.apache.camel.spi.Synchronization;
import org.apache.camel.util.CastUtils;
+import org.apache.camel.util.IOHelper;
import org.apache.camel.util.ObjectHelper;
import org.apache.camel.util.URISupport;
import org.slf4j.Logger;
@@ -112,12 +115,23 @@ public class S3Consumer extends ScheduledBatchPollingConsumer {
if (LOG.isTraceEnabled()) {
LOG.trace("Received {} messages in this poll", s3ObjectSummaries.size());
}
-
+
+ Collection<S3Object> s3Objects = new ArrayList<>();
Queue<Exchange> answer = new LinkedList<Exchange>();
- for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries) {
- S3Object s3Object = getAmazonS3Client().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey());
- Exchange exchange = getEndpoint().createExchange(s3Object);
- answer.add(exchange);
+ try {
+ for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries) {
+ S3Object s3Object = getAmazonS3Client().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey());
+ s3Objects.add(s3Object);
+
+ Exchange exchange = getEndpoint().createExchange(s3Object);
+ answer.add(exchange);
+ }
+ } catch (Throwable e) {
+ LOG.warn("Error getting S3Object due: " + e.getMessage(), e);
+ // ensure all previous gathered s3 objects are closed
+ // if there was an exception creating the exchanges in this batch
+ s3Objects.forEach(IOHelper::close);
+ throw e;
}
return answer;
http://git-wip-us.apache.org/repos/asf/camel/blob/a8bbcd65/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Endpoint.java
----------------------------------------------------------------------
diff --git a/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Endpoint.java b/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Endpoint.java
index 584daa7..b704171 100644
--- a/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Endpoint.java
+++ b/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Endpoint.java
@@ -49,6 +49,7 @@ import org.apache.camel.spi.UriEndpoint;
import org.apache.camel.spi.UriParam;
import org.apache.camel.spi.UriPath;
import org.apache.camel.support.SynchronizationAdapter;
+import org.apache.camel.util.IOHelper;
import org.apache.camel.util.ObjectHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -194,19 +195,13 @@ public class S3Endpoint extends ScheduledPollEndpoint {
* consumer does not close the stream or object on commit.
*/
if (!configuration.isIncludeBody()) {
- try {
- s3Object.close();
- } catch (IOException e) {
- }
+ IOHelper.close(s3Object);
} else {
if (configuration.isAutocloseBody()) {
exchange.addOnCompletion(new SynchronizationAdapter() {
@Override
public void onDone(Exchange exchange) {
- try {
- s3Object.close();
- } catch (IOException e) {
- }
+ IOHelper.close(s3Object);
}
});
}
[3/3] camel git commit: CAMEL-11698: Make s3 consumer auto close
s3objects if an exception was thrown during creating exchanges/batch to avoid
leaking resources. Thanks to Mykhailo Vlakh for reporting.
Posted by da...@apache.org.
CAMEL-11698: Make s3 consumer auto close s3objects if an exception was thrown during creating exchanges/batch to avoid leaking resources. Thanks to Mykhailo Vlakh for reporting.
Project: http://git-wip-us.apache.org/repos/asf/camel/repo
Commit: http://git-wip-us.apache.org/repos/asf/camel/commit/ccb8d4e6
Tree: http://git-wip-us.apache.org/repos/asf/camel/tree/ccb8d4e6
Diff: http://git-wip-us.apache.org/repos/asf/camel/diff/ccb8d4e6
Branch: refs/heads/camel-2.18.x
Commit: ccb8d4e60068fd607bfacc0eb2312557818f968a
Parents: fd6fb19
Author: Claus Ibsen <da...@apache.org>
Authored: Tue Sep 12 09:39:55 2017 +0200
Committer: Claus Ibsen <da...@apache.org>
Committed: Tue Sep 12 09:49:04 2017 +0200
----------------------------------------------------------------------
.../camel/component/aws/s3/S3Consumer.java | 24 ++++++++++++++++----
.../camel/component/aws/s3/S3Endpoint.java | 11 +++------
2 files changed, 22 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/camel/blob/ccb8d4e6/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Consumer.java
----------------------------------------------------------------------
diff --git a/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Consumer.java b/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Consumer.java
index 5fb4936..7483ecd 100644
--- a/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Consumer.java
+++ b/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Consumer.java
@@ -16,6 +16,8 @@
*/
package org.apache.camel.component.aws.s3;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
@@ -35,6 +37,7 @@ import org.apache.camel.Processor;
import org.apache.camel.impl.ScheduledBatchPollingConsumer;
import org.apache.camel.spi.Synchronization;
import org.apache.camel.util.CastUtils;
+import org.apache.camel.util.IOHelper;
import org.apache.camel.util.ObjectHelper;
import org.apache.camel.util.URISupport;
import org.slf4j.Logger;
@@ -112,12 +115,23 @@ public class S3Consumer extends ScheduledBatchPollingConsumer {
if (LOG.isTraceEnabled()) {
LOG.trace("Received {} messages in this poll", s3ObjectSummaries.size());
}
-
+
+ Collection<S3Object> s3Objects = new ArrayList<>();
Queue<Exchange> answer = new LinkedList<Exchange>();
- for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries) {
- S3Object s3Object = getAmazonS3Client().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey());
- Exchange exchange = getEndpoint().createExchange(s3Object);
- answer.add(exchange);
+ try {
+ for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries) {
+ S3Object s3Object = getAmazonS3Client().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey());
+ s3Objects.add(s3Object);
+
+ Exchange exchange = getEndpoint().createExchange(s3Object);
+ answer.add(exchange);
+ }
+ } catch (Throwable e) {
+ LOG.warn("Error getting S3Object due: " + e.getMessage(), e);
+ // ensure all previous gathered s3 objects are closed
+ // if there was an exception creating the exchanges in this batch
+ s3Objects.forEach(IOHelper::close);
+ throw e;
}
return answer;
http://git-wip-us.apache.org/repos/asf/camel/blob/ccb8d4e6/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Endpoint.java
----------------------------------------------------------------------
diff --git a/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Endpoint.java b/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Endpoint.java
index 05a61f4..86197a8 100644
--- a/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Endpoint.java
+++ b/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Endpoint.java
@@ -44,6 +44,7 @@ import org.apache.camel.spi.UriEndpoint;
import org.apache.camel.spi.UriParam;
import org.apache.camel.spi.UriPath;
import org.apache.camel.support.SynchronizationAdapter;
+import org.apache.camel.util.IOHelper;
import org.apache.camel.util.ObjectHelper;
import org.slf4j.Logger;
@@ -188,19 +189,13 @@ public class S3Endpoint extends ScheduledPollEndpoint {
* As of 2.17, the consumer does not close the stream or object on commit.
*/
if (!configuration.isIncludeBody()) {
- try {
- s3Object.close();
- } catch (IOException e) {
- }
+ IOHelper.close(s3Object);
} else {
if (configuration.isAutocloseBody()) {
exchange.addOnCompletion(new SynchronizationAdapter() {
@Override
public void onDone(Exchange exchange) {
- try {
- s3Object.close();
- } catch (IOException e) {
- }
+ IOHelper.close(s3Object);
}
});
}
[2/3] camel git commit: CAMEL-11698: Make s3 consumer auto close
s3objects if an exception was thrown during creating exchanges/batch to avoid
leaking resources. Thanks to Mykhailo Vlakh for reporting.
Posted by da...@apache.org.
CAMEL-11698: Make s3 consumer auto close s3objects if an exception was thrown during creating exchanges/batch to avoid leaking resources. Thanks to Mykhailo Vlakh for reporting.
Project: http://git-wip-us.apache.org/repos/asf/camel/repo
Commit: http://git-wip-us.apache.org/repos/asf/camel/commit/57786963
Tree: http://git-wip-us.apache.org/repos/asf/camel/tree/57786963
Diff: http://git-wip-us.apache.org/repos/asf/camel/diff/57786963
Branch: refs/heads/camel-2.19.x
Commit: 57786963749d3fac6e9254312dc43e623fe0eed1
Parents: f45e90b
Author: Claus Ibsen <da...@apache.org>
Authored: Tue Sep 12 09:39:55 2017 +0200
Committer: Claus Ibsen <da...@apache.org>
Committed: Tue Sep 12 09:48:21 2017 +0200
----------------------------------------------------------------------
.../camel/component/aws/s3/S3Consumer.java | 24 ++++++++++++++++----
.../camel/component/aws/s3/S3Endpoint.java | 11 +++------
2 files changed, 22 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/camel/blob/57786963/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Consumer.java
----------------------------------------------------------------------
diff --git a/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Consumer.java b/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Consumer.java
index 5fb4936..7483ecd 100644
--- a/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Consumer.java
+++ b/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Consumer.java
@@ -16,6 +16,8 @@
*/
package org.apache.camel.component.aws.s3;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
@@ -35,6 +37,7 @@ import org.apache.camel.Processor;
import org.apache.camel.impl.ScheduledBatchPollingConsumer;
import org.apache.camel.spi.Synchronization;
import org.apache.camel.util.CastUtils;
+import org.apache.camel.util.IOHelper;
import org.apache.camel.util.ObjectHelper;
import org.apache.camel.util.URISupport;
import org.slf4j.Logger;
@@ -112,12 +115,23 @@ public class S3Consumer extends ScheduledBatchPollingConsumer {
if (LOG.isTraceEnabled()) {
LOG.trace("Received {} messages in this poll", s3ObjectSummaries.size());
}
-
+
+ Collection<S3Object> s3Objects = new ArrayList<>();
Queue<Exchange> answer = new LinkedList<Exchange>();
- for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries) {
- S3Object s3Object = getAmazonS3Client().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey());
- Exchange exchange = getEndpoint().createExchange(s3Object);
- answer.add(exchange);
+ try {
+ for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries) {
+ S3Object s3Object = getAmazonS3Client().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey());
+ s3Objects.add(s3Object);
+
+ Exchange exchange = getEndpoint().createExchange(s3Object);
+ answer.add(exchange);
+ }
+ } catch (Throwable e) {
+ LOG.warn("Error getting S3Object due: " + e.getMessage(), e);
+ // ensure all previous gathered s3 objects are closed
+ // if there was an exception creating the exchanges in this batch
+ s3Objects.forEach(IOHelper::close);
+ throw e;
}
return answer;
http://git-wip-us.apache.org/repos/asf/camel/blob/57786963/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Endpoint.java
----------------------------------------------------------------------
diff --git a/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Endpoint.java b/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Endpoint.java
index e338527..c5ebe43 100644
--- a/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Endpoint.java
+++ b/components/camel-aws/src/main/java/org/apache/camel/component/aws/s3/S3Endpoint.java
@@ -44,6 +44,7 @@ import org.apache.camel.spi.UriEndpoint;
import org.apache.camel.spi.UriParam;
import org.apache.camel.spi.UriPath;
import org.apache.camel.support.SynchronizationAdapter;
+import org.apache.camel.util.IOHelper;
import org.apache.camel.util.ObjectHelper;
import org.slf4j.Logger;
@@ -191,19 +192,13 @@ public class S3Endpoint extends ScheduledPollEndpoint {
* As of 2.17, the consumer does not close the stream or object on commit.
*/
if (!configuration.isIncludeBody()) {
- try {
- s3Object.close();
- } catch (IOException e) {
- }
+ IOHelper.close(s3Object);
} else {
if (configuration.isAutocloseBody()) {
exchange.addOnCompletion(new SynchronizationAdapter() {
@Override
public void onDone(Exchange exchange) {
- try {
- s3Object.close();
- } catch (IOException e) {
- }
+ IOHelper.close(s3Object);
}
});
}