You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2021/12/29 14:09:36 UTC
[camel] branch main updated: CAMEL-17392 (#6598)
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 9231ea6 CAMEL-17392 (#6598)
9231ea6 is described below
commit 9231ea60e5d91c3edeabbfea49dd6062d9adf665
Author: aevangelista81 <96...@users.noreply.github.com>
AuthorDate: Wed Dec 29 15:08:35 2021 +0100
CAMEL-17392 (#6598)
Camel-google-storage - Provide a way to filter blobs in a bucket
Co-authored-by: Andrea Evangelista <an...@graphaware.com>
---
.../GoogleCloudStorageComponentConfigurer.java | 3 +
.../GoogleCloudStorageEndpointConfigurer.java | 3 +
.../GoogleCloudStorageEndpointUriFactory.java | 3 +-
.../component/google/storage/google-storage.json | 2 +
.../storage/GoogleCloudStorageConfiguration.java | 14 +++
.../google/storage/GoogleCloudStorageConsumer.java | 11 +-
.../storage/unit/ConsumerWithFilterOptionTest.java | 122 +++++++++++++++++++++
7 files changed, 156 insertions(+), 2 deletions(-)
diff --git a/components/camel-google/camel-google-storage/src/generated/java/org/apache/camel/component/google/storage/GoogleCloudStorageComponentConfigurer.java b/components/camel-google/camel-google-storage/src/generated/java/org/apache/camel/component/google/storage/GoogleCloudStorageComponentConfigurer.java
index 6ddc88f..86e582f 100644
--- a/components/camel-google/camel-google-storage/src/generated/java/org/apache/camel/component/google/storage/GoogleCloudStorageComponentConfigurer.java
+++ b/components/camel-google/camel-google-storage/src/generated/java/org/apache/camel/component/google/storage/GoogleCloudStorageComponentConfigurer.java
@@ -41,6 +41,7 @@ public class GoogleCloudStorageComponentConfigurer extends PropertyConfigurerSup
case "destinationBucket": getOrCreateConfiguration(target).setDestinationBucket(property(camelContext, java.lang.String.class, value)); return true;
case "downloadfilename":
case "downloadFileName": getOrCreateConfiguration(target).setDownloadFileName(property(camelContext, java.lang.String.class, value)); return true;
+ case "filter": getOrCreateConfiguration(target).setFilter(property(camelContext, java.lang.String.class, value)); return true;
case "includebody":
case "includeBody": getOrCreateConfiguration(target).setIncludeBody(property(camelContext, boolean.class, value)); return true;
case "includefolders":
@@ -85,6 +86,7 @@ public class GoogleCloudStorageComponentConfigurer extends PropertyConfigurerSup
case "destinationBucket": return java.lang.String.class;
case "downloadfilename":
case "downloadFileName": return java.lang.String.class;
+ case "filter": return java.lang.String.class;
case "includebody":
case "includeBody": return boolean.class;
case "includefolders":
@@ -125,6 +127,7 @@ public class GoogleCloudStorageComponentConfigurer extends PropertyConfigurerSup
case "destinationBucket": return getOrCreateConfiguration(target).getDestinationBucket();
case "downloadfilename":
case "downloadFileName": return getOrCreateConfiguration(target).getDownloadFileName();
+ case "filter": return getOrCreateConfiguration(target).getFilter();
case "includebody":
case "includeBody": return getOrCreateConfiguration(target).isIncludeBody();
case "includefolders":
diff --git a/components/camel-google/camel-google-storage/src/generated/java/org/apache/camel/component/google/storage/GoogleCloudStorageEndpointConfigurer.java b/components/camel-google/camel-google-storage/src/generated/java/org/apache/camel/component/google/storage/GoogleCloudStorageEndpointConfigurer.java
index 00d8366..a9ab367 100644
--- a/components/camel-google/camel-google-storage/src/generated/java/org/apache/camel/component/google/storage/GoogleCloudStorageEndpointConfigurer.java
+++ b/components/camel-google/camel-google-storage/src/generated/java/org/apache/camel/component/google/storage/GoogleCloudStorageEndpointConfigurer.java
@@ -42,6 +42,7 @@ public class GoogleCloudStorageEndpointConfigurer extends PropertyConfigurerSupp
case "exceptionHandler": target.setExceptionHandler(property(camelContext, org.apache.camel.spi.ExceptionHandler.class, value)); return true;
case "exchangepattern":
case "exchangePattern": target.setExchangePattern(property(camelContext, org.apache.camel.ExchangePattern.class, value)); return true;
+ case "filter": target.getConfiguration().setFilter(property(camelContext, java.lang.String.class, value)); return true;
case "greedy": target.setGreedy(property(camelContext, boolean.class, value)); return true;
case "includebody":
case "includeBody": target.getConfiguration().setIncludeBody(property(camelContext, boolean.class, value)); return true;
@@ -116,6 +117,7 @@ public class GoogleCloudStorageEndpointConfigurer extends PropertyConfigurerSupp
case "exceptionHandler": return org.apache.camel.spi.ExceptionHandler.class;
case "exchangepattern":
case "exchangePattern": return org.apache.camel.ExchangePattern.class;
+ case "filter": return java.lang.String.class;
case "greedy": return boolean.class;
case "includebody":
case "includeBody": return boolean.class;
@@ -186,6 +188,7 @@ public class GoogleCloudStorageEndpointConfigurer extends PropertyConfigurerSupp
case "exceptionHandler": return target.getExceptionHandler();
case "exchangepattern":
case "exchangePattern": return target.getExchangePattern();
+ case "filter": return target.getConfiguration().getFilter();
case "greedy": return target.isGreedy();
case "includebody":
case "includeBody": return target.getConfiguration().isIncludeBody();
diff --git a/components/camel-google/camel-google-storage/src/generated/java/org/apache/camel/component/google/storage/GoogleCloudStorageEndpointUriFactory.java b/components/camel-google/camel-google-storage/src/generated/java/org/apache/camel/component/google/storage/GoogleCloudStorageEndpointUriFactory.java
index a7cebe6..a5de686 100644
--- a/components/camel-google/camel-google-storage/src/generated/java/org/apache/camel/component/google/storage/GoogleCloudStorageEndpointUriFactory.java
+++ b/components/camel-google/camel-google-storage/src/generated/java/org/apache/camel/component/google/storage/GoogleCloudStorageEndpointUriFactory.java
@@ -20,7 +20,7 @@ public class GoogleCloudStorageEndpointUriFactory extends org.apache.camel.suppo
private static final Set<String> PROPERTY_NAMES;
private static final Set<String> SECRET_PROPERTY_NAMES;
static {
- Set<String> props = new HashSet<>(34);
+ Set<String> props = new HashSet<>(35);
props.add("backoffMultiplier");
props.add("bucketName");
props.add("destinationBucket");
@@ -46,6 +46,7 @@ public class GoogleCloudStorageEndpointUriFactory extends org.apache.camel.suppo
props.add("includeFolders");
props.add("storageClient");
props.add("backoffIdleThreshold");
+ props.add("filter");
props.add("lazyStartProducer");
props.add("delay");
props.add("includeBody");
diff --git a/components/camel-google/camel-google-storage/src/generated/resources/org/apache/camel/component/google/storage/google-storage.json b/components/camel-google/camel-google-storage/src/generated/resources/org/apache/camel/component/google/storage/google-storage.json
index 092da32..ac97787 100644
--- a/components/camel-google/camel-google-storage/src/generated/resources/org/apache/camel/component/google/storage/google-storage.json
+++ b/components/camel-google/camel-google-storage/src/generated/resources/org/apache/camel/component/google/storage/google-storage.json
@@ -32,6 +32,7 @@
"deleteAfterRead": { "kind": "property", "displayName": "Delete After Read", "group": "consumer", "label": "consumer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "configurationClass": "org.apache.camel.component.google.storage.GoogleCloudStorageConfiguration", "configurationField": "configuration", "description": "Delete objects from the bucket after they have been retrieved. The delete [...]
"destinationBucket": { "kind": "property", "displayName": "Destination Bucket", "group": "consumer", "label": "consumer", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.google.storage.GoogleCloudStorageConfiguration", "configurationField": "configuration", "description": "Define the destination bucket where an object must be moved when moveAfterRead is set [...]
"downloadFileName": { "kind": "property", "displayName": "Download File Name", "group": "consumer", "label": "consumer", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.google.storage.GoogleCloudStorageConfiguration", "configurationField": "configuration", "description": "The folder or filename to use when downloading the blob. By default, this specifies th [...]
+ "filter": { "kind": "property", "displayName": "Filter", "group": "consumer", "label": "consumer", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.google.storage.GoogleCloudStorageConfiguration", "configurationField": "configuration", "description": "A regular expression to include only blobs with name matching it." },
"includeBody": { "kind": "property", "displayName": "Include Body", "group": "consumer", "label": "consumer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "configurationClass": "org.apache.camel.component.google.storage.GoogleCloudStorageConfiguration", "configurationField": "configuration", "description": "If it is true, the Object exchange will be consumed and put into the body. If false [...]
"includeFolders": { "kind": "property", "displayName": "Include Folders", "group": "consumer", "label": "consumer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "configurationClass": "org.apache.camel.component.google.storage.GoogleCloudStorageConfiguration", "configurationField": "configuration", "description": "If it is true, the folders\/directories will be consumed. If it is false, the [...]
"moveAfterRead": { "kind": "property", "displayName": "Move After Read", "group": "consumer", "label": "consumer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "configurationClass": "org.apache.camel.component.google.storage.GoogleCloudStorageConfiguration", "configurationField": "configuration", "description": "Move objects from the origin bucket to a different bucket after they have bee [...]
@@ -51,6 +52,7 @@
"deleteAfterRead": { "kind": "parameter", "displayName": "Delete After Read", "group": "consumer", "label": "consumer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "configurationClass": "org.apache.camel.component.google.storage.GoogleCloudStorageConfiguration", "configurationField": "configuration", "description": "Delete objects from the bucket after they have been retrieved. The delete [...]
"destinationBucket": { "kind": "parameter", "displayName": "Destination Bucket", "group": "consumer", "label": "consumer", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.google.storage.GoogleCloudStorageConfiguration", "configurationField": "configuration", "description": "Define the destination bucket where an object must be moved when moveAfterRead is se [...]
"downloadFileName": { "kind": "parameter", "displayName": "Download File Name", "group": "consumer", "label": "consumer", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.google.storage.GoogleCloudStorageConfiguration", "configurationField": "configuration", "description": "The folder or filename to use when downloading the blob. By default, this specifies t [...]
+ "filter": { "kind": "parameter", "displayName": "Filter", "group": "consumer", "label": "consumer", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.google.storage.GoogleCloudStorageConfiguration", "configurationField": "configuration", "description": "A regular expression to include only blobs with name matching it." },
"includeBody": { "kind": "parameter", "displayName": "Include Body", "group": "consumer", "label": "consumer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "configurationClass": "org.apache.camel.component.google.storage.GoogleCloudStorageConfiguration", "configurationField": "configuration", "description": "If it is true, the Object exchange will be consumed and put into the body. If fals [...]
"includeFolders": { "kind": "parameter", "displayName": "Include Folders", "group": "consumer", "label": "consumer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "configurationClass": "org.apache.camel.component.google.storage.GoogleCloudStorageConfiguration", "configurationField": "configuration", "description": "If it is true, the folders\/directories will be consumed. If it is false, th [...]
"moveAfterRead": { "kind": "parameter", "displayName": "Move After Read", "group": "consumer", "label": "consumer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "configurationClass": "org.apache.camel.component.google.storage.GoogleCloudStorageConfiguration", "configurationField": "configuration", "description": "Move objects from the origin bucket to a different bucket after they have be [...]
diff --git a/components/camel-google/camel-google-storage/src/main/java/org/apache/camel/component/google/storage/GoogleCloudStorageConfiguration.java b/components/camel-google/camel-google-storage/src/main/java/org/apache/camel/component/google/storage/GoogleCloudStorageConfiguration.java
index cfbf2bf..7e98015 100644
--- a/components/camel-google/camel-google-storage/src/main/java/org/apache/camel/component/google/storage/GoogleCloudStorageConfiguration.java
+++ b/components/camel-google/camel-google-storage/src/main/java/org/apache/camel/component/google/storage/GoogleCloudStorageConfiguration.java
@@ -76,6 +76,9 @@ public class GoogleCloudStorageConfiguration implements Cloneable {
@Metadata(autowired = true)
private Storage storageClient;
+ @UriParam(label = "consumer", description = "A regular expression to include only blobs with name matching it.")
+ private String filter;
+
public String getBucketName() {
return this.bucketName;
}
@@ -260,4 +263,15 @@ public class GoogleCloudStorageConfiguration implements Cloneable {
}
}
+ /**
+ * A regular expression to include only blobs with name matching it.
+ */
+ public void setFilter(String filter) {
+ this.filter = filter;
+ }
+
+ public String getFilter() {
+ return filter;
+ }
+
}
diff --git a/components/camel-google/camel-google-storage/src/main/java/org/apache/camel/component/google/storage/GoogleCloudStorageConsumer.java b/components/camel-google/camel-google-storage/src/main/java/org/apache/camel/component/google/storage/GoogleCloudStorageConsumer.java
index 39a9b9f..5f37612 100644
--- a/components/camel-google/camel-google-storage/src/main/java/org/apache/camel/component/google/storage/GoogleCloudStorageConsumer.java
+++ b/components/camel-google/camel-google-storage/src/main/java/org/apache/camel/component/google/storage/GoogleCloudStorageConsumer.java
@@ -87,6 +87,7 @@ public class GoogleCloudStorageConsumer extends ScheduledBatchPollingConsumer {
String fileName = getConfiguration().getObjectName();
String bucketName = getConfiguration().getBucketName();
Queue<Exchange> exchanges;
+ String filter = getConfiguration().getFilter();
if (fileName != null) {
LOG.trace("Getting object in bucket [{}] with file name [{}]...", bucketName, fileName);
@@ -99,7 +100,15 @@ public class GoogleCloudStorageConsumer extends ScheduledBatchPollingConsumer {
List<Blob> bloblist = new LinkedList<>();
for (Blob blob : getStorageClient().list(bucketName).iterateAll()) {
- bloblist.add(blob);
+
+ if (filter != null && !filter.isEmpty()) {
+ if (blob.getBlobId().getName().matches(filter)) {
+ bloblist.add(blob);
+ }
+ } else {
+ bloblist.add(blob);
+ }
+
}
if (LOG.isTraceEnabled()) {
diff --git a/components/camel-google/camel-google-storage/src/test/java/org/apache/camel/component/google/storage/unit/ConsumerWithFilterOptionTest.java b/components/camel-google/camel-google-storage/src/test/java/org/apache/camel/component/google/storage/unit/ConsumerWithFilterOptionTest.java
new file mode 100644
index 0000000..46139bc
--- /dev/null
+++ b/components/camel-google/camel-google-storage/src/test/java/org/apache/camel/component/google/storage/unit/ConsumerWithFilterOptionTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.google.storage.unit;
+
+import java.nio.file.Path;
+
+import org.apache.camel.EndpointInject;
+import org.apache.camel.ProducerTemplate;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.google.storage.GoogleCloudStorageConstants;
+import org.apache.camel.component.mock.MockEndpoint;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+public class ConsumerWithFilterOptionTest extends GoogleCloudStorageBaseTest {
+
+ @EndpointInject
+ private ProducerTemplate template;
+
+ @EndpointInject("mock:result")
+ private MockEndpoint result;
+
+ @EndpointInject("mock:consumedObjects")
+ private MockEndpoint consumedObjects;
+
+ @Override
+ protected RouteBuilder createRouteBuilder() throws Exception {
+ return new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+
+ String endpoint = "google-storage://myCamelBucket?autoCreateBucket=true";
+
+ from("direct:putObject")
+ .startupOrder(1)
+ .to(endpoint)
+ .to("mock:result");
+
+ from("google-storage://myCamelBucket?"
+ + "moveAfterRead=true"
+ + "&destinationBucket=camelDestinationBucket"
+ + "&autoCreateBucket=true"
+ + "&deleteAfterRead=true"
+ + "&includeBody=true"
+ + "&filter=.*.csv")
+ .startupOrder(2)
+ .to("mock:consumedObjects");
+
+ }
+ };
+ }
+
+ @Test
+ public void onlyCsvFilesShouldBeFiltered(@TempDir Path tempDir) throws Exception {
+
+ final int totalNumberOfFiles = 5;
+ final int numberOfFilteredFiles = 2;
+
+ Path path = tempDir.resolve("file");
+
+ uploadFiles(path.toString(), "csv", 2);
+ uploadFiles(path.toString(), "txt", 3);
+
+ result.expectedMessageCount(totalNumberOfFiles);
+ consumedObjects.expectedMessageCount(numberOfFilteredFiles);
+
+ assertMockEndpointsSatisfied();
+
+ context.stop();
+
+ }
+
+ @Test
+ public void noFilesShouldBeFitered(@TempDir Path tempDir) throws Exception {
+
+ final int totalNumberOfFiles = 5;
+ final int numberOfFilteredFiles = 0;
+
+ Path path = tempDir.resolve("file");
+
+ uploadFiles(path.toString(), "json", 2);
+ uploadFiles(path.toString(), "txt", 3);
+
+ result.expectedMessageCount(totalNumberOfFiles);
+ consumedObjects.expectedMessageCount(numberOfFilteredFiles);
+
+ assertMockEndpointsSatisfied();
+
+ context.stop();
+
+ }
+
+ private void uploadFiles(final String fileNamePrefix, final String extension, final int numberOfFiles) {
+
+ for (int i = 0; i < numberOfFiles; i++) {
+
+ final String filename = String.format("%s_%s.%s", fileNamePrefix, i, extension);
+ final String body = String.format("body_%s", i);
+ //upload a file
+
+ template.send("direct:putObject", exchange -> {
+ exchange.getIn().setHeader(GoogleCloudStorageConstants.OBJECT_NAME, filename);
+ exchange.getIn().setBody(body);
+ });
+ }
+ }
+
+}