You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by on...@apache.org on 2017/11/23 13:39:36 UTC
[camel] branch master updated: CAMEL-11656 - add preSort option &
update docs
This is an automated email from the ASF dual-hosted git repository.
onders pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/master by this push:
new 20936b1 CAMEL-11656 - add preSort option & update docs
20936b1 is described below
commit 20936b173f332090c6fc5903737152e7d26eb60b
Author: onders86 <on...@gmail.com>
AuthorDate: Wed Nov 22 14:52:41 2017 +0300
CAMEL-11656 - add preSort option & update docs
---
camel-core/src/main/docs/file-component.adoc | 3 ++-
.../org/apache/camel/component/file/FileConsumer.java | 5 +++++
.../apache/camel/component/file/GenericFileEndpoint.java | 16 ++++++++++++++++
components/camel-ftp/src/main/docs/ftp-component.adoc | 3 ++-
components/camel-ftp/src/main/docs/ftps-component.adoc | 3 ++-
components/camel-ftp/src/main/docs/sftp-component.adoc | 3 ++-
.../apache/camel/component/file/remote/FtpConsumer.java | 7 +++++++
.../apache/camel/component/file/remote/SftpConsumer.java | 6 ++++++
8 files changed, 42 insertions(+), 4 deletions(-)
diff --git a/camel-core/src/main/docs/file-component.adoc b/camel-core/src/main/docs/file-component.adoc
index 8aa0247..294ced1 100644
--- a/camel-core/src/main/docs/file-component.adoc
+++ b/camel-core/src/main/docs/file-component.adoc
@@ -69,7 +69,7 @@ with the following path and query parameters:
| *directoryName* | *Required* The starting directory | | File
|===
-==== Query Parameters (80 parameters):
+==== Query Parameters (81 parameters):
[width="100%",cols="2,5,^1,2",options="header"]
|===
@@ -82,6 +82,7 @@ with the following path and query parameters:
| *moveFailed* (consumer) | Sets the move failure expression based on Simple language. For example to move files into a .error subdirectory use: .error. Note: When moving the files to the fail location Camel will handle the error and will not pick up the file again. | | String
| *noop* (consumer) | If true the file is not moved or deleted in any way. This option is good for readonly data or for ETL type requirements. If noop=true Camel will set idempotent=true as well to avoid consuming the same files over and over again. | false | boolean
| *preMove* (consumer) | Expression (such as File Language) used to dynamically set the filename when moving it before processing. For example to move in-progress files into the order directory set this value to order. | | String
+| *preSort* (consumer) | When pre-sort is enabled then the consumer will sort the file and directory names during polling that was retrieved from the file system. You may want to do this in case you need to operate on the files in a sorted order. The pre-sort is executed before the consumer starts to filter and accept files to process by Camel. This option is default=false meaning disabled. | false | boolean
| *recursive* (consumer) | If a directory will look for files in all the sub-directories as well. | false | boolean
| *sendEmptyMessageWhenIdle* (consumer) | If the polling consumer did not poll any files you can enable this option to send an empty message (no body) instead. | false | boolean
| *directoryMustExist* (consumer) | Similar to startingDirectoryMustExist but this applies during polling recursive sub directories. | false | boolean
diff --git a/camel-core/src/main/java/org/apache/camel/component/file/FileConsumer.java b/camel-core/src/main/java/org/apache/camel/component/file/FileConsumer.java
index 5e3a180..aa3dcfa 100644
--- a/camel-core/src/main/java/org/apache/camel/component/file/FileConsumer.java
+++ b/camel-core/src/main/java/org/apache/camel/component/file/FileConsumer.java
@@ -21,11 +21,13 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.camel.Exchange;
import org.apache.camel.Message;
@@ -84,6 +86,9 @@ public class FileConsumer extends GenericFileConsumer<File> {
}
}
List<File> files = Arrays.asList(dirFiles);
+ if (getEndpoint().isPreSort()) {
+ Collections.sort(files, (a, b) -> a.getAbsoluteFile().compareTo(a.getAbsoluteFile()));
+ }
for (File file : dirFiles) {
// check if we can continue polling in files
diff --git a/camel-core/src/main/java/org/apache/camel/component/file/GenericFileEndpoint.java b/camel-core/src/main/java/org/apache/camel/component/file/GenericFileEndpoint.java
index ec4c0c5..cc7c702 100644
--- a/camel-core/src/main/java/org/apache/camel/component/file/GenericFileEndpoint.java
+++ b/camel-core/src/main/java/org/apache/camel/component/file/GenericFileEndpoint.java
@@ -111,6 +111,8 @@ public abstract class GenericFileEndpoint<T> extends ScheduledPollEndpoint imple
protected boolean recursive;
@UriParam(label = "consumer")
protected boolean delete;
+ @UriParam(label = "consumer")
+ protected boolean preSort;
@UriParam(label = "consumer,filter")
protected int maxMessagesPerPoll;
@UriParam(label = "consumer,filter", defaultValue = "true")
@@ -414,6 +416,20 @@ public abstract class GenericFileEndpoint<T> extends ScheduledPollEndpoint imple
public GenericFileFilter<T> getAntFilter() {
return antFilter;
}
+
+ public boolean isPreSort() {
+ return preSort;
+ }
+
+ /**
+ * When pre-sort is enabled then the consumer will sort the file and directory names during polling,
+ * that was retrieved from the file system. You may want to do this in case you need to operate on the files
+ * in a sorted order. The pre-sort is executed before the consumer starts to filter, and accept files
+ * to process by Camel. This option is default=false meaning disabled.
+ */
+ public void setPreSort(boolean preSort) {
+ this.preSort = preSort;
+ }
public boolean isDelete() {
return delete;
diff --git a/components/camel-ftp/src/main/docs/ftp-component.adoc b/components/camel-ftp/src/main/docs/ftp-component.adoc
index 3c9de24..97fa84b 100644
--- a/components/camel-ftp/src/main/docs/ftp-component.adoc
+++ b/components/camel-ftp/src/main/docs/ftp-component.adoc
@@ -97,7 +97,7 @@ with the following path and query parameters:
| *directoryName* | The starting directory | | String
|===
-==== Query Parameters (103 parameters):
+==== Query Parameters (104 parameters):
[width="100%",cols="2,5,^1,2",options="header"]
|===
@@ -115,6 +115,7 @@ with the following path and query parameters:
| *moveFailed* (consumer) | Sets the move failure expression based on Simple language. For example to move files into a .error subdirectory use: .error. Note: When moving the files to the fail location Camel will handle the error and will not pick up the file again. | | String
| *noop* (consumer) | If true the file is not moved or deleted in any way. This option is good for readonly data or for ETL type requirements. If noop=true Camel will set idempotent=true as well to avoid consuming the same files over and over again. | false | boolean
| *preMove* (consumer) | Expression (such as File Language) used to dynamically set the filename when moving it before processing. For example to move in-progress files into the order directory set this value to order. | | String
+| *preSort* (consumer) | When pre-sort is enabled then the consumer will sort the file and directory names during polling that was retrieved from the file system. You may want to do this in case you need to operate on the files in a sorted order. The pre-sort is executed before the consumer starts to filter and accept files to process by Camel. This option is default=false meaning disabled. | false | boolean
| *recursive* (consumer) | If a directory will look for files in all the sub-directories as well. | false | boolean
| *sendEmptyMessageWhenIdle* (consumer) | If the polling consumer did not poll any files you can enable this option to send an empty message (no body) instead. | false | boolean
| *streamDownload* (consumer) | Sets the download method to use when not using a local working directory. If set to true the remote files are streamed to the route as they are read. When set to false the remote files are loaded into memory before being sent into the route. | false | boolean
diff --git a/components/camel-ftp/src/main/docs/ftps-component.adoc b/components/camel-ftp/src/main/docs/ftps-component.adoc
index 3c6260e..d01d6f9 100644
--- a/components/camel-ftp/src/main/docs/ftps-component.adoc
+++ b/components/camel-ftp/src/main/docs/ftps-component.adoc
@@ -57,7 +57,7 @@ with the following path and query parameters:
| *directoryName* | The starting directory | | String
|===
-==== Query Parameters (111 parameters):
+==== Query Parameters (112 parameters):
[width="100%",cols="2,5,^1,2",options="header"]
|===
@@ -75,6 +75,7 @@ with the following path and query parameters:
| *moveFailed* (consumer) | Sets the move failure expression based on Simple language. For example to move files into a .error subdirectory use: .error. Note: When moving the files to the fail location Camel will handle the error and will not pick up the file again. | | String
| *noop* (consumer) | If true the file is not moved or deleted in any way. This option is good for readonly data or for ETL type requirements. If noop=true Camel will set idempotent=true as well to avoid consuming the same files over and over again. | false | boolean
| *preMove* (consumer) | Expression (such as File Language) used to dynamically set the filename when moving it before processing. For example to move in-progress files into the order directory set this value to order. | | String
+| *preSort* (consumer) | When pre-sort is enabled then the consumer will sort the file and directory names during polling that was retrieved from the file system. You may want to do this in case you need to operate on the files in a sorted order. The pre-sort is executed before the consumer starts to filter and accept files to process by Camel. This option is default=false meaning disabled. | false | boolean
| *recursive* (consumer) | If a directory will look for files in all the sub-directories as well. | false | boolean
| *sendEmptyMessageWhenIdle* (consumer) | If the polling consumer did not poll any files you can enable this option to send an empty message (no body) instead. | false | boolean
| *streamDownload* (consumer) | Sets the download method to use when not using a local working directory. If set to true the remote files are streamed to the route as they are read. When set to false the remote files are loaded into memory before being sent into the route. | false | boolean
diff --git a/components/camel-ftp/src/main/docs/sftp-component.adoc b/components/camel-ftp/src/main/docs/sftp-component.adoc
index d00e65a..85122b1 100644
--- a/components/camel-ftp/src/main/docs/sftp-component.adoc
+++ b/components/camel-ftp/src/main/docs/sftp-component.adoc
@@ -48,7 +48,7 @@ with the following path and query parameters:
| *directoryName* | The starting directory | | String
|===
-==== Query Parameters (110 parameters):
+==== Query Parameters (111 parameters):
[width="100%",cols="2,5,^1,2",options="header"]
|===
@@ -65,6 +65,7 @@ with the following path and query parameters:
| *moveFailed* (consumer) | Sets the move failure expression based on Simple language. For example to move files into a .error subdirectory use: .error. Note: When moving the files to the fail location Camel will handle the error and will not pick up the file again. | | String
| *noop* (consumer) | If true the file is not moved or deleted in any way. This option is good for readonly data or for ETL type requirements. If noop=true Camel will set idempotent=true as well to avoid consuming the same files over and over again. | false | boolean
| *preMove* (consumer) | Expression (such as File Language) used to dynamically set the filename when moving it before processing. For example to move in-progress files into the order directory set this value to order. | | String
+| *preSort* (consumer) | When pre-sort is enabled then the consumer will sort the file and directory names during polling that was retrieved from the file system. You may want to do this in case you need to operate on the files in a sorted order. The pre-sort is executed before the consumer starts to filter and accept files to process by Camel. This option is default=false meaning disabled. | false | boolean
| *recursive* (consumer) | If a directory will look for files in all the sub-directories as well. | false | boolean
| *sendEmptyMessageWhenIdle* (consumer) | If the polling consumer did not poll any files you can enable this option to send an empty message (no body) instead. | false | boolean
| *streamDownload* (consumer) | Sets the download method to use when not using a local working directory. If set to true the remote files are streamed to the route as they are read. When set to false the remote files are loaded into memory before being sent into the route. | false | boolean
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java
index 1d72438..33f153d 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java
@@ -17,7 +17,9 @@
package org.apache.camel.component.file.remote;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import java.util.stream.Collectors;
import org.apache.camel.Exchange;
import org.apache.camel.Message;
@@ -143,6 +145,11 @@ public class FtpConsumer extends RemoteFileConsumer<FTPFile> {
// we found some files
log.trace("Found {} in directory: {}", files.size(), dir);
}
+
+
+ if (getEndpoint().isPreSort()) {
+ Collections.sort(files, (a, b) -> a.getName().compareTo(b.getName()));
+ }
for (FTPFile file : files) {
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java
index 8eb3e23..c7a4a44 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java
@@ -17,7 +17,9 @@
package org.apache.camel.component.file.remote;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import java.util.stream.Collectors;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.SftpException;
@@ -142,6 +144,10 @@ public class SftpConsumer extends RemoteFileConsumer<SftpRemoteFile> {
// we found some files
log.trace("Found {} in directory: {}", files.size(), dir);
}
+
+ if (getEndpoint().isPreSort()) {
+ Collections.sort(files, (a, b) -> a.getFilename().compareTo(b.getFilename()));
+ }
for (SftpRemoteFile file : files) {
--
To stop receiving notification emails like this one, please contact
['"commits@camel.apache.org" <co...@camel.apache.org>'].