You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2020/04/25 10:05:35 UTC
[camel] branch master updated (813d966 -> d15d559)
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git.
from 813d966 CAMEL-14947: camel-sftp: check for existance of remote directory using ls is very slow. Polished the prev PR and rename the option to a better name.
new 686fca6 CAMEL-14915: camel-ftp consumer should not try to create starting directory if its empty/home.
new d15d559 CAMEL-14947: camel-sftp: check for existance of remote directory using ls is very slow. Polished the prev PR and rename the option to a better name.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/camel/component/file/remote/sftp.json | 2 +-
.../camel-ftp/src/main/docs/sftp-component.adoc | 2 +-
.../camel/component/file/remote/FtpConsumer.java | 9 +--
.../component/file/remote/RemoteFileConsumer.java | 13 +++++
.../component/file/remote/SftpConfiguration.java | 8 +--
.../camel/component/file/remote/SftpConsumer.java | 9 +--
...ava => SftpSimpleConsumeNoStartingDirTest.java} | 25 ++++----
.../endpoint/dsl/SftpEndpointBuilderFactory.java | 66 ++++++++++------------
8 files changed, 74 insertions(+), 60 deletions(-)
copy components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/sftp/{SftpKeyUriConsumeFromClasspathTest.java => SftpSimpleConsumeNoStartingDirTest.java} (70%)
[camel] 01/02: CAMEL-14915: camel-ftp consumer should not try to
create starting directory if its empty/home.
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 686fca65ab92a828bb8359608d15a80d0085c434
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Sat Apr 25 11:43:23 2020 +0200
CAMEL-14915: camel-ftp consumer should not try to create starting directory if its empty/home.
---
.../camel/component/file/remote/FtpConsumer.java | 9 ++--
.../component/file/remote/RemoteFileConsumer.java | 13 +++++
.../camel/component/file/remote/SftpConsumer.java | 9 ++--
.../sftp/SftpSimpleConsumeNoStartingDirTest.java | 63 ++++++++++++++++++++++
4 files changed, 86 insertions(+), 8 deletions(-)
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java
index ccc7a70..c278947 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java
@@ -70,14 +70,15 @@ public class FtpConsumer extends RemoteFileConsumer<FTPFile> {
setStartScheduler(false);
try {
super.doStart();
- if (endpoint.isAutoCreate()) {
- LOG.debug("Auto creating directory: {}", endpoint.getConfiguration().getDirectory());
+ if (endpoint.isAutoCreate() && hasStartingDirectory()) {
+ String dir = endpoint.getConfiguration().getDirectory();
+ LOG.debug("Auto creating directory: {}", dir);
try {
connectIfNecessary();
- operations.buildDirectory(endpoint.getConfiguration().getDirectory(), true);
+ operations.buildDirectory(dir, true);
} catch (GenericFileOperationFailedException e) {
// log a WARN as we want to start the consumer.
- LOG.warn("Error auto creating directory: " + endpoint.getConfiguration().getDirectory() + " due " + e.getMessage() + ". This exception is ignored.", e);
+ LOG.warn("Error auto creating directory: " + dir + " due " + e.getMessage() + ". This exception is ignored.", e);
}
}
} finally {
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileConsumer.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileConsumer.java
index 63e2635..aaca933 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileConsumer.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileConsumer.java
@@ -28,6 +28,7 @@ import org.apache.camel.component.file.GenericFileConsumer;
import org.apache.camel.component.file.GenericFileOperationFailedException;
import org.apache.camel.component.file.GenericFileProcessStrategy;
import org.apache.camel.support.SynchronizationAdapter;
+import org.apache.camel.util.ObjectHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -150,6 +151,18 @@ public abstract class RemoteFileConsumer<T> extends GenericFileConsumer<T> {
return getEndpoint().isDownload();
}
+ /**
+ * Whether there is a starting directory configured.
+ */
+ protected boolean hasStartingDirectory() {
+ String dir = endpoint.getConfiguration().getDirectory();
+ if (ObjectHelper.isEmpty(dir)) {
+ return false;
+ }
+ // should not be a empty separator
+ return !dir.equals("/") && !dir.equals("\\");
+ }
+
@Override
protected void doStop() throws Exception {
super.doStop();
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java
index e2a3c8c..58529f8 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java
@@ -60,14 +60,15 @@ public class SftpConsumer extends RemoteFileConsumer<SftpRemoteFile> {
setStartScheduler(false);
try {
super.doStart();
- if (endpoint.isAutoCreate()) {
- LOG.debug("Auto creating directory: {}", endpoint.getConfiguration().getDirectory());
+ if (endpoint.isAutoCreate() && hasStartingDirectory()) {
+ String dir = endpoint.getConfiguration().getDirectory();
+ LOG.debug("Auto creating directory: {}", dir);
try {
connectIfNecessary();
- operations.buildDirectory(endpoint.getConfiguration().getDirectory(), true);
+ operations.buildDirectory(dir, true);
} catch (GenericFileOperationFailedException e) {
// log a WARN as we want to start the consumer.
- LOG.warn("Error auto creating directory: " + endpoint.getConfiguration().getDirectory() + " due " + e.getMessage() + ". This exception is ignored.", e);
+ LOG.warn("Error auto creating directory: " + dir + " due " + e.getMessage() + ". This exception is ignored.", e);
}
}
} finally {
diff --git a/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/sftp/SftpSimpleConsumeNoStartingDirTest.java b/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/sftp/SftpSimpleConsumeNoStartingDirTest.java
new file mode 100644
index 0000000..cfc6d22
--- /dev/null
+++ b/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/sftp/SftpSimpleConsumeNoStartingDirTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.file.remote.sftp;
+
+import java.io.File;
+import java.io.FileOutputStream;
+
+import org.apache.camel.Exchange;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.mock.MockEndpoint;
+import org.apache.camel.util.FileUtil;
+import org.apache.camel.util.IOHelper;
+import org.junit.jupiter.api.Test;
+
+public class SftpSimpleConsumeNoStartingDirTest extends SftpServerTestSupport {
+
+ @Test
+ public void testSftpSimpleConsume() throws Exception {
+ if (!canTest()) {
+ return;
+ }
+
+ // create files using regular file
+ File file = new File("a.txt");
+ FileOutputStream fos = new FileOutputStream(file, false);
+ fos.write("ABC".getBytes());
+ fos.close();
+
+ MockEndpoint mock = getMockEndpoint("mock:result");
+ mock.expectedMessageCount(1);
+
+ context.getRouteController().startRoute("foo");
+
+ assertMockEndpointsSatisfied();
+
+ FileUtil.deleteFile(file);
+ }
+
+ @Override
+ protected RouteBuilder createRouteBuilder() throws Exception {
+ return new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ from("sftp://localhost:" + getPort() + "/" + "?fileName=a.txt&username=admin&password=admin&delay=10000&disconnect=true").routeId("foo")
+ .noAutoStartup().to("log:result", "mock:result");
+ }
+ };
+ }
+}
[camel] 02/02: CAMEL-14947: camel-sftp: check for existance of
remote directory using ls is very slow. Polished the prev PR and rename the
option to a better name.
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git
commit d15d559848b0a210bf23405b70da88f554a5424b
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Sat Apr 25 12:05:00 2020 +0200
CAMEL-14947: camel-sftp: check for existance of remote directory using ls is very slow. Polished the prev PR and rename the option to a better name.
---
.../apache/camel/component/file/remote/sftp.json | 2 +-
.../camel-ftp/src/main/docs/sftp-component.adoc | 2 +-
.../component/file/remote/SftpConfiguration.java | 8 +--
.../endpoint/dsl/SftpEndpointBuilderFactory.java | 66 ++++++++++------------
4 files changed, 36 insertions(+), 42 deletions(-)
diff --git a/components/camel-ftp/src/generated/resources/org/apache/camel/component/file/remote/sftp.json b/components/camel-ftp/src/generated/resources/org/apache/camel/component/file/remote/sftp.json
index 386def2..156bb1e 100644
--- a/components/camel-ftp/src/generated/resources/org/apache/camel/component/file/remote/sftp.json
+++ b/components/camel-ftp/src/generated/resources/org/apache/camel/component/file/remote/sftp.json
@@ -78,7 +78,7 @@
"bulkRequests": { "kind": "parameter", "displayName": "Bulk Requests", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "secret": false, "configurationClass": "org.apache.camel.component.file.remote.SftpConfiguration", "configurationField": "configuration", "description": "Specifies how many requests may be outstanding at any one time. Increasing this value may slightly improve file transfer speed bu [...]
"compression": { "kind": "parameter", "displayName": "Compression", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "secret": false, "configurationClass": "org.apache.camel.component.file.remote.SftpConfiguration", "configurationField": "configuration", "description": "To use compression. Specify a level from 1 to 10. Important: You must manually add the needed JSCH zlib JAR to the classpath for compression support." },
"connectTimeout": { "kind": "parameter", "displayName": "Connect Timeout", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "secret": false, "defaultValue": "10000", "configurationClass": "org.apache.camel.component.file.remote.SftpConfiguration", "configurationField": "configuration", "description": "Sets the connect timeout for waiting for a connection to be established Used by both FTPClient and JSCH" },
- "existDirCheckUsingLs": { "kind": "parameter", "displayName": "Exist Dir Check Using Ls", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "secret": false, "defaultValue": "true", "configurationClass": "org.apache.camel.component.file.remote.SftpConfiguration", "configurationField": "configuration", "description": "Whether to check for existing directory using LS command instead of CD. By default CD is used wh [...]
+ "existDirCheckUsingLs": { "kind": "parameter", "displayName": "Exist Dir Check Using Ls", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "secret": false, "defaultValue": "true", "configurationClass": "org.apache.camel.component.file.remote.SftpConfiguration", "configurationField": "configuration", "description": "Whether to check for existing directory using LS command or CD. By default LS is used which is s [...]
"maximumReconnectAttempts": { "kind": "parameter", "displayName": "Maximum Reconnect Attempts", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "secret": false, "description": "Specifies the maximum reconnect attempts Camel performs when it tries to connect to the remote FTP server. Use 0 to disable this behavior." },
"proxy": { "kind": "parameter", "displayName": "Proxy", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "com.jcraft.jsch.Proxy", "deprecated": false, "secret": false, "description": "To use a custom configured com.jcraft.jsch.Proxy. This proxy is used to consume\/send messages from the target SFTP host." },
"reconnectDelay": { "kind": "parameter", "displayName": "Reconnect Delay", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "secret": false, "description": "Delay in millis Camel will wait before performing a reconnect attempt." },
diff --git a/components/camel-ftp/src/main/docs/sftp-component.adoc b/components/camel-ftp/src/main/docs/sftp-component.adoc
index bb3c5d2..552a324 100644
--- a/components/camel-ftp/src/main/docs/sftp-component.adoc
+++ b/components/camel-ftp/src/main/docs/sftp-component.adoc
@@ -123,7 +123,7 @@ with the following path and query parameters:
| *bulkRequests* (advanced) | Specifies how many requests may be outstanding at any one time. Increasing this value may slightly improve file transfer speed but will increase memory usage. | | Integer
| *compression* (advanced) | To use compression. Specify a level from 1 to 10. Important: You must manually add the needed JSCH zlib JAR to the classpath for compression support. | | int
| *connectTimeout* (advanced) | Sets the connect timeout for waiting for a connection to be established Used by both FTPClient and JSCH | 10000 | int
-| *existDirCheckUsingLs* (advanced) | Whether to check for existing directory using LS command instead of CD. By default CD is used which is faster but has been reported to maybe cause a problem on windows systems and therefore this option can be enabled to use LS. The LS command is slower as it transfers the full directory listing over the wire, which can be slow if the remote FTP server has many files in the directory. | true | boolean
+| *existDirCheckUsingLs* (advanced) | Whether to check for existing directory using LS command or CD. By default LS is used which is safer as otherwise Camel needs to change the directory back after checking. However LS has been reported to cause a problem on windows system in some situations and therefore you can disable this option to use CD. | true | boolean
| *maximumReconnectAttempts* (advanced) | Specifies the maximum reconnect attempts Camel performs when it tries to connect to the remote FTP server. Use 0 to disable this behavior. | | int
| *proxy* (advanced) | To use a custom configured com.jcraft.jsch.Proxy. This proxy is used to consume/send messages from the target SFTP host. | | Proxy
| *reconnectDelay* (advanced) | Delay in millis Camel will wait before performing a reconnect attempt. | | long
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConfiguration.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConfiguration.java
index 51523e1..5ac4487 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConfiguration.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConfiguration.java
@@ -338,10 +338,10 @@ public class SftpConfiguration extends RemoteFileConfiguration {
}
/**
- * Whether to check for existing directory using LS command instead of CD.
- * By default CD is used which is faster but has been reported to maybe cause a problem on windows systems
- * and therefore this option can be enabled to use LS. The LS command is slower as it transfers the full
- * directory listing over the wire, which can be slow if the remote FTP server has many files in the directory.
+ * Whether to check for existing directory using LS command or CD.
+ * By default LS is used which is safer as otherwise Camel needs to change the directory
+ * back after checking. However LS has been reported to cause a problem on windows system in some situations
+ * and therefore you can disable this option to use CD.
*/
public void setExistDirCheckUsingLs(boolean existDirCheckUsingLs) {
this.existDirCheckUsingLs = existDirCheckUsingLs;
diff --git a/core/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/SftpEndpointBuilderFactory.java b/core/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/SftpEndpointBuilderFactory.java
index c473d8a..3e107fb 100644
--- a/core/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/SftpEndpointBuilderFactory.java
+++ b/core/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/SftpEndpointBuilderFactory.java
@@ -2703,12 +2703,11 @@ public interface SftpEndpointBuilderFactory {
return this;
}
/**
- * Whether to check for existing directory using LS command instead of
- * CD. By default CD is used which is faster but has been reported to
- * maybe cause a problem on windows systems and therefore this option
- * can be enabled to use LS. The LS command is slower as it transfers
- * the full directory listing over the wire, which can be slow if the
- * remote FTP server has many files in the directory.
+ * Whether to check for existing directory using LS command or CD. By
+ * default LS is used which is safer as otherwise Camel needs to change
+ * the directory back after checking. However LS has been reported to
+ * cause a problem on windows system in some situations and therefore
+ * you can disable this option to use CD.
*
* The option is a: <code>boolean</code> type.
*
@@ -2721,12 +2720,11 @@ public interface SftpEndpointBuilderFactory {
return this;
}
/**
- * Whether to check for existing directory using LS command instead of
- * CD. By default CD is used which is faster but has been reported to
- * maybe cause a problem on windows systems and therefore this option
- * can be enabled to use LS. The LS command is slower as it transfers
- * the full directory listing over the wire, which can be slow if the
- * remote FTP server has many files in the directory.
+ * Whether to check for existing directory using LS command or CD. By
+ * default LS is used which is safer as otherwise Camel needs to change
+ * the directory back after checking. However LS has been reported to
+ * cause a problem on windows system in some situations and therefore
+ * you can disable this option to use CD.
*
* The option will be converted to a <code>boolean</code> type.
*
@@ -4185,12 +4183,11 @@ public interface SftpEndpointBuilderFactory {
return this;
}
/**
- * Whether to check for existing directory using LS command instead of
- * CD. By default CD is used which is faster but has been reported to
- * maybe cause a problem on windows systems and therefore this option
- * can be enabled to use LS. The LS command is slower as it transfers
- * the full directory listing over the wire, which can be slow if the
- * remote FTP server has many files in the directory.
+ * Whether to check for existing directory using LS command or CD. By
+ * default LS is used which is safer as otherwise Camel needs to change
+ * the directory back after checking. However LS has been reported to
+ * cause a problem on windows system in some situations and therefore
+ * you can disable this option to use CD.
*
* The option is a: <code>boolean</code> type.
*
@@ -4203,12 +4200,11 @@ public interface SftpEndpointBuilderFactory {
return this;
}
/**
- * Whether to check for existing directory using LS command instead of
- * CD. By default CD is used which is faster but has been reported to
- * maybe cause a problem on windows systems and therefore this option
- * can be enabled to use LS. The LS command is slower as it transfers
- * the full directory listing over the wire, which can be slow if the
- * remote FTP server has many files in the directory.
+ * Whether to check for existing directory using LS command or CD. By
+ * default LS is used which is safer as otherwise Camel needs to change
+ * the directory back after checking. However LS has been reported to
+ * cause a problem on windows system in some situations and therefore
+ * you can disable this option to use CD.
*
* The option will be converted to a <code>boolean</code> type.
*
@@ -5179,12 +5175,11 @@ public interface SftpEndpointBuilderFactory {
return this;
}
/**
- * Whether to check for existing directory using LS command instead of
- * CD. By default CD is used which is faster but has been reported to
- * maybe cause a problem on windows systems and therefore this option
- * can be enabled to use LS. The LS command is slower as it transfers
- * the full directory listing over the wire, which can be slow if the
- * remote FTP server has many files in the directory.
+ * Whether to check for existing directory using LS command or CD. By
+ * default LS is used which is safer as otherwise Camel needs to change
+ * the directory back after checking. However LS has been reported to
+ * cause a problem on windows system in some situations and therefore
+ * you can disable this option to use CD.
*
* The option is a: <code>boolean</code> type.
*
@@ -5197,12 +5192,11 @@ public interface SftpEndpointBuilderFactory {
return this;
}
/**
- * Whether to check for existing directory using LS command instead of
- * CD. By default CD is used which is faster but has been reported to
- * maybe cause a problem on windows systems and therefore this option
- * can be enabled to use LS. The LS command is slower as it transfers
- * the full directory listing over the wire, which can be slow if the
- * remote FTP server has many files in the directory.
+ * Whether to check for existing directory using LS command or CD. By
+ * default LS is used which is safer as otherwise Camel needs to change
+ * the directory back after checking. However LS has been reported to
+ * cause a problem on windows system in some situations and therefore
+ * you can disable this option to use CD.
*
* The option will be converted to a <code>boolean</code> type.
*