You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2018/01/08 10:20:57 UTC
[camel] 11/11: CAMEL-12127: camel-ftp - Add option to turn on
logging of transfer activity. Lets also see it in the JMX consumer so we
can monitor when it last downloaded something.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git
commit e8bb976d61f8e46ba71458197fce4dc3d8c7cb1b
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Mon Jan 8 11:16:03 2018 +0100
CAMEL-12127: camel-ftp - Add option to turn on logging of transfer activity. Lets also see it in the JMX consumer so we can monitor when it last downloaded something.
---
.../java/org/apache/camel/util/StringHelper.java | 16 +++++
.../org/apache/camel/util/StringHelperTest.java | 12 ++++
.../camel-ftp/src/main/docs/ftp-component.adoc | 3 +-
.../camel-ftp/src/main/docs/ftps-component.adoc | 3 +-
.../remote/DefaultFtpClientActivityListener.java | 74 ++++++++++++++++++----
.../camel/component/file/remote/FtpEndpoint.java | 22 ++++++-
.../camel/component/file/remote/FtpOperations.java | 3 +-
.../src/main/resources/ftp.properties | 2 +-
8 files changed, 116 insertions(+), 19 deletions(-)
diff --git a/camel-core/src/main/java/org/apache/camel/util/StringHelper.java b/camel-core/src/main/java/org/apache/camel/util/StringHelper.java
index a9ce41f..e028f9f 100644
--- a/camel-core/src/main/java/org/apache/camel/util/StringHelper.java
+++ b/camel-core/src/main/java/org/apache/camel/util/StringHelper.java
@@ -708,4 +708,20 @@ public final class StringHelper {
return false;
}
+ /**
+ * Outputs the bytes in human readable format in units of KB,MB,GB etc.
+ *
+ * @param bytes number of bytes
+ * @return human readable output
+ */
+ public static String humanReadableBytes(long bytes) {
+ int unit = 1024;
+ if (bytes < unit) {
+ return bytes + " B";
+ }
+ int exp = (int) (Math.log(bytes) / Math.log(unit));
+ String pre = "KMGTPE".charAt(exp - 1) + "";
+ return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
+ }
+
}
diff --git a/camel-core/src/test/java/org/apache/camel/util/StringHelperTest.java b/camel-core/src/test/java/org/apache/camel/util/StringHelperTest.java
index aa9c37e..85ec48c 100644
--- a/camel-core/src/test/java/org/apache/camel/util/StringHelperTest.java
+++ b/camel-core/src/test/java/org/apache/camel/util/StringHelperTest.java
@@ -248,4 +248,16 @@ public class StringHelperTest extends TestCase {
assertNull(StringHelper.trimToNull(" \t "));
assertNull(StringHelper.trimToNull(""));
}
+
+ public void testHumanReadableBytes() {
+ assertEquals("0 B", StringHelper.humanReadableBytes(0));
+ assertEquals("32 B", StringHelper.humanReadableBytes(32));
+ assertEquals("1.0 KB", StringHelper.humanReadableBytes(1024));
+ assertEquals("1.7 KB", StringHelper.humanReadableBytes(1730));
+ assertEquals("108.0 KB", StringHelper.humanReadableBytes(110592));
+ assertEquals("6.8 MB", StringHelper.humanReadableBytes(7077888));
+ assertEquals("432.0 MB", StringHelper.humanReadableBytes(452984832));
+ assertEquals("27.0 GB", StringHelper.humanReadableBytes(28991029248L));
+ assertEquals("1.7 TB", StringHelper.humanReadableBytes(1855425871872L));
+ }
}
diff --git a/components/camel-ftp/src/main/docs/ftp-component.adoc b/components/camel-ftp/src/main/docs/ftp-component.adoc
index c21b166..bab5a3c 100644
--- a/components/camel-ftp/src/main/docs/ftp-component.adoc
+++ b/components/camel-ftp/src/main/docs/ftp-component.adoc
@@ -97,7 +97,7 @@ with the following path and query parameters:
| *directoryName* | The starting directory | | String
|===
-==== Query Parameters (106 parameters):
+==== Query Parameters (107 parameters):
[width="100%",cols="2,5,^1,2",options="header"]
|===
@@ -109,6 +109,7 @@ with the following path and query parameters:
| *fileName* (common) | Use Expression such as File Language to dynamically set the filename. For consumers it's used as a filename filter. For producers it's used to evaluate the filename to write. If an expression is set it take precedence over the CamelFileName header. (Note: The header itself can also be an Expression). The expression options support both String and Expression types. If the expression is a String type it is always evaluated using the File Language. If the expression [...]
| *passiveMode* (common) | Sets passive mode connections. Default is active mode connections. | false | boolean
| *separator* (common) | Sets the path separator to be used. UNIX = Uses unix style path separator Windows = Uses windows style path separator Auto = (is default) Use existing path separator in file name | UNIX | PathSeparator
+| *transferLoggingInterval Seconds* (common) | Configures the interval in seconds to use when logging the progress of upload and download operations that are in-flight. This is used for logging progress when operations takes longer time. | 5 | int
| *transferLoggingLevel* (common) | Configure the logging level to use when logging the progress of upload and download operations. | DEBUG | LoggingLevel
| *transferLoggingVerbose* (common) | Configures whether the perform verbose (fine grained) logging of the progress of upload and download operations. | false | boolean
| *fastExistsCheck* (common) | If set this option to be true camel-ftp will use the list file directly to check if the file exists. Since some FTP server may not support to list the file directly if the option is false camel-ftp will use the old way to list the directory and check if the file exists. This option also influences readLock=changed to control whether it performs a fast check to update file information or not. This can be used to speed up the process if the FTP server has a l [...]
diff --git a/components/camel-ftp/src/main/docs/ftps-component.adoc b/components/camel-ftp/src/main/docs/ftps-component.adoc
index 414f774..2196fde 100644
--- a/components/camel-ftp/src/main/docs/ftps-component.adoc
+++ b/components/camel-ftp/src/main/docs/ftps-component.adoc
@@ -57,7 +57,7 @@ with the following path and query parameters:
| *directoryName* | The starting directory | | String
|===
-==== Query Parameters (114 parameters):
+==== Query Parameters (115 parameters):
[width="100%",cols="2,5,^1,2",options="header"]
|===
@@ -69,6 +69,7 @@ with the following path and query parameters:
| *fileName* (common) | Use Expression such as File Language to dynamically set the filename. For consumers it's used as a filename filter. For producers it's used to evaluate the filename to write. If an expression is set it take precedence over the CamelFileName header. (Note: The header itself can also be an Expression). The expression options support both String and Expression types. If the expression is a String type it is always evaluated using the File Language. If the expression [...]
| *passiveMode* (common) | Sets passive mode connections. Default is active mode connections. | false | boolean
| *separator* (common) | Sets the path separator to be used. UNIX = Uses unix style path separator Windows = Uses windows style path separator Auto = (is default) Use existing path separator in file name | UNIX | PathSeparator
+| *transferLoggingInterval Seconds* (common) | Configures the interval in seconds to use when logging the progress of upload and download operations that are in-flight. This is used for logging progress when operations takes longer time. | 5 | int
| *transferLoggingLevel* (common) | Configure the logging level to use when logging the progress of upload and download operations. | DEBUG | LoggingLevel
| *transferLoggingVerbose* (common) | Configures whether the perform verbose (fine grained) logging of the progress of upload and download operations. | false | boolean
| *fastExistsCheck* (common) | If set this option to be true camel-ftp will use the list file directly to check if the file exists. Since some FTP server may not support to list the file directly if the option is false camel-ftp will use the old way to list the directory and check if the file exists. This option also influences readLock=changed to control whether it performs a fast check to update file information or not. This can be used to speed up the process if the FTP server has a l [...]
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/DefaultFtpClientActivityListener.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/DefaultFtpClientActivityListener.java
index 333549f..9fea9cc 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/DefaultFtpClientActivityListener.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/DefaultFtpClientActivityListener.java
@@ -18,28 +18,38 @@ package org.apache.camel.component.file.remote;
import org.apache.camel.util.CamelLogger;
import org.apache.camel.util.ObjectHelper;
+import org.apache.camel.util.StopWatch;
+import org.apache.camel.util.StringHelper;
+import org.apache.camel.util.TimeUtils;
import org.apache.commons.net.io.CopyStreamEvent;
import org.apache.commons.net.io.CopyStreamListener;
public class DefaultFtpClientActivityListener implements FtpClientActivityListener, CopyStreamListener {
+ // TODO: allow to reconfigure level, interval, verbose etc via JMX
+
private final CamelLogger logger;
private final String host;
private final boolean verbose;
+ private final int intervalSeconds;
private boolean download = true;
private String fileName;
private long fileSize;
+ private String fileSizeText;
private String lastLogActivity;
private String lastVerboseLogActivity;
private long lastLogActivityTimestamp = -1;
private long lastVerboseLogActivityTimestamp = -1;
private long transferredBytes;
+ private final StopWatch watch = new StopWatch();
+ private final StopWatch interval = new StopWatch();
- public DefaultFtpClientActivityListener(CamelLogger logger, boolean verbose, String host) {
+ public DefaultFtpClientActivityListener(CamelLogger logger, boolean verbose, int intervalSeconds, String host) {
this.logger = logger;
- this.host = host;
this.verbose = verbose;
+ this.intervalSeconds = intervalSeconds;
+ this.host = host;
}
@Override
@@ -55,6 +65,7 @@ public class DefaultFtpClientActivityListener implements FtpClientActivityListen
@Override
public void setRemoteFileSize(long fileSize) {
this.fileSize = fileSize;
+ this.fileSizeText = StringHelper.humanReadableBytes(fileSize);
}
@Override
@@ -129,9 +140,11 @@ public class DefaultFtpClientActivityListener implements FtpClientActivityListen
@Override
public void onBeginDownloading(String host, String file) {
download = true;
- String msg = "Downloading from host: " + host + " file: " + file + " starting";
+ watch.restart();
+ interval.restart();
+ String msg = "Downloading from host: " + host + " file: " + file + " starting "; // add extra space to align with completed
if (fileSize > 0) {
- msg += " (file-size: " + fileSize + " bytes)";
+ msg += " (size: " + fileSizeText + ")";
}
doLog(msg);
}
@@ -142,24 +155,43 @@ public class DefaultFtpClientActivityListener implements FtpClientActivityListen
String msg = "Downloading from host: " + host + " file: " + file + " chunk (" + chunkSize + "/" + totalChunkSize + " bytes)";
if (fileSize > 0) {
- msg += " (file-size: " + fileSize + " bytes)";
+ float percent = ((float) totalChunkSize / (float) fileSize) * 100L;
+ String num = String.format("%.1f", percent);
+ // avoid 100.0 as its only done when we get the onDownloadComplete
+ if (totalChunkSize < fileSize && "100.0".equals(num)) {
+ num = "99.9";
+ }
+ msg += " (progress: " + num + "%)";
+ }
+ doLogVerbose(msg);
+ // however if the operation is slow then log once in a while
+ if (interval.taken() > intervalSeconds * 1000) {
+ doLog(msg);
+ interval.restart();
}
- doLog(msg);
}
@Override
public void onDownloadComplete(String host, String file) {
String msg = "Downloading from host: " + host + " file: " + file + " completed";
if (transferredBytes > 0) {
- msg += " (" + transferredBytes + " bytes)";
+ msg += " (size: " + StringHelper.humanReadableBytes(transferredBytes) + ")";
}
+ long taken = watch.taken();
+ String time = TimeUtils.printDuration(taken);
+ msg += " (took: " + time + ")";
doLog(msg);
}
@Override
public void onBeginUploading(String host, String file) {
download = false;
+ watch.restart();
+ interval.restart();
String msg = "Uploading to host: " + host + " file: " + file + " starting";
+ if (fileSize > 0) {
+ msg += " (size: " + fileSizeText + ")";
+ }
doLog(msg);
}
@@ -169,17 +201,32 @@ public class DefaultFtpClientActivityListener implements FtpClientActivityListen
String msg = "Uploading to host: " + host + " file: " + file + " chunk (" + chunkSize + "/" + totalChunkSize + " bytes)";
if (fileSize > 0) {
- msg += " (file-size: " + fileSize + " bytes)";
+ float percent = ((float) totalChunkSize / (float) fileSize) * 100L;
+ String num = String.format("%.1f", percent);
+ // avoid 100.0 as its only done when we get the onUploadComplete
+ if (totalChunkSize < fileSize && "100.0".equals(num)) {
+ num = "99.9";
+ }
+ msg += " (progress: " + num + "%)";
+ }
+ // each chunk is verbose
+ doLogVerbose(msg);
+ // however if the operation is slow then log once in a while
+ if (interval.taken() > intervalSeconds * 1000) {
+ doLog(msg);
+ interval.restart();
}
- doLog(msg);
}
@Override
public void onUploadComplete(String host, String file) {
String msg = "Uploading to host: " + host + " file: " + file + " completed";
if (transferredBytes > 0) {
- msg += " (" + transferredBytes + " bytes)";
+ msg += " (size: " + StringHelper.humanReadableBytes(transferredBytes) + ")";
}
+ long taken = watch.taken();
+ String time = TimeUtils.printDuration(taken);
+ msg += " (took: " + time + ")";
doLog(msg);
}
@@ -190,12 +237,11 @@ public class DefaultFtpClientActivityListener implements FtpClientActivityListen
@Override
public void bytesTransferred(long totalBytesTransferred, int bytesTransferred, long streamSize) {
- // if stream size is -1 from FTP client then use pre-calculated file size
- long size = streamSize > 0 ? streamSize : fileSize;
+ // stream size is always -1, so use pre-calculated fileSize instead
if (download) {
- onDownload(host, fileName, bytesTransferred, totalBytesTransferred, size);
+ onDownload(host, fileName, bytesTransferred, totalBytesTransferred, fileSize);
} else {
- onUpload(host, fileName, bytesTransferred, totalBytesTransferred, size);
+ onUpload(host, fileName, bytesTransferred, totalBytesTransferred, fileSize);
}
}
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpEndpoint.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpEndpoint.java
index 45b098a..00c169f 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpEndpoint.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpEndpoint.java
@@ -23,6 +23,7 @@ import org.apache.camel.FailedToCreateConsumerException;
import org.apache.camel.FailedToCreateProducerException;
import org.apache.camel.LoggingLevel;
import org.apache.camel.Processor;
+import org.apache.camel.api.management.ManagedAttribute;
import org.apache.camel.component.file.GenericFileConfiguration;
import org.apache.camel.component.file.GenericFileProducer;
import org.apache.camel.component.file.remote.RemoteFileConfiguration.PathSeparator;
@@ -56,6 +57,8 @@ public class FtpEndpoint<T extends FTPFile> extends RemoteFileEndpoint<FTPFile>
protected FTPClient ftpClient;
@UriParam(label = "common", defaultValue = "DEBUG")
protected LoggingLevel transferLoggingLevel = LoggingLevel.DEBUG;
+ @UriParam(label = "common", defaultValue = "5")
+ protected int transferLoggingIntervalSeconds = 5;
@UriParam(label = "common")
protected boolean transferLoggingVerbose;
@@ -257,7 +260,7 @@ public class FtpEndpoint<T extends FTPFile> extends RemoteFileEndpoint<FTPFile>
this.dataTimeout = dataTimeout;
}
-
+ @ManagedAttribute
public LoggingLevel getTransferLoggingLevel() {
return transferLoggingLevel;
}
@@ -265,10 +268,26 @@ public class FtpEndpoint<T extends FTPFile> extends RemoteFileEndpoint<FTPFile>
/**
* Configure the logging level to use when logging the progress of upload and download operations.
*/
+ @ManagedAttribute(description = "Logging level to use when logging the progress of upload and download operations")
public void setTransferLoggingLevel(LoggingLevel transferLoggingLevel) {
this.transferLoggingLevel = transferLoggingLevel;
}
+ @ManagedAttribute
+ public int getTransferLoggingIntervalSeconds() {
+ return transferLoggingIntervalSeconds;
+ }
+
+ /**
+ * Configures the interval in seconds to use when logging the progress of upload and download operations that are in-flight.
+ * This is used for logging progress when operations takes longer time.
+ */
+ @ManagedAttribute(description = "Interval in seconds to use when logging the progress of upload and download operations that are in-flight")
+ public void setTransferLoggingIntervalSeconds(int transferLoggingIntervalSeconds) {
+ this.transferLoggingIntervalSeconds = transferLoggingIntervalSeconds;
+ }
+
+ @ManagedAttribute
public boolean isTransferLoggingVerbose() {
return transferLoggingVerbose;
}
@@ -276,6 +295,7 @@ public class FtpEndpoint<T extends FTPFile> extends RemoteFileEndpoint<FTPFile>
/**
* Configures whether the perform verbose (fine grained) logging of the progress of upload and download operations.
*/
+ @ManagedAttribute(description = "Whether the perform verbose (fine grained) logging of the progress of upload and download operations")
public void setTransferLoggingVerbose(boolean transferLoggingVerbose) {
this.transferLoggingVerbose = transferLoggingVerbose;
}
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpOperations.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpOperations.java
index e456158..f19156b 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpOperations.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpOperations.java
@@ -70,7 +70,8 @@ public class FtpOperations implements RemoteFileOperations<FTPFile> {
this.endpoint = (FtpEndpoint<FTPFile>) endpoint;
// setup download listener/logger when we have the endpoint configured
transferLogger.setLevel(this.endpoint.getTransferLoggingLevel());
- this.clientActivityListener = new DefaultFtpClientActivityListener(transferLogger, this.endpoint.isTransferLoggingVerbose(), this.endpoint.getConfiguration().remoteServerInformation());
+ this.clientActivityListener = new DefaultFtpClientActivityListener(transferLogger, this.endpoint.isTransferLoggingVerbose(),
+ this.endpoint.getTransferLoggingIntervalSeconds(), this.endpoint.getConfiguration().remoteServerInformation());
}
public FtpClientActivityListener getClientActivityListener() {
diff --git a/examples/camel-example-ftp/src/main/resources/ftp.properties b/examples/camel-example-ftp/src/main/resources/ftp.properties
index fc42dc4..5047389 100644
--- a/examples/camel-example-ftp/src/main/resources/ftp.properties
+++ b/examples/camel-example-ftp/src/main/resources/ftp.properties
@@ -19,7 +19,7 @@
##ftp.client=ftp://changeme-to-ftp-server.com:21/mypath?autoCreate=false&username=bob&password=123
# this example is a local FTP server
-ftp.client=ftp://localhost:21?autoCreate=false&username=bob&password=123&passiveMode=true&transferLoggingLevel=INFO
+ftp.client=ftp://localhost:21?autoCreate=false&username=bob&password=123&passiveMode=true&transferLoggingLevel=INFO&transferLoggingIntervalSeconds=1
# for the server we want to delay 5 seconds between polling the server
# and keep the downloaded file as-is
--
To stop receiving notification emails like this one, please contact
"commits@camel.apache.org" <co...@camel.apache.org>.