You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2018/01/08 10:20:56 UTC
[camel] 10/11: CAMEL-12127: Include file-size in file operations
API so we can know how big the files are to download/upload.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git
commit e9b65f72dcf82c4302810c1f76869119a9ab6c17
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Mon Jan 8 09:25:24 2018 +0100
CAMEL-12127: Include file-size in file operations API so we can know how big the files are to download/upload.
---
.../org/apache/camel/component/file/FileOperations.java | 6 +++---
.../apache/camel/component/file/GenericFileConsumer.java | 2 +-
.../camel/component/file/GenericFileOperations.java | 8 +++++---
.../apache/camel/component/file/GenericFileProducer.java | 2 +-
.../file/strategy/GenericFileDeleteProcessStrategy.java | 4 ++--
.../file/strategy/GenericFileProcessStrategySupport.java | 6 +++---
.../file/strategy/GenericFileRenameProcessStrategy.java | 2 +-
.../strategy/GenericFileDeleteProcessStrategyTest.java | 8 ++++----
.../file/remote/DefaultFtpClientActivityListener.java | 15 +++++++++++++--
.../component/file/remote/FtpClientActivityListener.java | 2 ++
.../apache/camel/component/file/remote/FtpOperations.java | 8 +++++---
.../camel/component/file/remote/SftpOperations.java | 6 +++---
.../org/apache/camel/component/scp/ScpOperations.java | 8 ++++----
.../camel-example-ftp/src/main/resources/ftp.properties | 4 ++--
14 files changed, 49 insertions(+), 32 deletions(-)
diff --git a/camel-core/src/main/java/org/apache/camel/component/file/FileOperations.java b/camel-core/src/main/java/org/apache/camel/component/file/FileOperations.java
index 2ee56fe..415f60b 100644
--- a/camel-core/src/main/java/org/apache/camel/component/file/FileOperations.java
+++ b/camel-core/src/main/java/org/apache/camel/component/file/FileOperations.java
@@ -202,17 +202,17 @@ public class FileOperations implements GenericFileOperations<File> {
return null;
}
- public boolean retrieveFile(String name, Exchange exchange) throws GenericFileOperationFailedException {
+ public boolean retrieveFile(String name, Exchange exchange, long size) throws GenericFileOperationFailedException {
// noop as we use type converters to read the body content for java.io.File
return true;
}
@Override
- public void releaseRetreivedFileResources(Exchange exchange) throws GenericFileOperationFailedException {
+ public void releaseRetrievedFileResources(Exchange exchange) throws GenericFileOperationFailedException {
// noop as we used type converters to read the body content for java.io.File
}
- public boolean storeFile(String fileName, Exchange exchange) throws GenericFileOperationFailedException {
+ public boolean storeFile(String fileName, Exchange exchange, long size) throws GenericFileOperationFailedException {
ObjectHelper.notNull(endpoint, "endpoint");
File file = new File(fileName);
diff --git a/camel-core/src/main/java/org/apache/camel/component/file/GenericFileConsumer.java b/camel-core/src/main/java/org/apache/camel/component/file/GenericFileConsumer.java
index 6a942c1..1cd922f 100644
--- a/camel-core/src/main/java/org/apache/camel/component/file/GenericFileConsumer.java
+++ b/camel-core/src/main/java/org/apache/camel/component/file/GenericFileConsumer.java
@@ -406,7 +406,7 @@ public abstract class GenericFileConsumer<T> extends ScheduledBatchPollingConsum
boolean retrieved;
Exception cause = null;
try {
- retrieved = operations.retrieveFile(name, exchange);
+ retrieved = operations.retrieveFile(name, exchange, target.getFileLength());
} catch (Exception e) {
retrieved = false;
cause = e;
diff --git a/camel-core/src/main/java/org/apache/camel/component/file/GenericFileOperations.java b/camel-core/src/main/java/org/apache/camel/component/file/GenericFileOperations.java
index 23db2a0..9d1b3c8 100644
--- a/camel-core/src/main/java/org/apache/camel/component/file/GenericFileOperations.java
+++ b/camel-core/src/main/java/org/apache/camel/component/file/GenericFileOperations.java
@@ -73,10 +73,11 @@ public interface GenericFileOperations<T> {
*
* @param name name of the file
* @param exchange stream to write the content of the file into
+ * @param size the total file size to retrieve, if possible to determine
* @return true if file has been retrieved, false if not
* @throws GenericFileOperationFailedException can be thrown
*/
- boolean retrieveFile(String name, Exchange exchange) throws GenericFileOperationFailedException;
+ boolean retrieveFile(String name, Exchange exchange, long size) throws GenericFileOperationFailedException;
/**
* Releases the resources consumed by a retrieved file
@@ -84,17 +85,18 @@ public interface GenericFileOperations<T> {
* @param exchange exchange with the content of the file
* @throws GenericFileOperationFailedException can be thrown
*/
- void releaseRetreivedFileResources(Exchange exchange) throws GenericFileOperationFailedException;
+ void releaseRetrievedFileResources(Exchange exchange) throws GenericFileOperationFailedException;
/**
* Stores the content as a new remote file (upload)
*
* @param name name of new file
* @param exchange with the content content of the file
+ * @param size the total file size to store, if possible to determine
* @return true if the file was stored, false if not
* @throws GenericFileOperationFailedException can be thrown
*/
- boolean storeFile(String name, Exchange exchange) throws GenericFileOperationFailedException;
+ boolean storeFile(String name, Exchange exchange, long size) throws GenericFileOperationFailedException;
/**
* Gets the current remote directory
diff --git a/camel-core/src/main/java/org/apache/camel/component/file/GenericFileProducer.java b/camel-core/src/main/java/org/apache/camel/component/file/GenericFileProducer.java
index ee40b91..2dec738 100644
--- a/camel-core/src/main/java/org/apache/camel/component/file/GenericFileProducer.java
+++ b/camel-core/src/main/java/org/apache/camel/component/file/GenericFileProducer.java
@@ -302,7 +302,7 @@ public class GenericFileProducer<T> extends DefaultProducer {
log.trace("About to write [{}] to [{}] from exchange [{}]", new Object[]{fileName, getEndpoint(), exchange});
}
- boolean success = operations.storeFile(fileName, exchange);
+ boolean success = operations.storeFile(fileName, exchange, -1);
if (!success) {
throw new GenericFileOperationFailedException("Error writing file [" + fileName + "]");
}
diff --git a/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileDeleteProcessStrategy.java b/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileDeleteProcessStrategy.java
index 2c6490a..f4aeaeb 100644
--- a/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileDeleteProcessStrategy.java
+++ b/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileDeleteProcessStrategy.java
@@ -61,7 +61,7 @@ public class GenericFileDeleteProcessStrategy<T> extends GenericFileProcessStrat
try {
deleteLocalWorkFile(exchange);
- operations.releaseRetreivedFileResources(exchange);
+ operations.releaseRetrievedFileResources(exchange);
int retries = 3;
boolean deleted = false;
@@ -101,7 +101,7 @@ public class GenericFileDeleteProcessStrategy<T> extends GenericFileProcessStrat
public void rollback(GenericFileOperations<T> operations, GenericFileEndpoint<T> endpoint, Exchange exchange, GenericFile<T> file) throws Exception {
try {
deleteLocalWorkFile(exchange);
- operations.releaseRetreivedFileResources(exchange);
+ operations.releaseRetrievedFileResources(exchange);
// moved the failed file if specifying the moveFailed option
if (failureRenamer != null) {
diff --git a/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileProcessStrategySupport.java b/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileProcessStrategySupport.java
index cbe125e..f38fe67 100644
--- a/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileProcessStrategySupport.java
+++ b/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileProcessStrategySupport.java
@@ -58,7 +58,7 @@ public abstract class GenericFileProcessStrategySupport<T> implements GenericFil
public void abort(GenericFileOperations<T> operations, GenericFileEndpoint<T> endpoint, Exchange exchange, GenericFile<T> file) throws Exception {
deleteLocalWorkFile(exchange);
- operations.releaseRetreivedFileResources(exchange);
+ operations.releaseRetrievedFileResources(exchange);
// must release lock last
if (exclusiveReadLockStrategy != null) {
@@ -68,7 +68,7 @@ public abstract class GenericFileProcessStrategySupport<T> implements GenericFil
public void commit(GenericFileOperations<T> operations, GenericFileEndpoint<T> endpoint, Exchange exchange, GenericFile<T> file) throws Exception {
deleteLocalWorkFile(exchange);
- operations.releaseRetreivedFileResources(exchange);
+ operations.releaseRetrievedFileResources(exchange);
// must release lock last
if (exclusiveReadLockStrategy != null) {
@@ -78,7 +78,7 @@ public abstract class GenericFileProcessStrategySupport<T> implements GenericFil
public void rollback(GenericFileOperations<T> operations, GenericFileEndpoint<T> endpoint, Exchange exchange, GenericFile<T> file) throws Exception {
deleteLocalWorkFile(exchange);
- operations.releaseRetreivedFileResources(exchange);
+ operations.releaseRetrievedFileResources(exchange);
// must release lock last
if (exclusiveReadLockStrategy != null) {
diff --git a/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileRenameProcessStrategy.java b/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileRenameProcessStrategy.java
index 6359f91..ada2a9c 100644
--- a/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileRenameProcessStrategy.java
+++ b/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileRenameProcessStrategy.java
@@ -63,7 +63,7 @@ public class GenericFileRenameProcessStrategy<T> extends GenericFileProcessStrat
@Override
public void rollback(GenericFileOperations<T> operations, GenericFileEndpoint<T> endpoint, Exchange exchange, GenericFile<T> file) throws Exception {
try {
- operations.releaseRetreivedFileResources(exchange);
+ operations.releaseRetrievedFileResources(exchange);
if (failureRenamer != null) {
// create a copy and bind the file to the exchange to be used by the renamer to evaluate the file name
diff --git a/camel-core/src/test/java/org/apache/camel/component/file/strategy/GenericFileDeleteProcessStrategyTest.java b/camel-core/src/test/java/org/apache/camel/component/file/strategy/GenericFileDeleteProcessStrategyTest.java
index ce72224..7367ebd 100644
--- a/camel-core/src/test/java/org/apache/camel/component/file/strategy/GenericFileDeleteProcessStrategyTest.java
+++ b/camel-core/src/test/java/org/apache/camel/component/file/strategy/GenericFileDeleteProcessStrategyTest.java
@@ -67,15 +67,15 @@ public class GenericFileDeleteProcessStrategyTest extends ContextTestSupport {
return false;
}
- public boolean retrieveFile(String name, Exchange exchange) throws GenericFileOperationFailedException {
+ public boolean retrieveFile(String name, Exchange exchange, long size) throws GenericFileOperationFailedException {
return false;
}
- @Override
- public void releaseRetreivedFileResources(Exchange exchange) throws GenericFileOperationFailedException {
+
+ public void releaseRetrievedFileResources(Exchange exchange) throws GenericFileOperationFailedException {
// No-op
}
- public boolean storeFile(String name, Exchange exchange) throws GenericFileOperationFailedException {
+ public boolean storeFile(String name, Exchange exchange, long size) throws GenericFileOperationFailedException {
return false;
}
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/DefaultFtpClientActivityListener.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/DefaultFtpClientActivityListener.java
index 4b45403..333549f 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/DefaultFtpClientActivityListener.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/DefaultFtpClientActivityListener.java
@@ -29,6 +29,7 @@ public class DefaultFtpClientActivityListener implements FtpClientActivityListen
private boolean download = true;
private String fileName;
+ private long fileSize;
private String lastLogActivity;
private String lastVerboseLogActivity;
private long lastLogActivityTimestamp = -1;
@@ -52,6 +53,11 @@ public class DefaultFtpClientActivityListener implements FtpClientActivityListen
}
@Override
+ public void setRemoteFileSize(long fileSize) {
+ this.fileSize = fileSize;
+ }
+
+ @Override
public String getLastLogActivity() {
return lastLogActivity;
}
@@ -124,6 +130,9 @@ public class DefaultFtpClientActivityListener implements FtpClientActivityListen
public void onBeginDownloading(String host, String file) {
download = true;
String msg = "Downloading from host: " + host + " file: " + file + " starting";
+ if (fileSize > 0) {
+ msg += " (file-size: " + fileSize + " bytes)";
+ }
doLog(msg);
}
@@ -181,10 +190,12 @@ public class DefaultFtpClientActivityListener implements FtpClientActivityListen
@Override
public void bytesTransferred(long totalBytesTransferred, int bytesTransferred, long streamSize) {
+ // if stream size is -1 from FTP client then use pre-calculated file size
+ long size = streamSize > 0 ? streamSize : fileSize;
if (download) {
- onDownload(host, fileName, bytesTransferred, totalBytesTransferred, streamSize);
+ onDownload(host, fileName, bytesTransferred, totalBytesTransferred, size);
} else {
- onUpload(host, fileName, bytesTransferred, totalBytesTransferred, streamSize);
+ onUpload(host, fileName, bytesTransferred, totalBytesTransferred, size);
}
}
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpClientActivityListener.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpClientActivityListener.java
index 07bb0e8..4fb346f 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpClientActivityListener.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpClientActivityListener.java
@@ -38,6 +38,8 @@ public interface FtpClientActivityListener extends CopyStreamListener {
void setRemoteFileName(String fileName);
+ void setRemoteFileSize(long size);
+
void onGeneralError(String host, String errorMessage);
void onConnecting(String host);
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpOperations.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpOperations.java
index ea3dc1d..e456158 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpOperations.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpOperations.java
@@ -351,10 +351,11 @@ public class FtpOperations implements RemoteFileOperations<FTPFile> {
}
}
- public boolean retrieveFile(String name, Exchange exchange) throws GenericFileOperationFailedException {
+ public boolean retrieveFile(String name, Exchange exchange, long size) throws GenericFileOperationFailedException {
// store the name of the file to download on the listener
clientActivityListener.setDownload(true);
clientActivityListener.setRemoteFileName(name);
+ clientActivityListener.setRemoteFileSize(size);
clientActivityListener.onBeginDownloading(endpoint.getConfiguration().remoteServerInformation(), name);
boolean answer;
@@ -379,7 +380,7 @@ public class FtpOperations implements RemoteFileOperations<FTPFile> {
}
@Override
- public void releaseRetreivedFileResources(Exchange exchange) throws GenericFileOperationFailedException {
+ public void releaseRetrievedFileResources(Exchange exchange) throws GenericFileOperationFailedException {
InputStream is = exchange.getIn().getHeader(RemoteFileComponent.REMOTE_FILE_INPUT_STREAM, InputStream.class);
if (is != null) {
@@ -557,13 +558,14 @@ public class FtpOperations implements RemoteFileOperations<FTPFile> {
return result;
}
- public boolean storeFile(String name, Exchange exchange) throws GenericFileOperationFailedException {
+ public boolean storeFile(String name, Exchange exchange, long size) throws GenericFileOperationFailedException {
// must normalize name first
name = endpoint.getConfiguration().normalizePath(name);
// store the name of the file to upload on the listener
clientActivityListener.setDownload(false);
clientActivityListener.setRemoteFileName(name);
+ clientActivityListener.setRemoteFileSize(size);
clientActivityListener.onBeginUploading(endpoint.getConfiguration().remoteServerInformation(), name);
log.trace("storeFile({})", name);
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpOperations.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpOperations.java
index 8604fdf..a06ad5c 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpOperations.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpOperations.java
@@ -646,7 +646,7 @@ public class SftpOperations implements RemoteFileOperations<SftpRemoteFile> {
}
}
- public synchronized boolean retrieveFile(String name, Exchange exchange) throws GenericFileOperationFailedException {
+ public synchronized boolean retrieveFile(String name, Exchange exchange, long size) throws GenericFileOperationFailedException {
LOG.trace("retrieveFile({})", name);
if (ObjectHelper.isNotEmpty(endpoint.getLocalWorkDirectory())) {
// local work directory is configured so we should store file content as files in this local directory
@@ -657,7 +657,7 @@ public class SftpOperations implements RemoteFileOperations<SftpRemoteFile> {
}
}
- public synchronized void releaseRetreivedFileResources(Exchange exchange) throws GenericFileOperationFailedException {
+ public synchronized void releaseRetrievedFileResources(Exchange exchange) throws GenericFileOperationFailedException {
InputStream is = exchange.getIn().getHeader(RemoteFileComponent.REMOTE_FILE_INPUT_STREAM, InputStream.class);
if (is != null) {
@@ -818,7 +818,7 @@ public class SftpOperations implements RemoteFileOperations<SftpRemoteFile> {
return true;
}
- public synchronized boolean storeFile(String name, Exchange exchange) throws GenericFileOperationFailedException {
+ public synchronized boolean storeFile(String name, Exchange exchange, long size) throws GenericFileOperationFailedException {
// must normalize name first
name = endpoint.getConfiguration().normalizePath(name);
diff --git a/components/camel-jsch/src/main/java/org/apache/camel/component/scp/ScpOperations.java b/components/camel-jsch/src/main/java/org/apache/camel/component/scp/ScpOperations.java
index efd1eca..3854f4a 100644
--- a/components/camel-jsch/src/main/java/org/apache/camel/component/scp/ScpOperations.java
+++ b/components/camel-jsch/src/main/java/org/apache/camel/component/scp/ScpOperations.java
@@ -83,17 +83,17 @@ public class ScpOperations implements RemoteFileOperations<ScpFile> {
}
@Override
- public boolean retrieveFile(String name, Exchange exchange) throws GenericFileOperationFailedException {
+ public boolean retrieveFile(String name, Exchange exchange, long isze) throws GenericFileOperationFailedException {
return false;
}
@Override
- public void releaseRetreivedFileResources(Exchange exchange) throws GenericFileOperationFailedException {
- // No-op
+ public void releaseRetrievedFileResources(Exchange exchange) throws GenericFileOperationFailedException {
+ // noop
}
@Override
- public boolean storeFile(String name, Exchange exchange) throws GenericFileOperationFailedException {
+ public boolean storeFile(String name, Exchange exchange, long size) throws GenericFileOperationFailedException {
ObjectHelper.notNull(session, "session");
ScpConfiguration cfg = endpoint.getConfiguration();
diff --git a/examples/camel-example-ftp/src/main/resources/ftp.properties b/examples/camel-example-ftp/src/main/resources/ftp.properties
index 2f1369a..fc42dc4 100644
--- a/examples/camel-example-ftp/src/main/resources/ftp.properties
+++ b/examples/camel-example-ftp/src/main/resources/ftp.properties
@@ -16,10 +16,10 @@
## ---------------------------------------------------------------------------
# NOTE: you may need to turn on passive mode via, passiveMode=true
-ftp.client=ftp://changeme-to-ftp-server.com:21/mypath?autoCreate=false&username=bob&password=123
+##ftp.client=ftp://changeme-to-ftp-server.com:21/mypath?autoCreate=false&username=bob&password=123
# this example is a local FTP server
-### ftp.client=ftp://localhost:21?autoCreate=false&username=bob&password=123&passiveMode=true&transferLoggingLevel=INFO
+ftp.client=ftp://localhost:21?autoCreate=false&username=bob&password=123&passiveMode=true&transferLoggingLevel=INFO
# for the server we want to delay 5 seconds between polling the server
# and keep the downloaded file as-is
--
To stop receiving notification emails like this one, please contact
"commits@camel.apache.org" <co...@camel.apache.org>.