You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2012/08/16 12:53:41 UTC
svn commit: r1373785 - in /camel/trunk/components/camel-ftp/src:
main/java/org/apache/camel/component/file/remote/
main/java/org/apache/camel/component/file/remote/strategy/
test/java/org/apache/camel/component/file/remote/
Author: davsclaus
Date: Thu Aug 16 10:53:41 2012
New Revision: 1373785
URL: http://svn.apache.org/viewvc?rev=1373785&view=rev
Log:
CAMEL-5512: Made the readLock=changed faster for ftp component if you enable the fastExistsCheck=true option as well. Notice that not all FTP servers support this.
Added:
camel/trunk/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/FtpChangedReadLockFastExistCheckTest.java
- copied, changed from r1373769, camel/trunk/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/FtpChangedReadLockTest.java
Modified:
camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileEndpoint.java
camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/FtpChangedExclusiveReadLockStrategy.java
camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/FtpProcessStrategyFactory.java
camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/SftpChangedExclusiveReadLockStrategy.java
camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/SftpProcessStrategyFactory.java
Modified: camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileEndpoint.java
URL: http://svn.apache.org/viewvc/camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileEndpoint.java?rev=1373785&r1=1373784&r2=1373785&view=diff
==============================================================================
--- camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileEndpoint.java (original)
+++ camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileEndpoint.java Thu Aug 16 10:53:41 2012
@@ -16,6 +16,8 @@
*/
package org.apache.camel.component.file.remote;
+import java.util.Map;
+
import org.apache.camel.Exchange;
import org.apache.camel.Processor;
import org.apache.camel.component.file.GenericFile;
@@ -109,6 +111,13 @@ public abstract class RemoteFileEndpoint
ObjectHelper.notEmpty(config.getProtocol(), "protocol");
}
+ @Override
+ protected Map<String, Object> getParamsAsMap() {
+ Map<String, Object> map = super.getParamsAsMap();
+ map.put("fastExistsCheck", fastExistsCheck);
+ return map;
+ }
+
/**
* Remote File Endpoints, impl this method to create a custom consumer specific to their "protocol" etc.
*
Modified: camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/FtpChangedExclusiveReadLockStrategy.java
URL: http://svn.apache.org/viewvc/camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/FtpChangedExclusiveReadLockStrategy.java?rev=1373785&r1=1373784&r2=1373785&view=diff
==============================================================================
--- camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/FtpChangedExclusiveReadLockStrategy.java (original)
+++ camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/FtpChangedExclusiveReadLockStrategy.java Thu Aug 16 10:53:41 2012
@@ -33,6 +33,7 @@ public class FtpChangedExclusiveReadLock
private long timeout;
private long checkInterval = 5000;
private long minLength = 1;
+ private boolean fastExistsCheck;
@Override
public void prepareOnStartup(GenericFileOperations<FTPFile> tGenericFileOperations, GenericFileEndpoint<FTPFile> tGenericFileEndpoint) throws Exception {
@@ -61,7 +62,18 @@ public class FtpChangedExclusiveReadLock
long newLastModified = 0;
long newLength = 0;
- List<FTPFile> files = operations.listFiles(file.getParent());
+ List<FTPFile> files;
+ if (fastExistsCheck) {
+ // use the absolute file path to only pickup the file we want to check, this avoids expensive
+ // list operations if we have a lot of files in the directory
+ LOG.trace("Using fast exists to update file information for {}", file);
+ files = operations.listFiles(file.getAbsoluteFilePath());
+ } else {
+ LOG.trace("Using full directory listing to update file information for {}. Consider enabling fastExistsCheck option.", file);
+ // fast option not enabled, so list the directory and filter the file name
+ files = operations.listFiles(file.getParent());
+ }
+ LOG.trace("List files {} found {} files", file.getAbsoluteFilePath(), files.size());
for (FTPFile f : files) {
if (f.getName().equals(file.getFileName())) {
newLastModified = f.getTimestamp().getTimeInMillis();
@@ -130,4 +142,12 @@ public class FtpChangedExclusiveReadLock
public void setMinLength(long minLength) {
this.minLength = minLength;
}
+
+ public boolean isFastExistsCheck() {
+ return fastExistsCheck;
+ }
+
+ public void setFastExistsCheck(boolean fastExistsCheck) {
+ this.fastExistsCheck = fastExistsCheck;
+ }
}
\ No newline at end of file
Modified: camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/FtpProcessStrategyFactory.java
URL: http://svn.apache.org/viewvc/camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/FtpProcessStrategyFactory.java?rev=1373785&r1=1373784&r2=1373785&view=diff
==============================================================================
--- camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/FtpProcessStrategyFactory.java (original)
+++ camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/FtpProcessStrategyFactory.java Thu Aug 16 10:53:41 2012
@@ -126,6 +126,10 @@ public final class FtpProcessStrategyFac
if (minLength != null) {
readLockStrategy.setMinLength(minLength);
}
+ Boolean fastExistsCheck = (Boolean) params.get("fastExistsCheck");
+ if (fastExistsCheck != null) {
+ readLockStrategy.setFastExistsCheck(fastExistsCheck);
+ }
return readLockStrategy;
}
}
Modified: camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/SftpChangedExclusiveReadLockStrategy.java
URL: http://svn.apache.org/viewvc/camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/SftpChangedExclusiveReadLockStrategy.java?rev=1373785&r1=1373784&r2=1373785&view=diff
==============================================================================
--- camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/SftpChangedExclusiveReadLockStrategy.java (original)
+++ camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/SftpChangedExclusiveReadLockStrategy.java Thu Aug 16 10:53:41 2012
@@ -25,6 +25,7 @@ import org.apache.camel.component.file.G
import org.apache.camel.component.file.GenericFileExclusiveReadLockStrategy;
import org.apache.camel.component.file.GenericFileOperations;
import org.apache.camel.util.StopWatch;
+import org.apache.commons.net.ftp.FTPFile;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,6 +34,7 @@ public class SftpChangedExclusiveReadLoc
private long timeout;
private long checkInterval = 5000;
private long minLength = 1;
+ private boolean fastExistsCheck;
@Override
public void prepareOnStartup(GenericFileOperations<ChannelSftp.LsEntry> tGenericFileOperations, GenericFileEndpoint<ChannelSftp.LsEntry> tGenericFileEndpoint) throws Exception {
@@ -61,7 +63,18 @@ public class SftpChangedExclusiveReadLoc
long newLastModified = 0;
long newLength = 0;
- List<ChannelSftp.LsEntry> files = operations.listFiles(file.getParent());
+ List<ChannelSftp.LsEntry> files;
+ if (fastExistsCheck) {
+ // use the absolute file path to only pickup the file we want to check, this avoids expensive
+ // list operations if we have a lot of files in the directory
+ LOG.trace("Using fast exists to update file information for {}", file);
+ files = operations.listFiles(file.getAbsoluteFilePath());
+ } else {
+ LOG.trace("Using full directory listing to update file information for {}. Consider enabling fastExistsCheck option.", file);
+ // fast option not enabled, so list the directory and filter the file name
+ files = operations.listFiles(file.getParent());
+ }
+ LOG.trace("List files {} found {} files", file.getAbsoluteFilePath(), files.size());
for (ChannelSftp.LsEntry f : files) {
if (f.getFilename().equals(file.getFileName())) {
newLastModified = f.getAttrs().getMTime();
@@ -130,4 +143,12 @@ public class SftpChangedExclusiveReadLoc
public void setMinLength(long minLength) {
this.minLength = minLength;
}
+
+ public boolean isFastExistsCheck() {
+ return fastExistsCheck;
+ }
+
+ public void setFastExistsCheck(boolean fastExistsCheck) {
+ this.fastExistsCheck = fastExistsCheck;
+ }
}
\ No newline at end of file
Modified: camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/SftpProcessStrategyFactory.java
URL: http://svn.apache.org/viewvc/camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/SftpProcessStrategyFactory.java?rev=1373785&r1=1373784&r2=1373785&view=diff
==============================================================================
--- camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/SftpProcessStrategyFactory.java (original)
+++ camel/trunk/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/strategy/SftpProcessStrategyFactory.java Thu Aug 16 10:53:41 2012
@@ -127,6 +127,10 @@ public final class SftpProcessStrategyFa
if (minLength != null) {
readLockStrategy.setMinLength(minLength);
}
+ Boolean fastExistsCheck = (Boolean) params.get("fastExistsCheck");
+ if (fastExistsCheck != null) {
+ readLockStrategy.setFastExistsCheck(fastExistsCheck);
+ }
return readLockStrategy;
}
}
Copied: camel/trunk/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/FtpChangedReadLockFastExistCheckTest.java (from r1373769, camel/trunk/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/FtpChangedReadLockTest.java)
URL: http://svn.apache.org/viewvc/camel/trunk/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/FtpChangedReadLockFastExistCheckTest.java?p2=camel/trunk/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/FtpChangedReadLockFastExistCheckTest.java&p1=camel/trunk/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/FtpChangedReadLockTest.java&r1=1373769&r2=1373785&rev=1373785&view=diff
==============================================================================
--- camel/trunk/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/FtpChangedReadLockTest.java (original)
+++ camel/trunk/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/FtpChangedReadLockFastExistCheckTest.java Thu Aug 16 10:53:41 2012
@@ -16,68 +16,13 @@
*/
package org.apache.camel.component.file.remote;
-import java.io.File;
-import java.io.FileOutputStream;
-
-import org.apache.camel.builder.RouteBuilder;
-import org.apache.camel.component.mock.MockEndpoint;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
/**
*
*/
-public class FtpChangedReadLockTest extends FtpServerTestSupport {
-
- private static final transient Logger LOG = LoggerFactory.getLogger(FtpChangedReadLockTest.class);
+public class FtpChangedReadLockFastExistCheckTest extends FtpChangedReadLockTest {
protected String getFtpUrl() {
- return "ftp://admin@localhost:" + getPort() + "/changed?password=admin&readLock=changed&readLockCheckInterval=1000&delete=true";
- }
-
- @Test
- public void testChangedReadLock() throws Exception {
- MockEndpoint mock = getMockEndpoint("mock:result");
- mock.expectedMessageCount(1);
- mock.expectedFileExists("target/changed/out/slowfile.dat");
-
- writeSlowFile();
-
- assertMockEndpointsSatisfied();
-
- String content = context.getTypeConverter().convertTo(String.class, new File("target/changed/out/slowfile.dat").getAbsoluteFile());
- String[] lines = content.split(LS);
- assertEquals("There should be 20 lines in the file", 20, lines.length);
- for (int i = 0; i < 20; i++) {
- assertEquals("Line " + i, lines[i]);
- }
- }
-
- private void writeSlowFile() throws Exception {
- LOG.debug("Writing slow file...");
-
- createDirectory(FTP_ROOT_DIR + "/changed");
- FileOutputStream fos = new FileOutputStream(FTP_ROOT_DIR + "/changed/slowfile.dat", true);
- for (int i = 0; i < 20; i++) {
- fos.write(("Line " + i + LS).getBytes());
- LOG.debug("Writing line " + i);
- Thread.sleep(200);
- }
-
- fos.flush();
- fos.close();
- LOG.debug("Writing slow file DONE...");
- }
-
- @Override
- protected RouteBuilder createRouteBuilder() throws Exception {
- return new RouteBuilder() {
- @Override
- public void configure() throws Exception {
- from(getFtpUrl()).to("file:target/changed/out", "mock:result");
- }
- };
+ return "ftp://admin@localhost:" + getPort() + "/changed?password=admin&readLock=changed&readLockCheckInterval=1000&delete=true&fastExistsCheck=true";
}
}