You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2017/11/01 09:19:28 UTC
[camel] 01/03: CAMEL-11971: SFTP consumer to support useList which
requires an abstraction of the remote file.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 9804d062eb380dc6610bb736a061699df3089907
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Wed Nov 1 09:39:44 2017 +0100
CAMEL-11971: SFTP consumer to support useList which requires an abstraction of the remote file.
---
.../camel/component/file/remote/SftpComponent.java | 9 ++--
.../camel/component/file/remote/SftpConsumer.java | 60 ++++++++++++----------
.../camel/component/file/remote/SftpEndpoint.java | 10 ++--
.../component/file/remote/SftpOperations.java | 17 +++---
.../component/file/remote/SftpRemoteFile.java | 54 +++++++++++++++++++
.../file/remote/SftpRemoteFileJCraft.java | 58 +++++++++++++++++++++
6 files changed, 159 insertions(+), 49 deletions(-)
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpComponent.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpComponent.java
index 625bc21..a018b87 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpComponent.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpComponent.java
@@ -19,14 +19,13 @@ package org.apache.camel.component.file.remote;
import java.net.URI;
import java.util.Map;
-import com.jcraft.jsch.ChannelSftp;
import org.apache.camel.CamelContext;
import org.apache.camel.component.file.GenericFileEndpoint;
/**
* Secure FTP Component
*/
-public class SftpComponent extends RemoteFileComponent<ChannelSftp.LsEntry> {
+public class SftpComponent extends RemoteFileComponent<SftpRemoteFile> {
public SftpComponent() {
setEndpointClass(SftpEndpoint.class);
@@ -38,12 +37,12 @@ public class SftpComponent extends RemoteFileComponent<ChannelSftp.LsEntry> {
}
@Override
- protected GenericFileEndpoint<ChannelSftp.LsEntry> buildFileEndpoint(String uri, String remaining, Map<String, Object> parameters) throws Exception {
+ protected GenericFileEndpoint<SftpRemoteFile> buildFileEndpoint(String uri, String remaining, Map<String, Object> parameters) throws Exception {
// get the base uri part before the options as they can be non URI valid such as the expression using $ chars
// and the URI constructor will regard $ as an illegal character and we dont want to enforce end users to
// to escape the $ for the expression (file language)
String baseUri = uri;
- if (uri.indexOf("?") != -1) {
+ if (uri.contains("?")) {
baseUri = uri.substring(0, uri.indexOf("?"));
}
@@ -56,7 +55,7 @@ public class SftpComponent extends RemoteFileComponent<ChannelSftp.LsEntry> {
return new SftpEndpoint(uri, this, config);
}
- protected void afterPropertiesSet(GenericFileEndpoint<ChannelSftp.LsEntry> endpoint) throws Exception {
+ protected void afterPropertiesSet(GenericFileEndpoint<SftpRemoteFile> endpoint) throws Exception {
// noop
}
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java
index 1b3262f..e42ada3 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java
@@ -32,13 +32,13 @@ import org.apache.camel.util.URISupport;
/**
* Secure FTP consumer
*/
-public class SftpConsumer extends RemoteFileConsumer<ChannelSftp.LsEntry> {
+public class SftpConsumer extends RemoteFileConsumer<SftpRemoteFile> {
private String endpointPath;
private transient String sftpConsumerToString;
- public SftpConsumer(RemoteFileEndpoint<ChannelSftp.LsEntry> endpoint, Processor processor, RemoteFileOperations<ChannelSftp.LsEntry> operations) {
+ public SftpConsumer(RemoteFileEndpoint<SftpRemoteFile> endpoint, Processor processor, RemoteFileOperations<SftpRemoteFile> operations) {
super(endpoint, processor, operations);
this.endpointPath = endpoint.getConfiguration().getDirectory();
}
@@ -70,7 +70,7 @@ public class SftpConsumer extends RemoteFileConsumer<ChannelSftp.LsEntry> {
}
@Override
- protected boolean pollDirectory(String fileName, List<GenericFile<ChannelSftp.LsEntry>> fileList, int depth) {
+ protected boolean pollDirectory(String fileName, List<GenericFile<SftpRemoteFile>> fileList, int depth) {
String currentDir = null;
if (isStepwise()) {
// must remember current dir so we stay in that directory after the poll
@@ -88,7 +88,7 @@ public class SftpConsumer extends RemoteFileConsumer<ChannelSftp.LsEntry> {
return answer;
}
- protected boolean pollSubDirectory(String absolutePath, String dirName, List<GenericFile<ChannelSftp.LsEntry>> fileList, int depth) {
+ protected boolean pollSubDirectory(String absolutePath, String dirName, List<GenericFile<SftpRemoteFile>> fileList, int depth) {
boolean answer = doSafePollSubDirectory(absolutePath, dirName, fileList, depth);
// change back to parent directory when finished polling sub directory
if (isStepwise()) {
@@ -97,7 +97,7 @@ public class SftpConsumer extends RemoteFileConsumer<ChannelSftp.LsEntry> {
return answer;
}
- protected boolean doPollDirectory(String absolutePath, String dirName, List<GenericFile<ChannelSftp.LsEntry>> fileList, int depth) {
+ protected boolean doPollDirectory(String absolutePath, String dirName, List<GenericFile<SftpRemoteFile>> fileList, int depth) {
log.trace("doPollDirectory from absolutePath: {}, dirName: {}", absolutePath, dirName);
depth++;
@@ -114,7 +114,7 @@ public class SftpConsumer extends RemoteFileConsumer<ChannelSftp.LsEntry> {
}
log.trace("Polling directory: {}", dir);
- List<ChannelSftp.LsEntry> files;
+ List<SftpRemoteFile> files;
if (isStepwise()) {
files = operations.listFiles();
} else {
@@ -129,10 +129,10 @@ public class SftpConsumer extends RemoteFileConsumer<ChannelSftp.LsEntry> {
log.trace("Found {} in directory: {}", files.size(), dir);
}
- for (ChannelSftp.LsEntry file : files) {
+ for (SftpRemoteFile file : files) {
if (log.isTraceEnabled()) {
- log.trace("SftpFile[fileName={}, longName={}, dir={}]", new Object[]{file.getFilename(), file.getLongname(), file.getAttrs().isDir()});
+ log.trace("SftpFile[fileName={}, longName={}, dir={}]", new Object[]{file.getFilename(), file.getLongname(), file.isDirectory()});
}
// check if we can continue polling in files
@@ -140,8 +140,8 @@ public class SftpConsumer extends RemoteFileConsumer<ChannelSftp.LsEntry> {
return false;
}
- if (file.getAttrs().isDir()) {
- RemoteFile<ChannelSftp.LsEntry> remote = asRemoteFile(absolutePath, file, getEndpoint().getCharset());
+ if (file.isDirectory()) {
+ RemoteFile<SftpRemoteFile> remote = asRemoteFile(absolutePath, file, getEndpoint().getCharset());
if (endpoint.isRecursive() && depth < endpoint.getMaxDepth() && isValidFile(remote, true, files)) {
// recursive scan and add the sub files and folders
String subDirectory = file.getFilename();
@@ -154,7 +154,7 @@ public class SftpConsumer extends RemoteFileConsumer<ChannelSftp.LsEntry> {
// we cannot use file.getAttrs().isLink on Windows, so we dont invoke the method
// just assuming its a file we should poll
} else {
- RemoteFile<ChannelSftp.LsEntry> remote = asRemoteFile(absolutePath, file, getEndpoint().getCharset());
+ RemoteFile<SftpRemoteFile> remote = asRemoteFile(absolutePath, file, getEndpoint().getCharset());
if (depth >= endpoint.getMinDepth() && isValidFile(remote, false, files)) {
// matched file so add
fileList.add(remote);
@@ -166,10 +166,10 @@ public class SftpConsumer extends RemoteFileConsumer<ChannelSftp.LsEntry> {
}
@Override
- protected boolean isMatched(GenericFile<ChannelSftp.LsEntry> file, String doneFileName, List<ChannelSftp.LsEntry> files) {
+ protected boolean isMatched(GenericFile<SftpRemoteFile> file, String doneFileName, List<SftpRemoteFile> files) {
String onlyName = FileUtil.stripPath(doneFileName);
- for (ChannelSftp.LsEntry f : files) {
+ for (SftpRemoteFile f : files) {
if (f.getFilename().equals(onlyName)) {
return true;
}
@@ -190,17 +190,17 @@ public class SftpConsumer extends RemoteFileConsumer<ChannelSftp.LsEntry> {
return super.ignoreCannotRetrieveFile(name, exchange, cause);
}
- private RemoteFile<ChannelSftp.LsEntry> asRemoteFile(String absolutePath, ChannelSftp.LsEntry file, String charset) {
- RemoteFile<ChannelSftp.LsEntry> answer = new RemoteFile<ChannelSftp.LsEntry>();
+ private RemoteFile<SftpRemoteFile> asRemoteFile(String absolutePath, SftpRemoteFile file, String charset) {
+ RemoteFile<SftpRemoteFile> answer = new RemoteFile<SftpRemoteFile>();
answer.setCharset(charset);
answer.setEndpointPath(endpointPath);
answer.setFile(file);
answer.setFileNameOnly(file.getFilename());
- answer.setFileLength(file.getAttrs().getSize());
- answer.setLastModified(file.getAttrs().getMTime() * 1000L);
+ answer.setFileLength(file.getFileLength());
+ answer.setLastModified(file.getLastModified());
answer.setHostname(((RemoteFileConfiguration) endpoint.getConfiguration()).getHost());
- answer.setDirectory(file.getAttrs().isDir());
+ answer.setDirectory(file.isDirectory());
// absolute or relative path
boolean absolute = FileUtil.hasLeadingSeparator(absolutePath);
@@ -228,16 +228,20 @@ public class SftpConsumer extends RemoteFileConsumer<ChannelSftp.LsEntry> {
}
@Override
- protected void updateFileHeaders(GenericFile<ChannelSftp.LsEntry> file, Message message) {
- long length = file.getFile().getAttrs().getSize();
- long modified = file.getFile().getAttrs().getMTime() * 1000L;
- file.setFileLength(length);
- file.setLastModified(modified);
- if (length >= 0) {
- message.setHeader(Exchange.FILE_LENGTH, length);
- }
- if (modified >= 0) {
- message.setHeader(Exchange.FILE_LAST_MODIFIED, modified);
+ protected void updateFileHeaders(GenericFile<SftpRemoteFile> file, Message message) {
+ Object rf = file.getFile().getRemoteFile();
+ if (rf != null) {
+ ChannelSftp.LsEntry e = (ChannelSftp.LsEntry) rf;
+ long length = e.getAttrs().getSize();
+ long modified = e.getAttrs().getMTime() * 1000L;
+ file.setFileLength(length);
+ file.setLastModified(modified);
+ if (length >= 0) {
+ message.setHeader(Exchange.FILE_LENGTH, length);
+ }
+ if (modified >= 0) {
+ message.setHeader(Exchange.FILE_LAST_MODIFIED, modified);
+ }
}
}
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpEndpoint.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpEndpoint.java
index 81d702c..1ee23e8 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpEndpoint.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpEndpoint.java
@@ -30,7 +30,7 @@ import org.apache.camel.spi.UriParam;
@UriEndpoint(firstVersion = "1.1.0", scheme = "sftp", extendsScheme = "file", title = "SFTP",
syntax = "sftp:host:port/directoryName", consumerClass = SftpConsumer.class, label = "file",
excludeProperties = "binary,passiveMode,receiveBufferSize,siteCommand,useList")
-public class SftpEndpoint extends RemoteFileEndpoint<ChannelSftp.LsEntry> {
+public class SftpEndpoint extends RemoteFileEndpoint<SftpRemoteFile> {
@UriParam
protected SftpConfiguration configuration;
@@ -61,15 +61,15 @@ public class SftpEndpoint extends RemoteFileEndpoint<ChannelSftp.LsEntry> {
}
@Override
- protected RemoteFileConsumer<ChannelSftp.LsEntry> buildConsumer(Processor processor) {
+ protected RemoteFileConsumer<SftpRemoteFile> buildConsumer(Processor processor) {
return new SftpConsumer(this, processor, createRemoteFileOperations());
}
- protected GenericFileProducer<ChannelSftp.LsEntry> buildProducer() {
- return new RemoteFileProducer<ChannelSftp.LsEntry>(this, createRemoteFileOperations());
+ protected GenericFileProducer<SftpRemoteFile> buildProducer() {
+ return new RemoteFileProducer<SftpRemoteFile>(this, createRemoteFileOperations());
}
- public RemoteFileOperations<ChannelSftp.LsEntry> createRemoteFileOperations() {
+ public RemoteFileOperations<SftpRemoteFile> createRemoteFileOperations() {
SftpOperations operations = new SftpOperations(proxy);
operations.setEndpoint(this);
return operations;
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpOperations.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpOperations.java
index 84d427f..50cb753 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpOperations.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpOperations.java
@@ -24,9 +24,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.security.KeyPair;
import java.security.interfaces.DSAPrivateKey;
import java.security.interfaces.DSAPublicKey;
@@ -46,7 +43,6 @@ import com.jcraft.jsch.Session;
import com.jcraft.jsch.SftpException;
import com.jcraft.jsch.UIKeyboardInteractive;
import com.jcraft.jsch.UserInfo;
-
import org.apache.camel.Exchange;
import org.apache.camel.InvalidPayloadException;
import org.apache.camel.LoggingLevel;
@@ -59,7 +55,6 @@ import org.apache.camel.util.FileUtil;
import org.apache.camel.util.IOHelper;
import org.apache.camel.util.ObjectHelper;
import org.apache.camel.util.ResourceHelper;
-
import org.apache.camel.util.StopWatch;
import org.apache.camel.util.TimeUtils;
import org.slf4j.Logger;
@@ -72,7 +67,7 @@ import static org.apache.camel.util.ObjectHelper.isNotEmpty;
* <p/>
* The JSCH session and channel are not thread-safe so we need to synchronize access to using this operation.
*/
-public class SftpOperations implements RemoteFileOperations<ChannelSftp.LsEntry> {
+public class SftpOperations implements RemoteFileOperations<SftpRemoteFile> {
private static final Logger LOG = LoggerFactory.getLogger(SftpOperations.class);
private static final Pattern UP_DIR_PATTERN = Pattern.compile("/[^/]+");
private Proxy proxy;
@@ -93,7 +88,7 @@ public class SftpOperations implements RemoteFileOperations<ChannelSftp.LsEntry>
public interface ExtendedUserInfo extends UserInfo, UIKeyboardInteractive {
}
- public void setEndpoint(GenericFileEndpoint<ChannelSftp.LsEntry> endpoint) {
+ public void setEndpoint(GenericFileEndpoint<SftpRemoteFile> endpoint) {
this.endpoint = (SftpEndpoint) endpoint;
}
@@ -623,11 +618,11 @@ public class SftpOperations implements RemoteFileOperations<ChannelSftp.LsEntry>
changeCurrentDirectory(parent);
}
- public synchronized List<ChannelSftp.LsEntry> listFiles() throws GenericFileOperationFailedException {
+ public synchronized List<SftpRemoteFile> listFiles() throws GenericFileOperationFailedException {
return listFiles(".");
}
- public synchronized List<ChannelSftp.LsEntry> listFiles(String path) throws GenericFileOperationFailedException {
+ public synchronized List<SftpRemoteFile> listFiles(String path) throws GenericFileOperationFailedException {
LOG.trace("listFiles({})", path);
if (ObjectHelper.isEmpty(path)) {
// list current directory if file path is not given
@@ -635,14 +630,14 @@ public class SftpOperations implements RemoteFileOperations<ChannelSftp.LsEntry>
}
try {
- final List<ChannelSftp.LsEntry> list = new ArrayList<ChannelSftp.LsEntry>();
+ final List<SftpRemoteFile> list = new ArrayList<>();
@SuppressWarnings("rawtypes")
Vector files = channel.ls(path);
// can return either null or an empty list depending on FTP servers
if (files != null) {
for (Object file : files) {
- list.add((ChannelSftp.LsEntry) file);
+ list.add(new SftpRemoteFileJCraft((ChannelSftp.LsEntry) file));
}
}
return list;
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpRemoteFile.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpRemoteFile.java
new file mode 100644
index 0000000..bdbfe92
--- /dev/null
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpRemoteFile.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.file.remote;
+
+/**
+ * A remote file when using SFTP.
+ */
+public interface SftpRemoteFile<T> {
+
+ /**
+ * Gets the remote file object.
+ */
+ T getRemoteFile();
+
+ /**
+ * The file name
+ */
+ String getFilename();
+
+ /**
+ * The long file name
+ */
+ String getLongname();
+
+ /**
+ * Whether its a directory
+ */
+ boolean isDirectory();
+
+ /**
+ * The file size
+ */
+ long getFileLength();
+
+ /**
+ * The file modification timestamp (in millis)
+ */
+ long getLastModified();
+
+}
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpRemoteFileJCraft.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpRemoteFileJCraft.java
new file mode 100644
index 0000000..15e9023
--- /dev/null
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpRemoteFileJCraft.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.file.remote;
+
+import com.jcraft.jsch.ChannelSftp;
+
+public class SftpRemoteFileJCraft implements SftpRemoteFile<ChannelSftp.LsEntry> {
+
+ private final ChannelSftp.LsEntry entry;
+
+ public SftpRemoteFileJCraft(ChannelSftp.LsEntry entry) {
+ this.entry = entry;
+ }
+
+ @Override
+ public ChannelSftp.LsEntry getRemoteFile() {
+ return entry;
+ }
+
+ @Override
+ public String getFilename() {
+ return entry.getFilename();
+ }
+
+ @Override
+ public String getLongname() {
+ return entry.getLongname();
+ }
+
+ @Override
+ public boolean isDirectory() {
+ return entry.getAttrs().isDir();
+ }
+
+ @Override
+ public long getFileLength() {
+ return entry.getAttrs().getSize();
+ }
+
+ @Override
+ public long getLastModified() {
+ return entry.getAttrs().getMTime() * 1000L;
+ }
+}
--
To stop receiving notification emails like this one, please contact
"commits@camel.apache.org" <co...@camel.apache.org>.