You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@commons.apache.org by Ke...@noaa.gov on 2008/03/22 07:31:52 UTC
[vfs] SFTP program not exiting
I'm trying to automate the downloading of data from a remote SFTP server
using VFS. My program gets the files, but doesn't exit, it seems to
hang. It's not clear what cleanup steps I should be doing after
accessing files. I haven't found any VFS + SFTP examples or tutorials.
Below is a maven 2 project file (pom.xml) and an example program
(App.java). Maybe my release() routine need to to have more in it?
If you want to try running the example to see what I'm having trouble with,
the filename pattern is looking for data files with names like
"smoke20070429_wkt.txt". Source, destination and connection information
is hardcoded. Modify the pattern, directories and connection information
for some file on one of your systems.
The pom.xml would go at the top level. The App.java is in a
subdirectory;
src/main/java/gov/noaa/eds/byExample/trySftpFileDownload/App.java
Compile with
mvn assembly:assembly
Execute with
java -jar
target/trySftpFileDownload-1.0-SNAPSHOT-jar-with-dependencies.jar
If I get a good solution, I can then document a working example on the
VFS wiki.
Thanks,
Ken
=== start pom.xml ===
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>gov.noaa.eds.byExample</groupId>
<artifactId>trySftpFileDownload</artifactId>
<packaging>jar</packaging>
<version>1.0-SNAPSHOT</version>
<name>trySftpFileDownload</name>
<url>http://maven.apache.org</url>
<build>
<extensions>
<extension>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-ssh-external</artifactId>
<version>1.0-beta-2</version>
</extension>
</extensions>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.5</source>
<target>1.5</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>gov.noaa.eds.byExample.trySftpFileDownload.App</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<configuration>
<tasks>
<java
classname="gov.noaa.eds.byExample.trySftpFileDownload.App"
classpathref="maven.runtime.classpath">
</java>
</tasks>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
<version>0.1.23</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.1</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
<version>2.0.2</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>1.4.1</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>commons-vfs</groupId>
<artifactId>commons-vfs</artifactId>
<version>1.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
=== end pom.xml ===
=== start App.java ===
package gov.noaa.eds.byExample.trySftpFileDownload;
import java.io.File;
import java.util.regex.Pattern;
import org.apache.commons.vfs.AllFileSelector;
import org.apache.commons.vfs.FileObject;
import org.apache.commons.vfs.FileSystemException;
import org.apache.commons.vfs.FileSystemManager;
import org.apache.commons.vfs.FileSystemOptions;
import org.apache.commons.vfs.FileType;
import org.apache.commons.vfs.UserAuthenticator;
import org.apache.commons.vfs.VFS;
import org.apache.commons.vfs.auth.StaticUserAuthenticator;
import org.apache.commons.vfs.impl.DefaultFileSystemConfigBuilder;
import org.apache.commons.vfs.provider.local.LocalFile;
/**
* Example use of sftp
*
*/
public class App {
private String localDir = "/extra/data/fires/smoke";
private File localDirFile;
private String remoteDir = "/mnt/data1/gdsg/data/fires/src-smoke";
// Look for files with names like "smoke20070429_wkt.txt"
private String filePatternString = ".*smoke\\d{8}_wkt\\.txt";
private Pattern filePattern;
private boolean recursive = false;
private boolean overwrite = false;
private FileSystemManager fsManager = null;
private FileSystemOptions opts = null;
private FileObject sftpFile;
private String host = "source.example.com"; // The remote SFTP hostname
private String user = "smokey"; // Remote system login name
private String password = "bear"; // Remote system password
public static void main(String[] args) {
System.out.println("SFTP download");
App app = new App();
app.initialize();
app.process();
app.release();
} // main( String[] args )
/**
* Creates the download directory localDir if it
* does not exist and makes a connection to the remote SFTP server.
*
*/
public void initialize() {
if (localDirFile == null) {
localDirFile = new File(localDir);
}
if (!this.localDirFile.exists()) {
localDirFile.mkdirs();
}
try {
this.fsManager = VFS.getManager();
} catch (FileSystemException ex) {
throw new RuntimeException("failed to get fsManager from
VFS", ex);
}
UserAuthenticator auth = new StaticUserAuthenticator(null,
this.user,
this.password);
this.opts = new FileSystemOptions();
try {
DefaultFileSystemConfigBuilder.getInstance().setUserAuthenticator(opts,
auth);
} catch (FileSystemException ex) {
throw new RuntimeException("setUserAuthenticator failed", ex);
}
this.filePattern = Pattern.compile(filePatternString);
} // initialize()
/**
* Retrieves files that match the specified FileSpec from the SFTP
server
* and stores them in the work directory.
*/
public void process() {
// Set starting path on remote SFTP server.
try {
String startPath = "sftp://" + this.host + this.remoteDir;
this.sftpFile = this.fsManager.resolveFile(startPath, opts);
System.out.println("SFTP connection successfully established
to " +
startPath);
} catch (FileSystemException ex) {
throw new RuntimeException("SFTP error parsing path " +
this.remoteDir,
ex);
}
try {
searchCurrentDirectory("", this.sftpFile);
} catch (FileSystemException ex) {
throw new RuntimeException("Error processing remote
directory", ex);
}
} // process(Object obj)
public void release() {
try {
this.sftpFile.close();
} catch (FileSystemException ex) {
throw new RuntimeException("Failed to close sftp file", ex);
}
} // release()
/**
* Search the current working directory of the SFTP client, saving files
* to the path specified by localDir + the path to the file on the SFTP
* server. This method will optionally recursively search
directories on
* the remote server.
*
* @param path relative path to the local Directory and also corresponds
* to the relative path from the remote remoteDir
* @param file remote VFS FileObject directory to examine
*/
private void searchCurrentDirectory(String path, FileObject file)
throws FileSystemException {
// Get a directory listing
FileObject[] children = file.getChildren();
search:
for (FileObject f : children) {
String relativePath = path + File.separatorChar +
f.getName().getBaseName();
System.out.println("relativePath=(.)" + relativePath);
if (f.getType() == FileType.FOLDER && this.recursive) {
System.out.println("Recursing into directory " +
f.getName());
searchCurrentDirectory(relativePath, f);
} else {
System.out.println("Examining remote file " + f.getName());
if (!
this.filePattern.matcher(f.getName().getPath()).matches()) {
System.out.println("Remote file " + f.getName() +
" failed filename pattern check ");
continue search;
}
boolean getFile = true;
String localUri = "file://" + this.localDir + relativePath;
String standardPath = this.localDir + relativePath;
System.out.println("Standard local path is " +
standardPath);
LocalFile localFile =
(LocalFile) this.fsManager.resolveFile(localUri);
System.out.println("resolved local file " +
localFile.getName());
if (localFile.exists()) {
if (this.overwrite) {
System.out.println("Replacing existing local
file " +
standardPath);
getFile = true;
} else {
System.out.println("Skipping download, local
file exists " +
standardPath);
continue search;
}
} else {
getFile = true;
}
if (getFile) {
if (!localFile.getParent().exists()) {
localFile.getParent().createFolder();
}
localFile.copyFrom(f, new AllFileSelector());
}
}
} // for (FileObject f : children)
} // searchCurrentDirectory(String path, FileObject file)
} // class App
=== end App.java ===
---------------------------------------------------------------------
To unsubscribe, e-mail: user-unsubscribe@commons.apache.org
For additional commands, e-mail: user-help@commons.apache.org
AW: [vfs] SFTP program not exiting
Posted by "Moerk, Detlev" <mo...@konradin.de>.
Hi Ken,
I think I had the same Problem before. I got a hint about manually closing the Filesystem.
That worked for me:
FileObject src = mgr.resolveFile(from);
FileSystem fs = null;
FileSystemManager fsm = null;
<-- do your things -->
src.close();
fs = src.getFileSystem();
fsm = fs.getFileSystemManager();
fsm.closeFileSystem(fs);
I didn't test your testcase, but Maybe that helps.
Regards
Detlev
Mit freundlichen Grüßen
Detlev Mörk
Systementwickler
Konradin Business GmbH
Ernst-Mey-Str. 8
70771 Leinfelden-Echterdingen
Tel.: +49 711/7594-504
Fax.: +49 711/7594-1504
Email: detlev.moerk@konradin.de
Konradin Business GmbH
Sitz der Gesellschaft: Ernst-Mey-Str. 8, 70771 Leinfelden-Echterdingen Amtsgericht Stuttgart HRB 224875
Geschäftsführer: Katja Kohlhammer, Peter Dilger
-----Ursprüngliche Nachricht-----
Von: Ken.Tanaka@noaa.gov [mailto:Ken.Tanaka@noaa.gov]
Gesendet: Samstag, 22. März 2008 07:32
An: "Jakarta Commons Users List <us...@commons.apache.org>"@nike.apache.org
Betreff: [vfs] SFTP program not exiting
I'm trying to automate the downloading of data from a remote SFTP server using VFS. My program gets the files, but doesn't exit, it seems to hang. It's not clear what cleanup steps I should be doing after accessing files. I haven't found any VFS + SFTP examples or tutorials.
Below is a maven 2 project file (pom.xml) and an example program (App.java). Maybe my release() routine need to to have more in it?
If you want to try running the example to see what I'm having trouble with, the filename pattern is looking for data files with names like "smoke20070429_wkt.txt". Source, destination and connection information is hardcoded. Modify the pattern, directories and connection information for some file on one of your systems.
The pom.xml would go at the top level. The App.java is in a subdirectory; src/main/java/gov/noaa/eds/byExample/trySftpFileDownload/App.java
Compile with
mvn assembly:assembly
Execute with
java -jar
target/trySftpFileDownload-1.0-SNAPSHOT-jar-with-dependencies.jar
If I get a good solution, I can then document a working example on the VFS wiki.
Thanks,
Ken
=== start pom.xml ===
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>gov.noaa.eds.byExample</groupId>
<artifactId>trySftpFileDownload</artifactId>
<packaging>jar</packaging>
<version>1.0-SNAPSHOT</version>
<name>trySftpFileDownload</name>
<url>http://maven.apache.org</url>
<build>
<extensions>
<extension>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-ssh-external</artifactId>
<version>1.0-beta-2</version>
</extension>
</extensions>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.5</source>
<target>1.5</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>gov.noaa.eds.byExample.trySftpFileDownload.App</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<configuration>
<tasks>
<java
classname="gov.noaa.eds.byExample.trySftpFileDownload.App"
classpathref="maven.runtime.classpath">
</java>
</tasks>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
<version>0.1.23</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.1</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
<version>2.0.2</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>1.4.1</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>commons-vfs</groupId>
<artifactId>commons-vfs</artifactId>
<version>1.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
=== end pom.xml ===
=== start App.java ===
package gov.noaa.eds.byExample.trySftpFileDownload;
import java.io.File;
import java.util.regex.Pattern;
import org.apache.commons.vfs.AllFileSelector;
import org.apache.commons.vfs.FileObject;
import org.apache.commons.vfs.FileSystemException;
import org.apache.commons.vfs.FileSystemManager;
import org.apache.commons.vfs.FileSystemOptions;
import org.apache.commons.vfs.FileType;
import org.apache.commons.vfs.UserAuthenticator;
import org.apache.commons.vfs.VFS;
import org.apache.commons.vfs.auth.StaticUserAuthenticator;
import org.apache.commons.vfs.impl.DefaultFileSystemConfigBuilder;
import org.apache.commons.vfs.provider.local.LocalFile;
/**
* Example use of sftp
*
*/
public class App {
private String localDir = "/extra/data/fires/smoke";
private File localDirFile;
private String remoteDir = "/mnt/data1/gdsg/data/fires/src-smoke";
// Look for files with names like "smoke20070429_wkt.txt"
private String filePatternString = ".*smoke\\d{8}_wkt\\.txt";
private Pattern filePattern;
private boolean recursive = false;
private boolean overwrite = false;
private FileSystemManager fsManager = null;
private FileSystemOptions opts = null;
private FileObject sftpFile;
private String host = "source.example.com"; // The remote SFTP hostname
private String user = "smokey"; // Remote system login name
private String password = "bear"; // Remote system password
public static void main(String[] args) {
System.out.println("SFTP download");
App app = new App();
app.initialize();
app.process();
app.release();
} // main( String[] args )
/**
* Creates the download directory localDir if it
* does not exist and makes a connection to the remote SFTP server.
*
*/
public void initialize() {
if (localDirFile == null) {
localDirFile = new File(localDir);
}
if (!this.localDirFile.exists()) {
localDirFile.mkdirs();
}
try {
this.fsManager = VFS.getManager();
} catch (FileSystemException ex) {
throw new RuntimeException("failed to get fsManager from VFS", ex);
}
UserAuthenticator auth = new StaticUserAuthenticator(null, this.user,
this.password);
this.opts = new FileSystemOptions();
try {
DefaultFileSystemConfigBuilder.getInstance().setUserAuthenticator(opts,
auth);
} catch (FileSystemException ex) {
throw new RuntimeException("setUserAuthenticator failed", ex);
}
this.filePattern = Pattern.compile(filePatternString);
} // initialize()
/**
* Retrieves files that match the specified FileSpec from the SFTP server
* and stores them in the work directory.
*/
public void process() {
// Set starting path on remote SFTP server.
try {
String startPath = "sftp://" + this.host + this.remoteDir;
this.sftpFile = this.fsManager.resolveFile(startPath, opts);
System.out.println("SFTP connection successfully established to " +
startPath);
} catch (FileSystemException ex) {
throw new RuntimeException("SFTP error parsing path " +
this.remoteDir,
ex);
}
try {
searchCurrentDirectory("", this.sftpFile);
} catch (FileSystemException ex) {
throw new RuntimeException("Error processing remote directory", ex);
}
} // process(Object obj)
public void release() {
try {
this.sftpFile.close();
} catch (FileSystemException ex) {
throw new RuntimeException("Failed to close sftp file", ex);
}
} // release()
/**
* Search the current working directory of the SFTP client, saving files
* to the path specified by localDir + the path to the file on the SFTP
* server. This method will optionally recursively search directories on
* the remote server.
*
* @param path relative path to the local Directory and also corresponds
* to the relative path from the remote remoteDir
* @param file remote VFS FileObject directory to examine
*/
private void searchCurrentDirectory(String path, FileObject file) throws FileSystemException {
// Get a directory listing
FileObject[] children = file.getChildren();
search:
for (FileObject f : children) {
String relativePath = path + File.separatorChar +
f.getName().getBaseName();
System.out.println("relativePath=(.)" + relativePath);
if (f.getType() == FileType.FOLDER && this.recursive) {
System.out.println("Recursing into directory " + f.getName());
searchCurrentDirectory(relativePath, f);
} else {
System.out.println("Examining remote file " + f.getName());
if (!
this.filePattern.matcher(f.getName().getPath()).matches()) {
System.out.println("Remote file " + f.getName() +
" failed filename pattern check ");
continue search;
}
boolean getFile = true;
String localUri = "file://" + this.localDir + relativePath;
String standardPath = this.localDir + relativePath;
System.out.println("Standard local path is " + standardPath);
LocalFile localFile =
(LocalFile) this.fsManager.resolveFile(localUri);
System.out.println("resolved local file " + localFile.getName());
if (localFile.exists()) {
if (this.overwrite) {
System.out.println("Replacing existing local file " +
standardPath);
getFile = true;
} else {
System.out.println("Skipping download, local file exists " +
standardPath);
continue search;
}
} else {
getFile = true;
}
if (getFile) {
if (!localFile.getParent().exists()) {
localFile.getParent().createFolder();
}
localFile.copyFrom(f, new AllFileSelector());
}
}
} // for (FileObject f : children)
} // searchCurrentDirectory(String path, FileObject file)
} // class App
=== end App.java ===
---------------------------------------------------------------------
To unsubscribe, e-mail: user-unsubscribe@commons.apache.org
For additional commands, e-mail: user-help@commons.apache.org
______________________________________________________________________
This email has been scanned by the MessageLabs Email Security System.
For more information please visit http://www.messagelabs.com/email ______________________________________________________________________
---------------------------------------------------------------------
To unsubscribe, e-mail: user-unsubscribe@commons.apache.org
For additional commands, e-mail: user-help@commons.apache.org