You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by ma...@apache.org on 2012/01/02 19:59:12 UTC
svn commit: r1226517 - in /oodt/trunk: ./ pushpull/
pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/
pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/renamingconventions/
pushpull/src/main/java/org/apache/oodt/cas/pushpul...
Author: mattmann
Date: Mon Jan 2 18:59:11 2012
New Revision: 1226517
URL: http://svn.apache.org/viewvc?rev=1226517&view=rev
Log:
- fix for OODT-367 Integrate CAS protocol into PushPull (thanks to bfoster for the review!)
Added:
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java
Removed:
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/Protocol.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolFactory.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolFile.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolFileFilter.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ftp/
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/http/
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/imaps/
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/local/
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/sftp/
Modified:
oodt/trunk/CHANGES.txt
oodt/trunk/pushpull/pom.xml
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/ProtocolInfo.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/renamingconventions/RenamingConvention.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/DataFileToPropFileLinker.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/DownloadListener.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RemoteFile.java
oodt/trunk/pushpull/src/main/resources/policy/ProtocolFactoryInfo.xml
Modified: oodt/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/oodt/trunk/CHANGES.txt?rev=1226517&r1=1226516&r2=1226517&view=diff
==============================================================================
--- oodt/trunk/CHANGES.txt (original)
+++ oodt/trunk/CHANGES.txt Mon Jan 2 18:59:11 2012
@@ -3,6 +3,9 @@ Apache OODT Change Log
Release 0.4: Current Development
--------------------------------------------
+
+* OODT-367 Integrate CAS protocol into PushPull (mattmann, bfoster)
+
* OODT-364 Prevent XSS attacks via malformed query string (ahart)
* OODT-363 Add support for LuceneQuery action to optionally return more than
Modified: oodt/trunk/pushpull/pom.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/pom.xml?rev=1226517&r1=1226516&r2=1226517&view=diff
==============================================================================
--- oodt/trunk/pushpull/pom.xml (original)
+++ oodt/trunk/pushpull/pom.xml Mon Jan 2 18:59:11 2012
@@ -112,30 +112,30 @@
<version>${project.parent.version}</version>
</dependency>
<dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- <version>1.3</version>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>cas-protocol-api</artifactId>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <version>3.0</version>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>cas-protocol-ftp</artifactId>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
- <groupId>commons-net</groupId>
- <artifactId>commons-net</artifactId>
- <version>2.2</version>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>cas-protocol-http</artifactId>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- <version>1.1</version>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>cas-protocol-imaps</artifactId>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>2.3</version>
- </dependency>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>cas-protocol-sftp</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
<dependency>
<groupId>xmlrpc</groupId>
<artifactId>xmlrpc</artifactId>
@@ -147,61 +147,6 @@
<version>2.0.8</version>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>1.2.13</version>
- </dependency>
- <dependency>
- <groupId>org.apache.geronimo.javamail</groupId>
- <artifactId>geronimo-javamail_1.4_mail</artifactId>
- <version>1.8.1</version>
- </dependency>
- <dependency>
- <groupId>org.apache.geronimo.specs</groupId>
- <artifactId>geronimo-activation_1.1_spec</artifactId>
- <version>1.1</version>
- </dependency>
- <dependency>
- <groupId>org.apache.pdfbox</groupId>
- <artifactId>pdfbox</artifactId>
- <version>1.2.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.poi</groupId>
- <artifactId>poi</artifactId>
- <version>3.0-FINAL</version>
- </dependency>
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-core</artifactId>
- <version>0.8</version>
- </dependency>
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-parsers</artifactId>
- <version>0.8</version>
- </dependency>
- <dependency>
- <groupId>nekohtml</groupId>
- <artifactId>nekohtml</artifactId>
- <version>1.9.6.2</version>
- </dependency>
- <dependency>
- <groupId>com.ibm.icu</groupId>
- <artifactId>icu4j</artifactId>
- <version>3.4.4</version>
- </dependency>
- <dependency>
- <groupId>org.globus</groupId>
- <artifactId>cog-jglobus</artifactId>
- <version>1.8.0</version>
- </dependency>
- <dependency>
- <groupId>com.jcraft</groupId>
- <artifactId>jsch</artifactId>
- <version>0.1.42</version>
- </dependency>
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.2</version>
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/ProtocolInfo.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/ProtocolInfo.java?rev=1226517&r1=1226516&r2=1226517&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/ProtocolInfo.java (original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/ProtocolInfo.java Mon Jan 2 18:59:11 2012
@@ -19,8 +19,8 @@
package org.apache.oodt.cas.pushpull.config;
//OODT imports
+import org.apache.oodt.cas.protocol.ProtocolFactory;
import org.apache.oodt.cas.pushpull.exceptions.ConfigException;
-import org.apache.oodt.cas.pushpull.protocol.ProtocolFactory;
import org.apache.oodt.cas.metadata.util.PathUtils;
import org.apache.oodt.commons.xml.XMLUtils;
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/renamingconventions/RenamingConvention.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/renamingconventions/RenamingConvention.java?rev=1226517&r1=1226516&r2=1226517&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/renamingconventions/RenamingConvention.java (original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/renamingconventions/RenamingConvention.java Mon Jan 2 18:59:11 2012
@@ -18,14 +18,16 @@
package org.apache.oodt.cas.pushpull.filerestrictions.renamingconventions;
-//OODT imports
+//JDK imports
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.oodt.cas.pushpull.protocol.ProtocolFile;
+//OODT imports
import org.apache.oodt.cas.metadata.util.PathUtils;
+import org.apache.oodt.cas.protocol.ProtocolFile;
+import org.apache.oodt.cas.pushpull.protocol.RemoteSiteFile;
/**
*
@@ -51,7 +53,7 @@ public class RenamingConvention {
* The file for which a unique name will be generated
* @return The unique file name (just the name).
*/
- public static String rename(ProtocolFile fileToGenNewNameFor,
+ public static String rename(RemoteSiteFile fileToGenNewNameFor,
String renamingString) {
try {
renamingString = grepReplace(renamingString, fileToGenNewNameFor);
@@ -61,15 +63,15 @@ public class RenamingConvention {
renamingString = replace(renamingString, "[PATH_NO_FILENAME]",
getParentPath(fileToGenNewNameFor));
renamingString = replace(renamingString, "[HOST]", fileToGenNewNameFor
- .getHostName());
+ .getSite().getURL().getHost());
renamingString = replace(renamingString, "[PARENT_FILENAME]",
getParentFileName(fileToGenNewNameFor));
renamingString = replace(renamingString, "[PARENT_PATH_NO_FILENAME]",
getGrandParentPath(fileToGenNewNameFor));
renamingString = replace(renamingString, "[URL]", fileToGenNewNameFor
- .getURL().toExternalForm());
+ .getSite().getURL().toExternalForm());
renamingString = replace(renamingString, "[IS_DIR]", String
- .valueOf(fileToGenNewNameFor.isDirectory()));
+ .valueOf(fileToGenNewNameFor.isDir()));
renamingString = PathUtils.doDynamicReplacement(renamingString);
}catch (Exception e) {
LOG.log(Level.WARNING, "Failed to rename " + fileToGenNewNameFor
@@ -79,7 +81,7 @@ public class RenamingConvention {
}
private static String grepReplace(String theString,
- ProtocolFile fileToGenNewNameFor) {
+ RemoteSiteFile fileToGenNewNameFor) {
Pattern grepPattern = Pattern.compile("\\[GREP\\(.*\\,.*\\)\\]");
Matcher grepMatcher = grepPattern.matcher(theString);
while (grepMatcher.find()) {
@@ -103,7 +105,7 @@ public class RenamingConvention {
}
private static String grepRemoveReplace(String theString,
- ProtocolFile fileToGenNewNameFor) {
+ RemoteSiteFile fileToGenNewNameFor) {
Pattern grepPattern = Pattern.compile("\\[GREP_RM\\(.*,.*\\)\\]");
Matcher grepMatcher = grepPattern.matcher(theString);
while (grepMatcher.find()) {
@@ -139,8 +141,7 @@ public class RenamingConvention {
private static String getParentPath(ProtocolFile fileToGenNewNameFor) {
String parentPath = "";
try {
- parentPath = fileToGenNewNameFor.getParentFile().getProtocolPath()
- .getPathString();
+ parentPath = fileToGenNewNameFor.getParent().getPath();
} catch (Exception e) {
}
return parentPath;
@@ -149,7 +150,7 @@ public class RenamingConvention {
private static String getParentFileName(ProtocolFile fileToGenNewNameFor) {
String parentFileName = "";
try {
- parentFileName = fileToGenNewNameFor.getParentFile().getName();
+ parentFileName = fileToGenNewNameFor.getParent().getName();
} catch (Exception e) {
}
return parentFileName;
@@ -158,8 +159,8 @@ public class RenamingConvention {
private static String getGrandParentPath(ProtocolFile fileToGenNewNameFor) {
String grandParentPath = "";
try {
- grandParentPath = fileToGenNewNameFor.getParentFile()
- .getParentFile().getProtocolPath().getPathString();
+ grandParentPath = fileToGenNewNameFor.getParent()
+ .getParent().getPath();
} catch (Exception e) {
}
return grandParentPath;
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java?rev=1226517&r1=1226516&r2=1226517&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java (original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java Mon Jan 2 18:59:11 2012
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -15,16 +15,18 @@
* limitations under the License.
*/
-
package org.apache.oodt.cas.pushpull.protocol;
//OODT imports
+import org.apache.oodt.cas.protocol.Protocol;
+import org.apache.oodt.cas.protocol.ProtocolFactory;
+import org.apache.oodt.cas.protocol.ProtocolFile;
+import org.apache.oodt.cas.pushpull.protocol.RemoteSiteFile;
import org.apache.oodt.cas.pushpull.config.ProtocolInfo;
-import org.apache.oodt.cas.pushpull.exceptions.ProtocolException;
+import org.apache.oodt.cas.protocol.auth.BasicAuthentication;
+import org.apache.oodt.cas.protocol.exceptions.ProtocolException;
+import org.apache.oodt.cas.protocol.util.ProtocolFileFilter;
import org.apache.oodt.cas.pushpull.exceptions.RemoteConnectionException;
-import org.apache.oodt.cas.pushpull.protocol.Protocol;
-import org.apache.oodt.cas.pushpull.protocol.ProtocolFactory;
-import org.apache.oodt.cas.pushpull.protocol.ProtocolFile;
//JDK imports
import java.io.File;
@@ -35,6 +37,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.Map.Entry;
+import java.util.Vector;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -54,506 +57,529 @@ import java.util.logging.Logger;
*/
public class ProtocolHandler {
- private HashMap<URL, ProtocolFactory> urlAndProtocolFactory;
+ private HashMap<URL, ProtocolFactory> urlAndProtocolFactory;
- private HashMap<URL, Protocol> reuseProtocols;
+ private HashMap<URL, Protocol> reuseProtocols;
- private HashMap<ProtocolFile, PagingInfo> pageInfos;
+ private HashMap<RemoteSiteFile, PagingInfo> pageInfos;
- private HashMap<ProtocolFile, List<ProtocolFile>> pathAndFileListMap;
+ private HashMap<RemoteSiteFile, List<RemoteSiteFile>> pathAndFileListMap;
- private ProtocolInfo pi;
-
- private static final Logger LOG = Logger.getLogger(ProtocolHandler.class
- .getName());
-
- /**
- * Creates a new ProtocolHandler for the given Config object
- *
- * @param config
- * The Config object that guides this ProtocolHandler in making
- * class instanciations
- */
- public ProtocolHandler(ProtocolInfo pi) {
- this.pi = pi;
- urlAndProtocolFactory = new HashMap<URL, ProtocolFactory>();
- reuseProtocols = new HashMap<URL, Protocol>();
- pageInfos = new HashMap<ProtocolFile, PagingInfo>();
- pathAndFileListMap = new HashMap<ProtocolFile, List<ProtocolFile>>();
- }
-
- /**
- * Returns the appropriate protocol for the given Path
- *
- * @param ProtocolPath
- * Used to determine the appropriate Protocol to be returned and
- * the path to navigate on if navigateToPathLoc is set to true.
- * @param allowReuse
- * Set to true if you would like ProtocolHandler to take care of
- * the protocol returned (i.e. reuseable protocols may be
- * returned by this method again, if it is the appropriate
- * protocol type for a given Path. Also ProtocolHandler will take
- * care of disconnecting the reuseable protocols)
- * @param navigateToPathLoc
- * If true, will navigate the to the end of the Path location
- * specified
- * @return Protocol for the given Path
- * @throws RemoteCommunicationException
- * If there is an error creating the protocol
- */
- public Protocol getAppropriateProtocol(ProtocolFile pFile,
- boolean allowReuse, boolean navigateToPathLoc)
- throws RemoteConnectionException {
- try {
- Protocol protocol = getAppropriateProtocol(pFile, allowReuse);
- if (protocol != null && navigateToPathLoc) {
- if (pFile.isDirectory())
- this.cd(protocol, pFile);
- else
- this.cd(protocol, pFile.getParentFile());
+ private ProtocolInfo pi;
+
+ private static final Logger LOG = Logger.getLogger(ProtocolHandler.class
+ .getName());
+
+ /**
+ * Creates a new ProtocolHandler for the given Config object
+ *
+ * @param config
+ * The Config object that guides this ProtocolHandler in making class
+ * instanciations
+ */
+ public ProtocolHandler(ProtocolInfo pi) {
+ this.pi = pi;
+ urlAndProtocolFactory = new HashMap<URL, ProtocolFactory>();
+ reuseProtocols = new HashMap<URL, Protocol>();
+ pageInfos = new HashMap<RemoteSiteFile, PagingInfo>();
+ pathAndFileListMap = new HashMap<RemoteSiteFile, List<RemoteSiteFile>>();
+ }
+
+ /**
+ * Returns the appropriate protocol for the given Path
+ *
+ * @param ProtocolPath
+ * Used to determine the appropriate Protocol to be returned and the
+ * path to navigate on if navigateToPathLoc is set to true.
+ * @param allowReuse
+ * Set to true if you would like ProtocolHandler to take care of the
+ * protocol returned (i.e. reuseable protocols may be returned by
+ * this method again, if it is the appropriate protocol type for a
+ * given Path. Also ProtocolHandler will take care of disconnecting
+ * the reuseable protocols)
+ * @param navigateToPathLoc
+ * If true, will navigate the to the end of the Path location
+ * specified
+ * @return Protocol for the given Path
+ * @throws RemoteCommunicationException
+ * If there is an error creating the protocol
+ */
+ public Protocol getAppropriateProtocol(RemoteSiteFile pFile,
+ boolean allowReuse, boolean navigateToPathLoc)
+ throws RemoteConnectionException {
+ try {
+ Protocol protocol = getAppropriateProtocol(pFile, allowReuse);
+ if (protocol != null && navigateToPathLoc) {
+ if (pFile.isDir())
+ this.cd(protocol, pFile);
+ else
+ this.cd(protocol, (RemoteSiteFile) pFile.getParent());
+ }
+ return protocol;
+ } catch (Exception e) {
+ throw new RemoteConnectionException(
+ "Failed to get appropriate protocol for " + pFile + " : "
+ + e.getMessage());
+ }
+ }
+
+ private Protocol getAppropriateProtocol(RemoteSiteFile pFile,
+ boolean allowReuse) throws ProtocolException, MalformedURLException {
+ return this.getAppropriateProtocolBySite(pFile.getSite(), allowReuse);
+ }
+
+ public Protocol getAppropriateProtocolBySite(RemoteSite remoteSite,
+ boolean allowReuse) throws ProtocolException {
+ Protocol protocol = null;
+ if ((allowReuse && ((protocol = reuseProtocols.get(remoteSite.getURL())) == null))
+ || !allowReuse) {
+ ProtocolFactory protocolFactory = this.urlAndProtocolFactory
+ .get(remoteSite.getURL());
+ if (protocolFactory == null) {
+ LinkedList<Class<ProtocolFactory>> protocolClasses = pi
+ .getProtocolClassesForProtocolType(remoteSite.getURL()
+ .getProtocol());
+ for (Class<ProtocolFactory> clazz : protocolClasses) {
+ try {
+ if ((protocol = (protocolFactory = clazz.newInstance())
+ .newInstance()) != null) {
+ if (!connect(protocol, remoteSite, true)) {
+ LOG.log(
+ Level.WARNING,
+ "ProtocolFactory "
+ + protocolFactory.getClass().getCanonicalName()
+ + " is not compatible with server at "
+ + remoteSite.getURL());
+ protocol = null;
+ } else {
+ this.urlAndProtocolFactory.put(remoteSite.getURL(),
+ protocolFactory);
+ break;
+ }
}
- return protocol;
- } catch (Exception e) {
- throw new RemoteConnectionException(
- "Failed to get appropriate protocol for " + pFile + " : "
- + e.getMessage());
- }
- }
-
- private Protocol getAppropriateProtocol(ProtocolFile pFile,
- boolean allowReuse) throws ProtocolException, MalformedURLException {
- return this.getAppropriateProtocolBySite(pFile.getRemoteSite(),
- allowReuse);
- }
-
- public Protocol getAppropriateProtocolBySite(RemoteSite remoteSite,
- boolean allowReuse) throws ProtocolException {
- Protocol protocol = null;
- if ((allowReuse && ((protocol = reuseProtocols.get(remoteSite.getURL())) == null))
- || !allowReuse) {
- ProtocolFactory protocolFactory = this.urlAndProtocolFactory
- .get(remoteSite.getURL());
- if (protocolFactory == null) {
- LinkedList<Class<ProtocolFactory>> protocolClasses = pi
- .getProtocolClassesForProtocolType(remoteSite.getURL()
- .getProtocol());
- for (Class<ProtocolFactory> clazz : protocolClasses) {
- try {
- if ((protocol = (protocolFactory = clazz.newInstance())
- .newInstance()) != null) {
- if (!connect(protocol, remoteSite, true)) {
- LOG.log(Level.WARNING, "ProtocolFactory "
- + protocolFactory.getClass()
- .getCanonicalName()
- + " is not compatible with server at "
- + remoteSite.getURL());
- protocol = null;
- } else {
- this.urlAndProtocolFactory.put(remoteSite
- .getURL(), protocolFactory);
- break;
- }
- }
- } catch (Exception e) {
- LOG.log(Level.WARNING,
- "Failed to instanciate protocol " + clazz
- + " for " + remoteSite.getURL());
- }
- }
- if (protocol == null)
- throw new ProtocolException(
- "Failed to get appropriate protocol for "
- + remoteSite);
- } else {
- connect(protocol = protocolFactory.newInstance(), remoteSite,
- false);
+ } catch (Exception e) {
+ LOG.log(Level.WARNING, "Failed to instanciate protocol " + clazz
+ + " for " + remoteSite.getURL());
+ }
+ }
+ if (protocol == null)
+ throw new ProtocolException("Failed to get appropriate protocol for "
+ + remoteSite);
+ } else {
+ connect(protocol = protocolFactory.newInstance(), remoteSite, false);
+ }
+ if (allowReuse)
+ this.reuseProtocols.put(remoteSite.getURL(), protocol);
+ }
+ return protocol;
+ }
+
+ public synchronized List<RemoteSiteFile> nextPage(Protocol protocol)
+ throws RemoteConnectionException, ProtocolException {
+ return nextPage(protocol, null);
+ }
+
+ /**
+ * @param protocol
+ * @return
+ * @throws RemoteConnectionException
+ * @throws ProtocolException
+ */
+ public synchronized List<RemoteSiteFile> nextPage(Protocol protocol,
+ ProtocolFileFilter filter) throws RemoteConnectionException,
+ ProtocolException {
+
+ PagingInfo pgInfo = this.getPagingInfo(this.pwd(protocol));
+ try {
+ System.out.println("PageSize: " + pi.getPageSize() + " PageLoc: "
+ + pgInfo.getPageLoc());
+ List<RemoteSiteFile> fileList = this.ls(protocol);
+ System.out.println("FileList size: " + fileList.size());
+
+ if (this.getDynamicFileList(protocol) == null
+ && !this.passesDynamicDetection(pgInfo, fileList)) {
+ LOG.log(
+ Level.SEVERE,
+ "Remote directory '"
+ + this.pwd(protocol)
+ + "' file list size has changed -- setting directory as dynamic and resetting page location");
+ this.putDynamicFileList(protocol, fileList);
+ pgInfo.updatePageInfo(0, fileList);
+ }
+
+ List<RemoteSiteFile> page = new LinkedList<RemoteSiteFile>();
+ int curLoc = pgInfo.getPageLoc();
+ for (; page.size() < pi.getPageSize() && curLoc < fileList.size(); curLoc++) {
+ if (filter == null || filter.accept(fileList.get(curLoc)))
+ page.add(fileList.get(curLoc));
+ }
+ pgInfo.updatePageInfo(curLoc, fileList);
+
+ return page;
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new RemoteConnectionException(
+ "Failed getting next page for protocol " + protocol + "-- pgStart = "
+ + pgInfo.getPageLoc() + " pgSize = " + pi.getPageSize() + " : "
+ + e.getMessage());
+ }
+
+ }
+
+ private boolean passesDynamicDetection(PagingInfo pgInfo,
+ List<RemoteSiteFile> newLS) throws MalformedURLException,
+ ProtocolException {
+ if (pgInfo.getSizeOfLastLS() != -1
+ && (pgInfo.getSizeOfLastLS() != newLS.size() || (newLS.size() != 0
+ && pgInfo.getPageLoc() < newLS.size() && (newLS.get(pgInfo
+ .getPageLoc()) == null || !newLS.get(pgInfo.getPageLoc()).equals(
+ pgInfo.getRemoteSiteFileAtPageLoc()))))) {
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ public void download(Protocol protocol, RemoteSiteFile fromFile, File toFile,
+ boolean delete) throws RemoteConnectionException {
+
+ // rename file for download
+ File downloadFile = new File(toFile.getParent() + "/Downloading_"
+ + toFile.getName());
+ toFile.renameTo(downloadFile);
+
+ LOG.log(Level.INFO, "Starting to download " + fromFile);
+ try {
+ // try to download the file
+ protocol.get(fromFile, downloadFile);
+
+ // delete file is specified
+ if (delete) {
+ if (!this.delete(protocol, fromFile))
+ LOG.log(Level.WARNING, "Failed to delete file '" + fromFile
+ + "' from server '" + fromFile.getSite() + "'");
+ else
+ LOG.log(Level.INFO, "Successfully deleted file '" + fromFile
+ + "' from server '" + fromFile.getSite() + "'");
+ }
+
+ LOG.log(Level.INFO, "Finished downloading " + fromFile + " to " + toFile);
+
+ // rename file back to original name
+ downloadFile.renameTo(toFile);
+
+ } catch (Exception e) {
+ downloadFile.delete();
+ throw new RemoteConnectionException("Failed to download file " + fromFile
+ + " : " + e.getMessage());
+ }
+ }
+
+ /**
+ * Connects the given Protocol to the given URL
+ *
+ * @param protocol
+ * The Protocol that will be connected
+ * @param url
+ * The server to which the Protocol will connect
+ * @throws RemoteConnectionException
+ * If connection fails
+ * @throws RemoteLoginException
+ * If login fails
+ */
+ public boolean connect(Protocol protocol, RemoteSite remoteSite, boolean test) {
+ for (int tries = 0; tries < 3; tries++) {
+
+ // wait for 5 secs before next retry
+ if (tries > 0) {
+ LOG.log(Level.INFO, "Will retry connecting to " + remoteSite
+ + " in 5 seconds");
+ synchronized (this) {
+ try {
+ System.out.print("Waiting");
+ for (int i = 0; i < 5; i++) {
+ System.out.print(" .");
+ wait(1000);
}
- if (allowReuse)
- this.reuseProtocols.put(remoteSite.getURL(), protocol);
+ System.out.println();
+ } catch (Exception e) {
+ }
}
- return protocol;
- }
-
- public synchronized List<ProtocolFile> nextPage(Protocol protocol)
- throws RemoteConnectionException, ProtocolException {
- return nextPage(protocol, null);
- }
-
- /**
- * @param protocol
- * @return
- * @throws RemoteConnectionException
- * @throws ProtocolException
- */
- public synchronized List<ProtocolFile> nextPage(Protocol protocol,
- ProtocolFileFilter filter) throws RemoteConnectionException,
- ProtocolException {
+ }
- PagingInfo pgInfo = this.getPagingInfo(this.pwd(protocol));
+ try {
+ // make sure protocol is disconnected
try {
- System.out.println("PageSize: " + pi.getPageSize() + " PageLoc: "
- + pgInfo.getPageLoc());
- List<ProtocolFile> fileList = this.ls(protocol);
- System.out.println("FileList size: " + fileList.size());
-
- if (this.getDynamicFileList(protocol) == null
- && !this.passesDynamicDetection(pgInfo, fileList)) {
- LOG
- .log(
- Level.SEVERE,
- "Remote directory '"
- + this.pwd(protocol)
- + "' file list size has changed -- setting directory as dynamic and resetting page location");
- this.putDynamicFileList(protocol, fileList);
- pgInfo.updatePageInfo(0, fileList);
- }
-
- List<ProtocolFile> page = new LinkedList<ProtocolFile>();
- int curLoc = pgInfo.getPageLoc();
- for (; page.size() < pi.getPageSize() && curLoc < fileList.size(); curLoc++) {
- if (filter == null || filter.accept(fileList.get(curLoc)))
- page.add(fileList.get(curLoc));
- }
- pgInfo.updatePageInfo(curLoc, fileList);
-
- return page;
+ protocol.close();
} catch (Exception e) {
- e.printStackTrace();
- throw new RemoteConnectionException(
- "Failed getting next page for protocol " + protocol
- + "-- pgStart = " + pgInfo.getPageLoc()
- + " pgSize = " + pi.getPageSize() + " : "
- + e.getMessage());
}
- }
-
- private boolean passesDynamicDetection(PagingInfo pgInfo,
- List<ProtocolFile> newLS) throws MalformedURLException,
- ProtocolException {
- if (pgInfo.getSizeOfLastLS() != -1
- && (pgInfo.getSizeOfLastLS() != newLS.size() || (newLS.size() != 0
- && pgInfo.getPageLoc() < newLS.size() && (newLS
- .get(pgInfo.getPageLoc()) == null || !newLS.get(
- pgInfo.getPageLoc()).equals(
- pgInfo.getProtocolFileAtPageLoc()))))) {
- return false;
- } else {
- return true;
- }
- }
-
- public void download(Protocol protocol, ProtocolFile fromFile, File toFile,
- boolean delete) throws RemoteConnectionException {
-
- // rename file for download
- File downloadFile = new File(toFile.getParent() + "/Downloading_"
- + toFile.getName());
- toFile.renameTo(downloadFile);
-
- LOG.log(Level.INFO, "Starting to download " + fromFile);
- try {
- // try to download the file
- protocol.download(fromFile, downloadFile);
-
- // delete file is specified
- if (delete) {
- if (!this.delete(protocol, fromFile))
- LOG.log(Level.WARNING, "Failed to delete file '" + fromFile
- + "' from server '" + protocol.getRemoteSite()
- + "'");
- else
- LOG.log(Level.INFO, "Successfully deleted file '"
- + fromFile + "' from server '"
- + protocol.getRemoteSite() + "'");
- }
-
- LOG.log(Level.INFO, "Finished downloading " + fromFile + " to "
- + toFile);
-
- // rename file back to original name
- downloadFile.renameTo(toFile);
-
- } catch (Exception e) {
- downloadFile.delete();
- throw new RemoteConnectionException("Failed to download file "
- + fromFile + " : " + e.getMessage());
- }
- }
-
- /**
- * Connects the given Protocol to the given URL
- *
- * @param protocol
- * The Protocol that will be connected
- * @param url
- * The server to which the Protocol will connect
- * @throws RemoteConnectionException
- * If connection fails
- * @throws RemoteLoginException
- * If login fails
- */
- public boolean connect(Protocol protocol, RemoteSite remoteSite,
- boolean test) {
- for (int tries = 0; tries < 3; tries++) {
-
- // wait for 5 secs before next retry
- if (tries > 0) {
- LOG.log(Level.INFO, "Will retry connecting to " + remoteSite
- + " in 5 seconds");
- synchronized (this) {
- try {
- System.out.print("Waiting");
- for (int i = 0; i < 5; i++) {
- System.out.print(" .");
- wait(1000);
- }
- System.out.println();
- } catch (Exception e) {
- }
- }
- }
-
- try {
- // make sure protocol is disconnected
- try {
- protocol.disconnect();
- } catch (Exception e) {
- }
-
- // try connecting Protocol
- protocol.connect(remoteSite);
-
- // check connection
- if (protocol.isConnected() && (!test || isOkProtocol(protocol, remoteSite))) {
- LOG.log(Level.INFO, "Successfully connected to "
- + remoteSite.getURL() + " with protocol '"
- + protocol.getClass().getCanonicalName()
- + "' and username '" + remoteSite.getUsername()
- + "'");
- return true;
- } else
- return false;
-
- } catch (Exception e) {
- LOG.log(Level.WARNING, "Error occurred while connecting to "
- + remoteSite + " : " + e.getMessage());
- }
-
- }
- return false;
- }
-
- private boolean isOkProtocol(Protocol protocol, RemoteSite remoteSite) {
- try {
- LOG.log(Level.INFO, "Testing protocol "
- + protocol.getClass().getCanonicalName()
- + " . . . this may take a few minutes . . .");
- // test ls, cd, and pwd
- this.cdToHOME(protocol);
- ProtocolFile home = this.pwd(protocol);
- this.ls(protocol);
- if (remoteSite.getCdTestDir() != null)
- this.cd(protocol, new ProtocolFile(remoteSite,
- new ProtocolPath(remoteSite.getCdTestDir(), true)));
- else
- this.cdToROOT(protocol);
- this.cdToHOME(protocol);
- if (home == null || !home.equals(protocol.pwd()))
- throw new ProtocolException(
- "Home directory not the same after cd");
- } catch (Exception e) {
- LOG.log(Level.SEVERE, "Protocol "
- + protocol.getClass().getCanonicalName()
- + " failed compatibility test : " + e.getMessage());
- return false;
- }
+ // try connecting Protocol
+ protocol.connect(
+ remoteSite.getURL().getHost(),
+ new BasicAuthentication(remoteSite.getUsername(), remoteSite
+ .getPassword()));
+
+ // check connection
+ if (protocol.connected()
+ && (!test || isOkProtocol(protocol, remoteSite))) {
+ LOG.log(Level.INFO,
+ "Successfully connected to " + remoteSite.getURL()
+ + " with protocol '" + protocol.getClass().getCanonicalName()
+ + "' and username '" + remoteSite.getUsername() + "'");
+ return true;
+ } else
+ return false;
+
+ } catch (Exception e) {
+ LOG.log(Level.WARNING, "Error occurred while connecting to "
+ + remoteSite + " : " + e.getMessage());
+ }
+
+ }
+ return false;
+ }
+
+ private boolean isOkProtocol(Protocol protocol, RemoteSite remoteSite) {
+ try {
+ LOG.log(Level.INFO, "Testing protocol "
+ + protocol.getClass().getCanonicalName()
+ + " . . . this may take a few minutes . . .");
+ // test ls, cd, and pwd
+ this.cdToHOME(protocol);
+ RemoteSiteFile home = this.pwd(protocol);
+ this.ls(protocol);
+ if (remoteSite.getCdTestDir() != null)
+ this.cd(protocol, new RemoteSiteFile(home, remoteSite.getCdTestDir(),
+ true, remoteSite));
+ else
+ this.cdToROOT(protocol);
+ this.cdToHOME(protocol);
+ if (home == null || !home.equals(protocol.pwd()))
+ throw new ProtocolException("Home directory not the same after cd");
+ } catch (Exception e) {
+ LOG.log(Level.SEVERE, "Protocol "
+ + protocol.getClass().getCanonicalName()
+ + " failed compatibility test : " + e.getMessage());
+ return false;
+ }
+ return true;
+ }
+
+ public void cdToROOT(Protocol protocol) throws ProtocolException {
+ protocol.cdRoot();
+ }
+
+ public void cdToHOME(Protocol protocol) throws ProtocolException {
+ protocol.cdHome();
+ }
+
+ public boolean isProtocolConnected(Protocol protocol)
+ throws ProtocolException {
+ return protocol.connected();
+ }
+
+ public void cd(Protocol protocol, RemoteSiteFile file)
+ throws ProtocolException {
+ protocol.cd(file);
+ }
+
+ public RemoteSiteFile getProtocolFileFor(Protocol protocol, String file,
+ boolean isDir) throws ProtocolException {
+ return this.getProtocolFileByProtocol(protocol, file, isDir);
+ }
+
+ public synchronized boolean delete(Protocol protocol, RemoteSiteFile file)
+ throws MalformedURLException, ProtocolException {
+ try {
+ PagingInfo pgInfo = this.getPagingInfo(file.getRemoteParent());
+ List<RemoteSiteFile> fileList = this.ls(protocol, file.getRemoteParent());
+ int indexOfFile = fileList.indexOf(file);
+ if (indexOfFile != -1) {
+ protocol.delete(file);
+ fileList.remove(indexOfFile);
+ System.out.println("IndexOfFile: " + indexOfFile + " PageIndex: "
+ + pgInfo.getPageLoc());
+ if (indexOfFile < pgInfo.getPageLoc()
+ || indexOfFile == fileList.size() - 1)
+ pgInfo.updatePageInfo(pgInfo.getPageLoc() - 1, fileList);
+ else
+ pgInfo.updatePageInfo(pgInfo.getPageLoc(), fileList);
return true;
+ } else {
+ return false;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ return false;
+ }
+ }
+
+ private synchronized void putPgInfo(PagingInfo pgInfo, RemoteSiteFile pFile) {
+ this.pageInfos.put(pFile, pgInfo);
+ }
+
+ private synchronized PagingInfo getPagingInfo(RemoteSiteFile pFile) {
+ PagingInfo pgInfo = this.pageInfos.get(pFile);
+ if (pgInfo == null)
+ this.putPgInfo(pgInfo = new PagingInfo(), pFile);
+ return pgInfo;
+ }
+
+ public RemoteSiteFile pwd(Protocol protocol) throws ProtocolException {
+ return new RemoteSiteFile(protocol.pwd());
+ }
+
+ public List<RemoteSiteFile> ls(Protocol protocol, RemoteSiteFile dir)
+ throws ProtocolException {
+ List<RemoteSiteFile> fileList = this.getDynamicFileList(protocol);
+ if (fileList == null) {
+ protocol.cd(dir);
+ fileList = toRemoteSiteFiles(protocol.ls());
+ }
+ return fileList;
+ }
+
+ public List<RemoteSiteFile> ls(Protocol protocol) throws ProtocolException {
+ List<RemoteSiteFile> fileList = this.getDynamicFileList(protocol);
+ if (fileList == null)
+ fileList = toRemoteSiteFiles(protocol.ls());
+ return fileList;
+ }
+
+ public List<RemoteSiteFile> ls(Protocol protocol, ProtocolFileFilter filter)
+ throws ProtocolException {
+ List<RemoteSiteFile> fileList = this.getDynamicFileList(protocol);
+ if (fileList == null)
+ fileList = toRemoteSiteFiles(protocol.ls(filter));
+ return fileList;
+ }
+
+ private synchronized List<RemoteSiteFile> getDynamicFileList(Protocol protocol)
+ throws ProtocolException {
+ return (List<RemoteSiteFile>) (List<?>) this.pathAndFileListMap.get(this
+ .pwd(protocol));
+ }
+
+ private synchronized void putDynamicFileList(Protocol protocol,
+ List<RemoteSiteFile> fileList) throws ProtocolException {
+ this.pathAndFileListMap.put(this.pwd(protocol), fileList);
+ }
+
+ public synchronized RemoteSiteFile getHomeDir(Protocol protocol) {
+ try {
+ protocol.cdHome();
+ return new RemoteSiteFile(protocol.pwd());
+ } catch (Exception e) {
+ e.printStackTrace();
+ return null;
+ }
+ }
+
+ public String getAbsPathFor(Protocol protocol, String path, boolean isDir) {
+ try {
+ protocol.cd(new ProtocolFile(path, isDir));
+ return protocol.pwd().getAbsoluteFile().getPath();
+ } catch (Exception e) {
+ e.printStackTrace();
+ return null;
+ }
+ }
+
+ /**
+ * Disconnects and logs out the given Protocol
+ *
+ * @param protocol
+ * The Protocol to be logout out and disconnected
+ * @throws RemoteConnectionException
+ */
+ public void disconnect(Protocol protocol) throws RemoteConnectionException {
+ URL url = null;
+ try {
+ url = ((RemoteSiteFile) protocol.pwd()).getSite().getURL();
+ LOG.log(Level.INFO, "Disconnecting protocol from " + url);
+ protocol.close();
+ } catch (Exception e) {
+ throw new RemoteConnectionException("Error disconnecting from " + url
+ + " : " + e.getMessage());
+ }
+ }
+
+ /**
+ * Disconnects all waiting Protocols and clears the waiting lists. Also clears
+ * the current Protocol
+ *
+ * @throws RemoteConnectionException
+ */
+ public void close() throws RemoteConnectionException {
+ Set<Entry<URL, Protocol>> entries = reuseProtocols.entrySet();
+ for (Entry<URL, Protocol> entry : entries) {
+ disconnect(entry.getValue());
+ }
+ this.reuseProtocols.clear();
+ this.urlAndProtocolFactory.clear();
+ this.pageInfos.clear();
+ this.pathAndFileListMap.clear();
+ }
+
+ private synchronized RemoteSiteFile getProtocolFileByProtocol(
+ Protocol protocol, String file, boolean isDir) throws ProtocolException {
+ try {
+ if (!file.startsWith("/")) {
+ protocol.cdHome();
+ file = protocol.pwd().getPath() + "/" + file;
+ }
+ return new RemoteSiteFile(file, isDir, null);
+ } catch (Exception e) {
+ throw new ProtocolException("Failed to create protocol for " + file
+ + " : " + e.getMessage());
+ }
+ }
+
+ private List<RemoteSiteFile> toRemoteSiteFiles(List<ProtocolFile> files) {
+ List<RemoteSiteFile> newFiles = new Vector<RemoteSiteFile>();
+ if (files != null) {
+ for (ProtocolFile file : files) {
+ newFiles.add(new RemoteSiteFile(file));
+ }
+ }
+ return newFiles;
+ }
+
+ class PagingInfo {
+
+ private int pageLoc;
+
+ private int sizeOfLastLS;
+
+ private RemoteSiteFile pFileAtPageLoc;
+
+ PagingInfo() {
+ this.pageLoc = 0;
+ this.sizeOfLastLS = -1;
+ this.pFileAtPageLoc = null;
+ }
+
+ synchronized void updatePageInfo(int newPageLoc, List<RemoteSiteFile> ls)
+ throws MalformedURLException, ProtocolException {
+ this.sizeOfLastLS = ls.size();
+ this.pageLoc = newPageLoc < 0 ? 0 : newPageLoc;
+ this.pFileAtPageLoc = (this.sizeOfLastLS > 0 && newPageLoc < ls.size()) ? ls
+ .get(newPageLoc) : null;
+ }
+
+ synchronized int getPageLoc() {
+ return this.pageLoc;
+ }
+
+ synchronized int getSizeOfLastLS() {
+ return this.sizeOfLastLS;
}
- public void cdToROOT(Protocol protocol) throws ProtocolException {
- protocol.cdToROOT();
- }
-
- public void cdToHOME(Protocol protocol) throws ProtocolException {
- protocol.cdToHOME();
- }
-
- public boolean isProtocolConnected(Protocol protocol)
- throws ProtocolException {
- return protocol.isProtocolConnected();
- }
-
- public void cd(Protocol protocol, ProtocolFile file)
- throws ProtocolException {
- protocol.cd(file);
- }
-
- public ProtocolFile getProtocolFileFor(Protocol protocol, String file,
- boolean isDir) throws ProtocolException {
- return protocol.getProtocolFileFor(file, isDir);
- }
-
- public synchronized boolean delete(Protocol protocol, ProtocolFile file)
- throws MalformedURLException, ProtocolException {
- try {
- PagingInfo pgInfo = this.getPagingInfo(file.getParentFile());
- List<ProtocolFile> fileList = this.ls(protocol, file
- .getParentFile());
- int indexOfFile = fileList.indexOf(file);
- if (indexOfFile != -1 && protocol.delete(file)) {
- fileList.remove(indexOfFile);
- System.out.println("IndexOfFile: " + indexOfFile
- + " PageIndex: " + pgInfo.getPageLoc());
- if (indexOfFile < pgInfo.getPageLoc()
- || indexOfFile == fileList.size() - 1)
- pgInfo.updatePageInfo(pgInfo.getPageLoc() - 1, fileList);
- else
- pgInfo.updatePageInfo(pgInfo.getPageLoc(), fileList);
- return true;
- } else {
- return false;
- }
- } catch (Exception e) {
- e.printStackTrace();
- return false;
- }
- }
-
- private synchronized void putPgInfo(PagingInfo pgInfo, ProtocolFile pFile) {
- this.pageInfos.put(pFile, pgInfo);
- }
-
- private synchronized PagingInfo getPagingInfo(ProtocolFile pFile) {
- PagingInfo pgInfo = this.pageInfos.get(pFile);
- if (pgInfo == null)
- this.putPgInfo(pgInfo = new PagingInfo(), pFile);
- return pgInfo;
- }
-
- public ProtocolFile pwd(Protocol protocol) throws ProtocolException {
- return protocol.pwd();
- }
-
- public List<ProtocolFile> ls(Protocol protocol, ProtocolFile dir)
- throws ProtocolException {
- List<ProtocolFile> fileList = this.getDynamicFileList(protocol);
- if (fileList == null)
- fileList = protocol.ls(dir);
- return fileList;
- }
-
- public List<ProtocolFile> ls(Protocol protocol) throws ProtocolException {
- List<ProtocolFile> fileList = this.getDynamicFileList(protocol);
- if (fileList == null)
- fileList = protocol.ls();
- return fileList;
- }
-
- public List<ProtocolFile> ls(Protocol protocol, ProtocolFileFilter filter)
- throws ProtocolException {
- List<ProtocolFile> fileList = this.getDynamicFileList(protocol);
- if (fileList == null)
- fileList = protocol.ls(filter);
- return fileList;
- }
-
- private synchronized List<ProtocolFile> getDynamicFileList(Protocol protocol)
- throws ProtocolException {
- return this.pathAndFileListMap.get(this.pwd(protocol));
- }
-
- private synchronized void putDynamicFileList(Protocol protocol,
- List<ProtocolFile> fileList) throws ProtocolException {
- this.pathAndFileListMap.put(this.pwd(protocol), fileList);
- }
-
- public String getProtocolType(Protocol protocol) {
- return protocol.getProtocolType();
- }
-
- public synchronized RemoteSite getRemoteSite(Protocol protocol) {
- return protocol.getRemoteSite();
- }
-
- public synchronized ProtocolFile getHomeDir(Protocol protocol) {
- return protocol.getHomeDir();
- }
-
- public String getAbsPathFor(Protocol protocol, String path) {
- return protocol.getAbsPathFor(path);
- }
-
- /**
- * Disconnects and logs out the given Protocol
- *
- * @param protocol
- * The Protocol to be logout out and disconnected
- * @throws RemoteConnectionException
- */
- public void disconnect(Protocol protocol) throws RemoteConnectionException {
- URL url = null;
- try {
- url = protocol.getRemoteSite().getURL();
- LOG.log(Level.INFO, "Disconnecting protocol from " + url);
- protocol.disconnect();
- } catch (Exception e) {
- throw new RemoteConnectionException("Error disconnecting from "
- + url + " : " + e.getMessage());
- }
- }
-
- /**
- * Disconnects all waiting Protocols and clears the waiting lists. Also
- * clears the current Protocol
- *
- * @throws RemoteConnectionException
- */
- public void close() throws RemoteConnectionException {
- Set<Entry<URL, Protocol>> entries = reuseProtocols.entrySet();
- for (Entry<URL, Protocol> entry : entries) {
- disconnect(entry.getValue());
- }
- this.reuseProtocols.clear();
- this.urlAndProtocolFactory.clear();
- this.pageInfos.clear();
- this.pathAndFileListMap.clear();
+ synchronized RemoteSiteFile getRemoteSiteFileAtPageLoc() {
+ return this.pFileAtPageLoc;
}
- class PagingInfo {
-
- private int pageLoc;
-
- private int sizeOfLastLS;
-
- private ProtocolFile pFileAtPageLoc;
-
- PagingInfo() {
- this.pageLoc = 0;
- this.sizeOfLastLS = -1;
- this.pFileAtPageLoc = null;
- }
-
- synchronized void updatePageInfo(int newPageLoc, List<ProtocolFile> ls)
- throws MalformedURLException, ProtocolException {
- this.sizeOfLastLS = ls.size();
- this.pageLoc = newPageLoc < 0 ? 0 : newPageLoc;
- this.pFileAtPageLoc = (this.sizeOfLastLS > 0 && newPageLoc < ls
- .size()) ? ls.get(newPageLoc) : null;
- }
-
- synchronized int getPageLoc() {
- return this.pageLoc;
- }
-
- synchronized int getSizeOfLastLS() {
- return this.sizeOfLastLS;
- }
-
- synchronized ProtocolFile getProtocolFileAtPageLoc() {
- return this.pFileAtPageLoc;
- }
-
- }
+ }
}
Added: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java?rev=1226517&view=auto
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java (added)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java Mon Jan 2 18:59:11 2012
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.cas.pushpull.protocol;
+
+//OODT imports
+import org.apache.oodt.cas.protocol.ProtocolFile;
+
+/**
+ *
+ * Extends {@link ProtocolFile} and links it to a {@link RemoteSite}.
+ *
+ * @author mattmann
+ * @version $Revision$
+ *
+ */
+public class RemoteSiteFile extends ProtocolFile {
+
+ private RemoteSite site;
+
+ public RemoteSiteFile(ProtocolFile file){
+ this(file.getPath(), file.isDir(), null);
+ }
+
+ /**
+ * @param parent
+ * @param path
+ * @param isDir
+ * @param site
+ */
+ public RemoteSiteFile(ProtocolFile parent, String path, boolean isDir,
+ RemoteSite site) {
+ super(parent, path, isDir);
+ this.site = site;
+ }
+
+ /**
+ * @param path
+ * @param isDir
+ * @param site
+ */
+ public RemoteSiteFile(String path, boolean isDir, RemoteSite site) {
+ super(path, isDir);
+ this.site = site;
+ }
+
+ /**
+ * @return the site
+ */
+ public RemoteSite getSite() {
+ return site;
+ }
+
+ /**
+ * @param site
+ * the site to set
+ */
+ public void setSite(RemoteSite site) {
+ this.site = site;
+ }
+
+ public RemoteSiteFile getRemoteParent() {
+ ProtocolFile parent = super.getParent();
+ return new RemoteSiteFile(parent.getPath(), parent.isDir(), this.site);
+ }
+
+ public RemoteSiteFile getAbsoluteFile() {
+ ProtocolFile parent = super.getAbsoluteFile();
+ return new RemoteSiteFile(parent.getPath(), parent.isDir(), this.site);
+ }
+
+}
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java?rev=1226517&r1=1226516&r2=1226517&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java (original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java Mon Jan 2 18:59:11 2012
@@ -31,9 +31,11 @@ import org.apache.oodt.cas.pushpull.file
import org.apache.oodt.cas.pushpull.filerestrictions.Parser;
import org.apache.oodt.cas.pushpull.filerestrictions.VirtualFile;
import org.apache.oodt.cas.pushpull.filerestrictions.VirtualFileStructure;
-import org.apache.oodt.cas.pushpull.protocol.ProtocolFile;
-import org.apache.oodt.cas.pushpull.protocol.ProtocolFileFilter;
+import org.apache.oodt.cas.protocol.ProtocolFile;
+import org.apache.oodt.cas.protocol.util.ProtocolFileFilter;
+import org.apache.oodt.cas.pushpull.protocol.ProtocolPath;
import org.apache.oodt.cas.pushpull.protocol.RemoteSite;
+import org.apache.oodt.cas.pushpull.protocol.RemoteSiteFile;
import org.apache.oodt.cas.pushpull.retrievalsystem.DataFileToPropFileLinker;
import org.apache.oodt.cas.pushpull.retrievalsystem.FileRetrievalSystem;
@@ -85,8 +87,7 @@ public class RemoteCrawler implements Re
// modify vfs to be root based if HOME directory based
if (!vfs.isRootBased()) {
- String homeDirPath = frs.getHomeDir(remoteSite).getProtocolPath()
- .getPathString();
+ String homeDirPath = frs.getHomeDir(remoteSite).getPath();
VirtualFile root = new VirtualFile(homeDirPath, true);
root.addChild(vfs.getRootVirtualFile());
vfs = new VirtualFileStructure(homeDirPath + "/"
@@ -102,22 +103,23 @@ public class RemoteCrawler implements Re
frs.changeToDir(initialCdPath, remoteSite);
// add starting directory to stack
- Stack<ProtocolFile> files = new Stack<ProtocolFile>();
- files.add(frs.getCurrentFile(remoteSite));
+ Stack<RemoteSiteFile> files = new Stack<RemoteSiteFile>();
+ files.add(new RemoteSiteFile(frs.getCurrentFile(remoteSite)));
// start crawling
while (!files.isEmpty()) {
- ProtocolFile file = files.peek();
+ RemoteSiteFile file = files.peek();
try {
// if directory, then add its children to the crawl list
- if (file.isDirectory()) {
+ if (file.isDir()) {
// get next page worth of children
- List<ProtocolFile> children = frs.getNextPage(file,
+ List<RemoteSiteFile> children = frs.getNextPage(file,
new ProtocolFileFilter() {
public boolean accept(ProtocolFile pFile) {
- return FileRestrictions.isAllowed(pFile
- .getProtocolPath(), vf);
+ return FileRestrictions.isAllowed(new
+ ProtocolPath(pFile
+ .getPath(), pFile.isDir()), vf);
}
});
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/DataFileToPropFileLinker.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/DataFileToPropFileLinker.java?rev=1226517&r1=1226516&r2=1226517&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/DataFileToPropFileLinker.java (original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/DataFileToPropFileLinker.java Mon Jan 2 18:59:11 2012
@@ -25,7 +25,7 @@ import java.util.LinkedList;
import java.util.Map.Entry;
//OODT imports
-import org.apache.oodt.cas.pushpull.protocol.ProtocolFile;
+import org.apache.oodt.cas.protocol.ProtocolFile;
/**
*
@@ -58,8 +58,7 @@ public class DataFileToPropFileLinker im
public synchronized void addPropFileToDataFileLink(File propFile,
ProtocolFile pFile) {
- this.addPropFileToDataFileLink(propFile, pFile.getProtocolPath()
- .getPathString());
+ this.addPropFileToDataFileLink(propFile, pFile.getPath());
}
public synchronized void addPropFileToDataFileLink(File propFile,
@@ -76,7 +75,7 @@ public class DataFileToPropFileLinker im
}
public synchronized void markAsFailed(ProtocolFile pFile, String errorMsg) {
- this.markAsFailed(pFile.getProtocolPath().getPathString(), errorMsg);
+ this.markAsFailed(pFile.getPath(), errorMsg);
}
public synchronized void markAsFailed(String pFilePath, String errorMsg) {
@@ -146,8 +145,7 @@ public class DataFileToPropFileLinker im
File propFile, LinkedList<ProtocolFile> list) {
LinkedList<ProtocolFile> returnList = new LinkedList<ProtocolFile>();
for (ProtocolFile pFile : list) {
- if (this.protocolFilePathAndPropFileMap.get(pFile.getProtocolPath()
- .getPathString()) != null)
+ if (this.protocolFilePathAndPropFileMap.get(pFile.getPath()) != null)
returnList.add(pFile);
}
return returnList;
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/DownloadListener.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/DownloadListener.java?rev=1226517&r1=1226516&r2=1226517&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/DownloadListener.java (original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/DownloadListener.java Mon Jan 2 18:59:11 2012
@@ -19,7 +19,7 @@
package org.apache.oodt.cas.pushpull.retrievalsystem;
//OODT imports
-import org.apache.oodt.cas.pushpull.protocol.ProtocolFile;
+import org.apache.oodt.cas.protocol.ProtocolFile;
/**
*
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java?rev=1226517&r1=1226516&r2=1226517&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java (original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java Mon Jan 2 18:59:11 2012
@@ -23,18 +23,19 @@ import org.apache.oodt.cas.pushpull.conf
import org.apache.oodt.cas.pushpull.config.SiteInfo;
import org.apache.oodt.cas.pushpull.exceptions.AlreadyInDatabaseException;
import org.apache.oodt.cas.pushpull.exceptions.CrawlerException;
-import org.apache.oodt.cas.pushpull.exceptions.ProtocolException;
import org.apache.oodt.cas.pushpull.exceptions.ProtocolFileException;
import org.apache.oodt.cas.pushpull.exceptions.RemoteConnectionException;
import org.apache.oodt.cas.pushpull.exceptions.ThreadEvaluatorException;
import org.apache.oodt.cas.pushpull.exceptions.ToManyFailedDownloadsException;
import org.apache.oodt.cas.pushpull.exceptions.UndefinedTypeException;
import org.apache.oodt.cas.pushpull.filerestrictions.renamingconventions.RenamingConvention;
-import org.apache.oodt.cas.pushpull.protocol.ProtocolFileFilter;
-import org.apache.oodt.cas.pushpull.protocol.Protocol;
-import org.apache.oodt.cas.pushpull.protocol.ProtocolFile;
+import org.apache.oodt.cas.protocol.exceptions.ProtocolException;
+import org.apache.oodt.cas.protocol.util.ProtocolFileFilter;
+import org.apache.oodt.cas.protocol.Protocol;
+import org.apache.oodt.cas.protocol.ProtocolFile;
import org.apache.oodt.cas.pushpull.protocol.ProtocolHandler;
import org.apache.oodt.cas.pushpull.protocol.RemoteSite;
+import org.apache.oodt.cas.pushpull.protocol.RemoteSiteFile;
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.metadata.util.MimeTypeUtils;
@@ -253,7 +254,7 @@ public class FileRetrievalSystem {
: false;
}
- public List<ProtocolFile> getNextPage(final ProtocolFile dir,
+ public List<RemoteSiteFile> getNextPage(final RemoteSiteFile dir,
final ProtocolFileFilter filter) throws RemoteConnectionException {
for (int i = 0; i < 3; i++) {
try {
@@ -276,7 +277,7 @@ public class FileRetrievalSystem {
}
public void changeToRoot(RemoteSite remoteSite) throws ProtocolException,
- MalformedURLException {
+ MalformedURLException, org.apache.oodt.cas.protocol.exceptions.ProtocolException {
if (validate(remoteSite))
protocolHandler.cdToROOT(protocolHandler
.getAppropriateProtocolBySite(remoteSite, true));
@@ -304,9 +305,9 @@ public class FileRetrievalSystem {
throw new ProtocolException("Not a valid remote site " + remoteSite);
}
- public void changeToDir(ProtocolFile pFile) throws ProtocolException,
+ public void changeToDir(RemoteSiteFile pFile) throws ProtocolException,
MalformedURLException {
- RemoteSite remoteSite = pFile.getRemoteSite();
+ RemoteSite remoteSite = pFile.getSite();
if (validate(remoteSite))
protocolHandler.cd(protocolHandler.getAppropriateProtocolBySite(
remoteSite, true), pFile);
@@ -392,7 +393,7 @@ public class FileRetrievalSystem {
}
}
- public boolean addToDownloadQueue(ProtocolFile file, String renamingString,
+ public boolean addToDownloadQueue(RemoteSiteFile file, String renamingString,
File downloadToDir, String uniqueMetadataElement,
boolean deleteAfterDownload) throws ToManyFailedDownloadsException,
RemoteConnectionException, AlreadyInDatabaseException,
@@ -559,7 +560,7 @@ public class FileRetrievalSystem {
}
}
- Protocol getNextAvaliableSession(ProtocolFile file) throws CrawlerException {
+ Protocol getNextAvaliableSession(RemoteSiteFile file) throws CrawlerException {
// wait for available session, then load it
Protocol session;
while ((session = getSession(file)) == null) {
@@ -612,11 +613,11 @@ public class FileRetrievalSystem {
* If downloading session Protocol has to be reconnected and
* there is an error communicating with the server
*/
- synchronized Protocol getSession(ProtocolFile file) throws CrawlerException {
+ synchronized Protocol getSession(RemoteSiteFile file) throws CrawlerException {
try {
Protocol session = null;
- if (file.getRemoteSite().getMaxConnections() < 0
- || file.getRemoteSite().getMaxConnections() > this.getCurrentlyDownloadingFiles().size()) {
+ if (file.getSite().getMaxConnections() < 0
+ || file.getSite().getMaxConnections() > this.getCurrentlyDownloadingFiles().size()) {
if (avaliableSessions.size() > 0) {
session = modifyAvailableSessionForPath(file);
} else if (numberOfSessions < max_sessions) {
@@ -631,27 +632,27 @@ public class FileRetrievalSystem {
}
}
- Protocol createNewSessionForPath(ProtocolFile file)
+ Protocol createNewSessionForPath(RemoteSiteFile file)
throws RemoteConnectionException {
return protocolHandler.getAppropriateProtocol(file, /* reuse */false, /* navigate */
true);
}
- Protocol modifyAvailableSessionForPath(ProtocolFile file)
+ Protocol modifyAvailableSessionForPath(RemoteSiteFile file)
throws ProtocolException, RemoteConnectionException {
Protocol session = getAvailableSession();
- if (!protocolHandler.getRemoteSite(session).getURL().getHost().equals(
- file.getURL().getHost())
+ if (!file.getSite().getURL().getHost().equals(
+ file.getSite().getURL().getHost())
|| !protocolHandler.isProtocolConnected(session)) {
protocolHandler.disconnect(session);
session = protocolHandler.getAppropriateProtocol(file, /* reuse */
false, /* navigate */true);
} else {
try {
- if (file.isDirectory())
+ if (file.isDir())
protocolHandler.cd(session, file);
else
- protocolHandler.cd(session, file.getParentFile());
+ protocolHandler.cd(session, new RemoteSiteFile(file.getParent()));
} catch (Exception e) {
e.printStackTrace();
try {
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RemoteFile.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RemoteFile.java?rev=1226517&r1=1226516&r2=1226517&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RemoteFile.java (original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RemoteFile.java Mon Jan 2 18:59:11 2012
@@ -21,7 +21,7 @@ package org.apache.oodt.cas.pushpull.ret
//OODT imports
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.SerializableMetadata;
-import org.apache.oodt.cas.pushpull.protocol.ProtocolFile;
+import org.apache.oodt.cas.pushpull.protocol.RemoteSiteFile;
//JDK imports
import java.io.FileOutputStream;
@@ -40,15 +40,14 @@ public class RemoteFile implements Remot
private Metadata metadata;
- private ProtocolFile pFile;
+ private RemoteSiteFile pFile;
- public RemoteFile(ProtocolFile pFile) {
+ public RemoteFile(RemoteSiteFile pFile) {
this.pFile = pFile;
this.metadata = new Metadata();
- this.metadata.addMetadata(RETRIEVED_FROM_LOC, pFile.getProtocolPath()
- .getPathString());
+ this.metadata.addMetadata(RETRIEVED_FROM_LOC, pFile.getPath());
this.metadata.addMetadata(FILENAME, pFile.getName());
- this.metadata.addMetadata(DATA_PROVIDER, pFile.getHostName());
+ this.metadata.addMetadata(DATA_PROVIDER, pFile.getSite().getURL().getHost());
}
public void setUniqueMetadataElement(String uniqueMetadataElement) {
@@ -68,7 +67,7 @@ public class RemoteFile implements Remot
return this.metadata;
}
- public ProtocolFile getProtocolFile() {
+ public RemoteSiteFile getProtocolFile() {
return this.pFile;
}
Modified: oodt/trunk/pushpull/src/main/resources/policy/ProtocolFactoryInfo.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/resources/policy/ProtocolFactoryInfo.xml?rev=1226517&r1=1226516&r2=1226517&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/resources/policy/ProtocolFactoryInfo.xml (original)
+++ oodt/trunk/pushpull/src/main/resources/policy/ProtocolFactoryInfo.xml Mon Jan 2 18:59:11 2012
@@ -18,24 +18,24 @@
<protocols>
<protocol type="ftp">
- <protocolFactory class="org.apache.oodt.cas.pushpull.protocol.ftp.CogJGlobusFtpClientFactory"/>
- <protocolFactory class="org.apache.oodt.cas.pushpull.protocol.ftp.CommonsNetFtpClientFactory"/>
+ <protocolFactory class="org.apache.oodt.cas.protocol.ftp.CogJGlobusFtpClientFactory"/>
+ <protocolFactory class="org.apache.oodt.cas.protocol.ftp.CommonsNetFtpClientFactory"/>
</protocol>
<protocol type="http">
- <protocolFactory class="org.apache.oodt.cas.pushpull.protocol.http.HttpClientFactory"/>
+ <protocolFactory class="org.apache.oodt.cas.protocol.http.HttpClientFactory"/>
</protocol>
<protocol type="sftp">
- <protocolFactory class="org.apache.oodt.cas.pushpull.protocol.sftp.JschSftpClientFactory"/>
+ <protocolFactory class="org.apache.oodt.cas.protocol.sftp.JschSftpClientFactory"/>
</protocol>
<protocol type="imaps">
- <protocolFactory class="org.apache.oodt.cas.pushpull.protocol.imaps.ImapsClientFactory"/>
+ <protocolFactory class="org.apache.oodt.cas.protocol.imaps.ImapsClientFactory"/>
</protocol>
<protocol type="file">
- <protocolFactory class="org.apache.oodt.cas.pushpull.protocol.local.LocalClientFactory"/>
+ <protocolFactory class="org.apache.oodt.cas.protocol.local.LocalClientFactory"/>
</protocol>
</protocols>