You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by bf...@apache.org on 2011/03/16 23:28:39 UTC
svn commit: r1082328 - in /oodt/trunk: ./
pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/
pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/
pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/
pushpull/src/main/r...
Author: bfoster
Date: Wed Mar 16 22:28:39 2011
New Revision: 1082328
URL: http://svn.apache.org/viewvc?rev=1082328&view=rev
Log:
- pushpull can now limit number of concurrent file downloading sessions on a site per site bases
----------------------
OODT-160
Modified:
oodt/trunk/CHANGES.txt
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/ConfigParserMetKeys.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/RemoteSpecs.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSite.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java
oodt/trunk/pushpull/src/main/resources/examples/ExternalSourcesFiles/ExternalSources.xml
Modified: oodt/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/oodt/trunk/CHANGES.txt?rev=1082328&r1=1082327&r2=1082328&view=diff
==============================================================================
--- oodt/trunk/CHANGES.txt (original)
+++ oodt/trunk/CHANGES.txt Wed Mar 16 22:28:39 2011
@@ -4,6 +4,9 @@ Apache OODT Change Log
Release 0.3-SNAPSHOT (in progress)
--------------------------------------------
+* OODT-160 Allow number of session protocol connections in
+ pushpull to be configurable for each site (bfoster)
+
* OODT-161 PushPull getTextContext()/getNode() value still poses an issue for some JVMs (bfoster)
* OODT-159 Dynamic Queue/Node was not added to command-line usage printout (bfoster)
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/ConfigParserMetKeys.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/ConfigParserMetKeys.java?rev=1082328&r1=1082327&r2=1082328&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/ConfigParserMetKeys.java (original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/ConfigParserMetKeys.java Wed Mar 16 22:28:39 2011
@@ -106,4 +106,6 @@ public interface ConfigParserMetKeys {
public static final String CD_TEST_DIR_TAG = "cdTestDir";
+ public static final String MAX_CONN_TAG = "maxConn";
+
}
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/RemoteSpecs.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/RemoteSpecs.java?rev=1082328&r1=1082327&r2=1082328&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/RemoteSpecs.java (original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/RemoteSpecs.java Wed Mar 16 22:28:39 2011
@@ -301,6 +301,7 @@ public class RemoteSpecs implements Conf
.replaceEnvVariables(((Element) loginNode)
.getAttribute(ALIAS_ATTR));
String username = null, password = null, cdTestDir = null;
+ int maxConnections = -1;
// parse this login info
NodeList loginInfo = loginNode.getChildNodes();
@@ -319,11 +320,14 @@ public class RemoteSpecs implements Conf
} else if (node.getNodeName().equals(CD_TEST_DIR_TAG)) {
cdTestDir = PathUtils.replaceEnvVariables(
XMLUtils.getSimpleElementText((Element) node, true));
+ } else if (node.getNodeName().equals(MAX_CONN_TAG)) {
+ maxConnections = Integer.parseInt(PathUtils.replaceEnvVariables(
+ XMLUtils.getSimpleElementText((Element) node, true)));
}
}
this.siteInfo.addSite(new RemoteSite(alias, new URL(type
- + "://" + host), username, password, cdTestDir));
+ + "://" + host), username, password, cdTestDir, maxConnections));
}
}
} catch (Exception e) {
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSite.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSite.java?rev=1082328&r1=1082327&r2=1082328&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSite.java (original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSite.java Wed Mar 16 22:28:39 2011
@@ -31,8 +31,9 @@ import java.net.URL;
* </p>.
*/
public class RemoteSite {
+
private String alias, username, password, cdTestDir;
-
+ private int maxConnections;
private URL url;
public RemoteSite(String alias, URL url, String username, String password) {
@@ -40,12 +41,18 @@ public class RemoteSite {
this.username = username;
this.password = password;
this.url = url;
+ this.maxConnections = -1;
}
public RemoteSite(String alias, URL url, String username, String password, String cdTestDir) {
this(alias, url, username, password);
this.cdTestDir = cdTestDir;
}
+
+ public RemoteSite(String alias, URL url, String username, String password, String cdTestDir, int maxConnections) {
+ this(alias, url, username, password, cdTestDir);
+ this.maxConnections = maxConnections;
+ }
public String getAlias() {
return this.alias;
@@ -67,12 +74,17 @@ public class RemoteSite {
return this.cdTestDir;
}
+ public int getMaxConnections() {
+ return this.maxConnections;
+ }
+
public void copy(RemoteSite rs) {
this.alias = rs.alias;
this.url = rs.url;
this.username = rs.username;
this.password = rs.password;
this.cdTestDir = rs.cdTestDir;
+ this.maxConnections = rs.maxConnections;
}
public boolean equals(Object obj) {
@@ -80,7 +92,7 @@ public class RemoteSite {
RemoteSite rs = (RemoteSite) obj;
return (rs.alias.equals(this.alias) && rs.url.equals(this.url)
&& rs.username.equals(this.username) && rs.password
- .equals(this.password));
+ .equals(this.password) && rs.maxConnections == this.maxConnections);
} else
return false;
}
@@ -88,6 +100,6 @@ public class RemoteSite {
public String toString() {
return "RemoteSite: alias = '" + this.alias + "' url = '" + this.url
+ "' username = '" + this.username + "' cdTestDir = '"
- + this.cdTestDir + "'";
+ + this.cdTestDir + "' maxConnections = '" + this.maxConnections + "'";
}
}
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java?rev=1082328&r1=1082327&r2=1082328&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java (original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java Wed Mar 16 22:28:39 2011
@@ -604,11 +604,14 @@ public class FileRetrievalSystem {
synchronized Protocol getSession(ProtocolFile file) throws CrawlerException {
try {
Protocol session = null;
- if (avaliableSessions.size() > 0) {
- session = modifyAvailableSessionForPath(file);
- } else if (numberOfSessions < max_sessions) {
- session = createNewSessionForPath(file);
- incrementSessions();
+ if (file.getRemoteSite().getMaxConnections() < 0
+ || file.getRemoteSite().getMaxConnections() > this.getCurrentlyDownloadingFiles().size()) {
+ if (avaliableSessions.size() > 0) {
+ session = modifyAvailableSessionForPath(file);
+ } else if (numberOfSessions < max_sessions) {
+ session = createNewSessionForPath(file);
+ incrementSessions();
+ }
}
return session;
} catch (Exception e) {
Modified: oodt/trunk/pushpull/src/main/resources/examples/ExternalSourcesFiles/ExternalSources.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/resources/examples/ExternalSourcesFiles/ExternalSources.xml?rev=1082328&r1=1082327&r2=1082328&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/resources/examples/ExternalSourcesFiles/ExternalSources.xml (original)
+++ oodt/trunk/pushpull/src/main/resources/examples/ExternalSourcesFiles/ExternalSources.xml Wed Mar 16 22:28:39 2011
@@ -27,6 +27,7 @@
<username></username>
<password></password>
<cdTestDir>2B-CWC-RO.R04/2007/304</cdTestDir>
+ <maxConn>5</maxConn>
</login>
</source>
<source host="ftp.class.noaa.gov">