You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by ml...@apache.org on 2013/07/03 14:05:28 UTC
svn commit: r1499348 - in
/manifoldcf/trunk/connectors/generic/connector/src/main:
java/org/apache/manifoldcf/authorities/authorities/generic/
java/org/apache/manifoldcf/crawler/connectors/generic/
native2ascii/org/apache/manifoldcf/authorities/authori...
Author: mlizewski
Date: Wed Jul 3 12:05:27 2013
New Revision: 1499348
URL: http://svn.apache.org/r1499348
Log:
improvement of Generic Connector and Generic Authority - added configuration for connection timeout and socket timeout.
Modified:
manifoldcf/trunk/connectors/generic/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/generic/GenericAuthority.java
manifoldcf/trunk/connectors/generic/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/generic/GenericConnector.java
manifoldcf/trunk/connectors/generic/connector/src/main/native2ascii/org/apache/manifoldcf/authorities/authorities/generic/common_en_US.properties
manifoldcf/trunk/connectors/generic/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/generic/common_en_US.properties
Modified: manifoldcf/trunk/connectors/generic/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/generic/GenericAuthority.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/generic/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/generic/GenericAuthority.java?rev=1499348&r1=1499347&r2=1499348&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/generic/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/generic/GenericAuthority.java (original)
+++ manifoldcf/trunk/connectors/generic/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/generic/GenericAuthority.java Wed Jul 3 12:05:27 2013
@@ -33,11 +33,11 @@ import org.apache.http.HttpStatus;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;
import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.ConnectionBackoffStrategy;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.params.HttpConnectionParams;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.apache.manifoldcf.authorities.interfaces.AuthorizationResponse;
@@ -88,6 +88,10 @@ public class GenericAuthority extends or
private String genericEntryPoint = null;
+ private int connectionTimeoutMillis = 60 * 1000;
+
+ private int socketTimeoutMillis = 30 * 60 * 1000;
+
private long responseLifetime = 60000L; //60sec
private int LRUsize = 1000;
@@ -132,6 +136,18 @@ public class GenericAuthority extends or
genericPassword = ManifoldCF.deobfuscate(getParam(configParams, "genericPassword", ""));
} catch (ManifoldCFException ignore) {
}
+ connectionTimeoutMillis = Integer.parseInt(getParam(configParams, "genericConnectionTimeout", "60000"));
+ if (connectionTimeoutMillis == 0) {
+ connectionTimeoutMillis = 60000;
+ }
+ socketTimeoutMillis = Integer.parseInt(getParam(configParams, "genericSocketTimeout", "1800000"));
+ if (socketTimeoutMillis == 0) {
+ socketTimeoutMillis = 1800000;
+ }
+ responseLifetime = Long.parseLong(getParam(configParams, "genericResponseLifetime", "60000"));
+ if (responseLifetime == 0) {
+ responseLifetime = 60000;
+ }
}
protected DefaultHttpClient getClient() throws ManifoldCFException {
@@ -152,6 +168,8 @@ public class GenericAuthority extends or
throw new ManifoldCFException("getClient exception: " + ex.getMessage(), ex);
}
}
+ HttpConnectionParams.setConnectionTimeout(cl.getParams(), connectionTimeoutMillis);
+ HttpConnectionParams.setSoTimeout(cl.getParams(), socketTimeoutMillis);
sessionExpirationTime = System.currentTimeMillis() + 300000L;
client = cl;
return cl;
@@ -274,7 +292,7 @@ public class GenericAuthority extends or
if (auth == null) {
return userNotFoundResponse;
}
- if(!auth.exists) {
+ if (!auth.exists) {
return userNotFoundResponse;
}
if (auth.tokens == null) {
@@ -349,6 +367,9 @@ public class GenericAuthority extends or
password = ManifoldCF.deobfuscate(getParam(parameters, "genericPassword", ""));
} catch (ManifoldCFException ignore) {
}
+ String conTimeout = getParam(parameters, "genericConnectionTimeout", "60000");
+ String soTimeout = getParam(parameters, "genericSocketTimeout", "1800000");
+ String respLifetime = getParam(parameters, "genericResponseLifetime", "60000");
if (tabName.equals(Messages.getString(locale, "generic.EntryPoint"))) {
out.print(
@@ -366,11 +387,26 @@ public class GenericAuthority extends or
+ " <td class=\"description\"><nobr>" + Messages.getBodyString(locale, "generic.PasswordColon") + "</nobr></td>\n"
+ " <td class=\"value\"><input type=\"password\" size=\"32\" name=\"genericPassword\" value=\"" + Encoder.attributeEscape(password) + "\"/></td>\n"
+ " </tr>\n"
+ + " <tr>\n"
+ + " <td class=\"description\"><nobr>" + Messages.getBodyString(locale, "generic.ConnectionTimeoutColon") + "</nobr></td>\n"
+ + " <td class=\"value\"><input type=\"text\" size=\"32\" name=\"genericConTimeout\" value=\"" + Encoder.attributeEscape(conTimeout) + "\"/></td>\n"
+ + " </tr>\n"
+ + " <tr>\n"
+ + " <td class=\"description\"><nobr>" + Messages.getBodyString(locale, "generic.SocketTimeoutColon") + "</nobr></td>\n"
+ + " <td class=\"value\"><input type=\"text\" size=\"32\" name=\"genericSoTimeout\" value=\"" + Encoder.attributeEscape(soTimeout) + "\"/></td>\n"
+ + " </tr>\n"
+ + " <tr>\n"
+ + " <td class=\"description\"><nobr>" + Messages.getBodyString(locale, "generic.ResponseLifetimeColon") + "</nobr></td>\n"
+ + " <td class=\"value\"><input type=\"text\" size=\"32\" name=\"genericResponseLifetime\" value=\"" + Encoder.attributeEscape(respLifetime) + "\"/></td>\n"
+ + " </tr>\n"
+ "</table>\n");
} else {
out.print("<input type=\"hidden\" name=\"genericEntryPoint\" value=\"" + Encoder.attributeEscape(server) + "\"/>\n");
out.print("<input type=\"hidden\" name=\"genericLogin\" value=\"" + Encoder.attributeEscape(login) + "\"/>\n");
out.print("<input type=\"hidden\" name=\"genericPassword\" value=\"" + Encoder.attributeEscape(password) + "\"/>\n");
+ out.print("<input type=\"hidden\" name=\"genericConTimeout\" value=\"" + Encoder.attributeEscape(conTimeout) + "\"/>\n");
+ out.print("<input type=\"hidden\" name=\"genericSoTimeout\" value=\"" + Encoder.attributeEscape(soTimeout) + "\"/>\n");
+ out.print("<input type=\"hidden\" name=\"genericResponseLifetime\" value=\"" + Encoder.attributeEscape(respLifetime) + "\"/>\n");
}
}
@@ -381,6 +417,9 @@ public class GenericAuthority extends or
copyParam(variableContext, parameters, "genericLogin");
copyParam(variableContext, parameters, "genericEntryPoint");
+ copyParam(variableContext, parameters, "genericConTimeout");
+ copyParam(variableContext, parameters, "genericSoTimeout");
+ copyParam(variableContext, parameters, "genericResponseLifetime");
String password = variableContext.getParameter("genericPassword");
if (password == null) {
@@ -396,6 +435,9 @@ public class GenericAuthority extends or
throws ManifoldCFException, IOException {
String login = getParam(parameters, "genericLogin", "");
String server = getParam(parameters, "genericEntryPoint", "");
+ String conTimeout = getParam(parameters, "genericConnectionTimeout", "60000");
+ String soTimeout = getParam(parameters, "genericSocketTimeout", "1800000");
+ String respLifetime = getParam(parameters, "genericResponseLifetime", "60000");
out.print(
"<table class=\"displaytable\">\n"
@@ -412,6 +454,18 @@ public class GenericAuthority extends or
+ " <td class=\"description\"><nobr>" + Messages.getBodyString(locale, "generic.PasswordColon") + "</nobr></td>\n"
+ " <td class=\"value\">**********</td>\n"
+ " </tr>\n"
+ + " <tr>\n"
+ + " <td class=\"description\"><nobr>" + Messages.getBodyString(locale, "generic.ConnectionTimeoutColon") + "</nobr></td>\n"
+ + " <td class=\"value\">" + Encoder.bodyEscape(conTimeout) + "</td>\n"
+ + " </tr>\n"
+ + " <tr>\n"
+ + " <td class=\"description\"><nobr>" + Messages.getBodyString(locale, "generic.SocketTimeoutColon") + "</nobr></td>\n"
+ + " <td class=\"value\">" + Encoder.bodyEscape(soTimeout) + "</td>\n"
+ + " </tr>\n"
+ + " <tr>\n"
+ + " <td class=\"description\"><nobr>" + Messages.getBodyString(locale, "generic.ResponseLifetimeColon") + "</nobr></td>\n"
+ + " <td class=\"value\">" + Encoder.bodyEscape(respLifetime) + "</td>\n"
+ + " </tr>\n"
+ "</table>\n");
}
Modified: manifoldcf/trunk/connectors/generic/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/generic/GenericConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/generic/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/generic/GenericConnector.java?rev=1499348&r1=1499347&r2=1499348&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/generic/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/generic/GenericConnector.java (original)
+++ manifoldcf/trunk/connectors/generic/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/generic/GenericConnector.java Wed Jul 3 12:05:27 2013
@@ -43,6 +43,7 @@ import org.apache.http.client.HttpClient
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.params.HttpConnectionParams;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.apache.manifoldcf.agents.interfaces.*;
@@ -85,6 +86,10 @@ public class GenericConnector extends Ba
private String genericEntryPoint = null;
+ private int connectionTimeoutMillis = 60 * 1000;
+
+ private int socketTimeoutMillis = 30 * 60 * 1000;
+
protected static final String RELATIONSHIP_RELATED = "related";
private ConcurrentHashMap<String, Item> documentCache = new ConcurrentHashMap<String, Item>(10);
@@ -138,6 +143,14 @@ public class GenericConnector extends Ba
genericPassword = ManifoldCF.deobfuscate(getParam(configParams, "genericPassword", ""));
} catch (ManifoldCFException ignore) {
}
+ connectionTimeoutMillis = Integer.parseInt(getParam(configParams, "genericConnectionTimeout", "60000"));
+ if (connectionTimeoutMillis == 0) {
+ connectionTimeoutMillis = 60000;
+ }
+ socketTimeoutMillis = Integer.parseInt(getParam(configParams, "genericSocketTimeout", "1800000"));
+ if (socketTimeoutMillis == 0) {
+ socketTimeoutMillis = 1800000;
+ }
}
protected DefaultHttpClient getClient() throws ManifoldCFException {
@@ -152,6 +165,8 @@ public class GenericConnector extends Ba
throw new ManifoldCFException("getClient exception: " + ex.getMessage(), ex);
}
}
+ HttpConnectionParams.setConnectionTimeout(cl.getParams(), connectionTimeoutMillis);
+ HttpConnectionParams.setSoTimeout(cl.getParams(), socketTimeoutMillis);
return cl;
}
@@ -501,6 +516,8 @@ public class GenericConnector extends Ba
password = ManifoldCF.deobfuscate(getParam(parameters, "genericPassword", ""));
} catch (ManifoldCFException ignore) {
}
+ String conTimeout = getParam(parameters, "genericConnectionTimeout", "60000");
+ String soTimeout = getParam(parameters, "genericSocketTimeout", "1800000");
if (tabName.equals(Messages.getString(locale, "generic.EntryPoint"))) {
out.print(
@@ -518,11 +535,21 @@ public class GenericConnector extends Ba
+ " <td class=\"description\"><nobr>" + Messages.getBodyString(locale, "generic.PasswordColon") + "</nobr></td>\n"
+ " <td class=\"value\"><input type=\"password\" size=\"32\" name=\"genericPassword\" value=\"" + Encoder.attributeEscape(password) + "\"/></td>\n"
+ " </tr>\n"
+ + " <tr>\n"
+ + " <td class=\"description\"><nobr>" + Messages.getBodyString(locale, "generic.ConnectionTimeoutColon") + "</nobr></td>\n"
+ + " <td class=\"value\"><input type=\"text\" size=\"32\" name=\"genericConTimeout\" value=\"" + Encoder.attributeEscape(conTimeout) + "\"/></td>\n"
+ + " </tr>\n"
+ + " <tr>\n"
+ + " <td class=\"description\"><nobr>" + Messages.getBodyString(locale, "generic.SocketTimeoutColon") + "</nobr></td>\n"
+ + " <td class=\"value\"><input type=\"text\" size=\"32\" name=\"genericSoTimeout\" value=\"" + Encoder.attributeEscape(soTimeout) + "\"/></td>\n"
+ + " </tr>\n"
+ "</table>\n");
} else {
out.print("<input type=\"hidden\" name=\"genericEntryPoint\" value=\"" + Encoder.attributeEscape(server) + "\"/>\n");
out.print("<input type=\"hidden\" name=\"genericLogin\" value=\"" + Encoder.attributeEscape(login) + "\"/>\n");
out.print("<input type=\"hidden\" name=\"genericPassword\" value=\"" + Encoder.attributeEscape(password) + "\"/>\n");
+ out.print("<input type=\"hidden\" name=\"genericConTimeout\" value=\"" + Encoder.attributeEscape(conTimeout) + "\"/>\n");
+ out.print("<input type=\"hidden\" name=\"genericSoTimeout\" value=\"" + Encoder.attributeEscape(soTimeout) + "\"/>\n");
}
}
@@ -533,6 +560,8 @@ public class GenericConnector extends Ba
copyParam(variableContext, parameters, "genericLogin");
copyParam(variableContext, parameters, "genericEntryPoint");
+ copyParam(variableContext, parameters, "genericConTimeout");
+ copyParam(variableContext, parameters, "genericSoTimeout");
String password = variableContext.getParameter("genericPassword");
if (password == null) {
@@ -548,7 +577,9 @@ public class GenericConnector extends Ba
throws ManifoldCFException, IOException {
String login = getParam(parameters, "genericLogin", "");
String server = getParam(parameters, "genericEntryPoint", "");
-
+ String conTimeout = getParam(parameters, "genericConnectionTimeout", "60000");
+ String soTimeout = getParam(parameters, "genericSocketTimeout", "1800000");
+
out.print(
"<table class=\"displaytable\">\n"
+ " <tr><td class=\"separator\" colspan=\"2\"><hr/></td></tr>\n"
@@ -564,6 +595,14 @@ public class GenericConnector extends Ba
+ " <td class=\"description\"><nobr>" + Messages.getBodyString(locale, "generic.PasswordColon") + "</nobr></td>\n"
+ " <td class=\"value\">**********</td>\n"
+ " </tr>\n"
+ + " <tr>\n"
+ + " <td class=\"description\"><nobr>" + Messages.getBodyString(locale, "generic.ConnectionTimeoutColon") + "</nobr></td>\n"
+ + " <td class=\"value\">" + Encoder.bodyEscape(conTimeout) + "</td>\n"
+ + " </tr>\n"
+ + " <tr>\n"
+ + " <td class=\"description\"><nobr>" + Messages.getBodyString(locale, "generic.SocketTimeoutColon") + "</nobr></td>\n"
+ + " <td class=\"value\">" + Encoder.bodyEscape(soTimeout) + "</td>\n"
+ + " </tr>\n"
+ "</table>\n");
}
Modified: manifoldcf/trunk/connectors/generic/connector/src/main/native2ascii/org/apache/manifoldcf/authorities/authorities/generic/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/generic/connector/src/main/native2ascii/org/apache/manifoldcf/authorities/authorities/generic/common_en_US.properties?rev=1499348&r1=1499347&r2=1499348&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/generic/connector/src/main/native2ascii/org/apache/manifoldcf/authorities/authorities/generic/common_en_US.properties (original)
+++ manifoldcf/trunk/connectors/generic/connector/src/main/native2ascii/org/apache/manifoldcf/authorities/authorities/generic/common_en_US.properties Wed Jul 3 12:05:27 2013
@@ -18,3 +18,6 @@ generic.EntryPointColon=Entry Point:
generic.LoginColon=Login:
generic.PasswordColon=Password:
generic.EntryPointCannotBeBlank=EntryPoint cannot be blank
+generic.ConnectionTimeoutColon=Connection timeout (milis):
+generic.SocketTimeoutColon=Socket timeout (milis):
+generic.ResponseLifetimeColon=Cache responses (milis):
\ No newline at end of file
Modified: manifoldcf/trunk/connectors/generic/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/generic/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/generic/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/generic/common_en_US.properties?rev=1499348&r1=1499347&r2=1499348&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/generic/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/generic/common_en_US.properties (original)
+++ manifoldcf/trunk/connectors/generic/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/generic/common_en_US.properties Wed Jul 3 12:05:27 2013
@@ -17,6 +17,8 @@ generic.EntryPoint=Entry Point
generic.EntryPointColon=Entry Point:
generic.LoginColon=Login:
generic.PasswordColon=Password:
+generic.ConnectionTimeoutColon=Connection timeout (milis):
+generic.SocketTimeoutColon=Socket timeout (milis):
generic.Parameters=Parameters
generic.ParametersColon=Parameters: