You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2015/04/10 13:55:40 UTC
svn commit: r1672621 - in /manifoldcf/branches/dev_1x: ./
connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/
connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/
co...
Author: kwright
Date: Fri Apr 10 11:55:39 2015
New Revision: 1672621
URL: http://svn.apache.org/r1672621
Log:
Pull up fix for CONNECTORS-1185 from trunk.
Added:
manifoldcf/branches/dev_1x/framework/connector-common/src/main/java/org/apache/http/
- copied from r1672616, manifoldcf/trunk/framework/connector-common/src/main/java/org/apache/http/
Modified:
manifoldcf/branches/dev_1x/ (props changed)
manifoldcf/branches/dev_1x/CHANGES.txt
manifoldcf/branches/dev_1x/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
manifoldcf/branches/dev_1x/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraSession.java
manifoldcf/branches/dev_1x/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java
manifoldcf/branches/dev_1x/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
manifoldcf/branches/dev_1x/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/meridio/meridiowrapper/MeridioWrapper.java
manifoldcf/branches/dev_1x/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
manifoldcf/branches/dev_1x/connectors/rss/ (props changed)
manifoldcf/branches/dev_1x/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
manifoldcf/branches/dev_1x/connectors/sharepoint/ (props changed)
manifoldcf/branches/dev_1x/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java
manifoldcf/branches/dev_1x/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
manifoldcf/branches/dev_1x/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java
manifoldcf/branches/dev_1x/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
manifoldcf/branches/dev_1x/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java
manifoldcf/branches/dev_1x/connectors/wiki/ (props changed)
manifoldcf/branches/dev_1x/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
manifoldcf/branches/dev_1x/framework/ (props changed)
manifoldcf/branches/dev_1x/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyShutdown.java
manifoldcf/branches/dev_1x/framework/script-engine/src/main/java/org/apache/manifoldcf/scriptengine/ScriptParser.java
Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Apr 10 11:55:39 2015
@@ -123,4 +123,4 @@
/manifoldcf/branches/CONNECTORS-981:1605049-1605773
/manifoldcf/branches/CONNECTORS-989:1611600-1612101
/manifoldcf/branches/CONNECTORS-990:1610284-1610707
-/manifoldcf/trunk:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1631750,1631953,1632013,1632225,1632289,1632562,1632844,1632847,1632854,1633062-1633063,1633108,1633193,1633202,1633282,1633284,1633295,1633336,1633339,1633345,1633348,1633364,1633378,1633383,1633432,1633546,1633590,1633634,1633668,1633727,1633760,1633764,1633786,1633910,1633923,1634021,1634028,1634067,1634132,1634145,1634148,163
4155,1634188,1634202,1634264,1634373,1634530,1634688,1634850,1634857,1635103,1635116,1635421,1635438,1635478,1635481,1635484,1635490,1635809,1635939,1636146,1636167,1636180,1636207,1636215,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1637310,1637350,1637364,1637373,1637378,1639259,1639593,1639600,1640018,1640101,1640199,1640204,1640208,1640314,1640319,1640749,1640772,1640805,1640888,1640925,1640941-1640942,1641222,1641328,1641557,1641559,1641629,1641633,1641724,1641754,1641911,1642163,1642255,1642318,1642531,1642650,1642658,1642673,1642716,1644197,1644399,1644538,1644920,1644931,1646317,1646397,1646403,1646408,1646640,1646947,1647574,1647585,1647608,1648686,1648976,1649201,1649203,1649529,1649605,1649628,1649794,1650351,1650722,1650741-1650742,1650745,1650747,1650911,1650954,1651332,1651539,1651907,1651921,1652071,1652974,1653175,1653899,1654651,1655205,1655261,1655264,1655377,1655411,1655618,1655914,1657346,1657443,1658004,1658036,1658121,1658155,1658188,1658463,1658476,
1660258,1660276,1661454,1665848,1666160,1666781,1666820,1668312,1669100,1669238,1669487,1669523,1669586,1669660,1670614,1670625,1670715,1671496,1672169,1672301
+/manifoldcf/trunk:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1631750,1631953,1632013,1632225,1632289,1632562,1632844,1632847,1632854,1633062-1633063,1633108,1633193,1633202,1633282,1633284,1633295,1633336,1633339,1633345,1633348,1633364,1633378,1633383,1633432,1633546,1633590,1633634,1633668,1633727,1633760,1633764,1633786,1633910,1633923,1634021,1634028,1634067,1634132,1634145,1634148,163
4155,1634188,1634202,1634264,1634373,1634530,1634688,1634850,1634857,1635103,1635116,1635421,1635438,1635478,1635481,1635484,1635490,1635809,1635939,1636146,1636167,1636180,1636207,1636215,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1637310,1637350,1637364,1637373,1637378,1639259,1639593,1639600,1640018,1640101,1640199,1640204,1640208,1640314,1640319,1640749,1640772,1640805,1640888,1640925,1640941-1640942,1641222,1641328,1641557,1641559,1641629,1641633,1641724,1641754,1641911,1642163,1642255,1642318,1642531,1642650,1642658,1642673,1642716,1644197,1644399,1644538,1644920,1644931,1646317,1646397,1646403,1646408,1646640,1646947,1647574,1647585,1647608,1648686,1648976,1649201,1649203,1649529,1649605,1649628,1649794,1650351,1650722,1650741-1650742,1650745,1650747,1650911,1650954,1651332,1651539,1651907,1651921,1652071,1652974,1653175,1653899,1654651,1655205,1655261,1655264,1655377,1655411,1655618,1655914,1657346,1657443,1658004,1658036,1658121,1658155,1658188,1658463,1658476,
1660258,1660276,1661454,1665848,1666160,1666781,1666820,1668312,1669100,1669238,1669487,1669523,1669586,1669660,1670614,1670625,1670715,1671496,1672169,1672301,1672616
Modified: manifoldcf/branches/dev_1x/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/CHANGES.txt?rev=1672621&r1=1672620&r2=1672621&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/CHANGES.txt (original)
+++ manifoldcf/branches/dev_1x/CHANGES.txt Fri Apr 10 11:55:39 2015
@@ -3,6 +3,10 @@ $Id$
======================= 1.9-dev =====================
+CONNECTORS-1185: Fix HttpClient deprecation warnings, and make sure
+socket timeouts are actually honored in http connections.
+(Karl Wright)
+
CONNECTORS-1183: Fix for GridFS connector broken UI server tab.
(Kishore Kumar)
Modified: manifoldcf/branches/dev_1x/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1672621&r1=1672620&r2=1672621&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java (original)
+++ manifoldcf/branches/dev_1x/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java Fri Apr 10 11:55:39 2015
@@ -123,20 +123,24 @@ public class ElasticSearchConnector exte
{
if (client == null)
{
- connectionManager = new PoolingHttpClientConnectionManager();
-
int socketTimeout = 900000;
int connectionTimeout = 60000;
// Set up connection manager
- connectionManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ connectionManager = poolingConnectionManager;
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
@@ -146,10 +150,6 @@ public class ElasticSearchConnector exte
.setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- //.setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(credentialsProvider)
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
.build();
Modified: manifoldcf/branches/dev_1x/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraSession.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraSession.java?rev=1672621&r1=1672620&r2=1672621&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraSession.java (original)
+++ manifoldcf/branches/dev_1x/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraSession.java Fri Apr 10 11:55:39 2015
@@ -63,6 +63,7 @@ import org.apache.http.util.EntityUtils;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.protocol.HttpContext;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.entity.ContentType;
import org.apache.http.client.AuthCache;
import org.apache.http.impl.client.BasicAuthCache;
@@ -123,9 +124,16 @@ public class JiraSession {
javax.net.ssl.SSLSocketFactory httpsSocketFactory = KeystoreManagerFactory.getTrustingSecureSocketFactory();
SSLConnectionSocketFactory myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(httpsSocketFactory,connectionTimeout),
- SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ NoopHostnameVerifier.INSTANCE);
- connectionManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ connectionManager = poolingConnectionManager;
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
@@ -140,7 +148,6 @@ public class JiraSession {
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
@@ -168,13 +175,8 @@ public class JiraSession {
httpClient = HttpClients.custom()
.setConnectionManager(connectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(credentialsProvider)
.setSSLSocketFactory(myFactory)
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
Modified: manifoldcf/branches/dev_1x/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java?rev=1672621&r1=1672620&r2=1672621&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java (original)
+++ manifoldcf/branches/dev_1x/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java Fri Apr 10 11:55:39 2015
@@ -61,6 +61,7 @@ import org.apache.http.util.EntityUtils;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.protocol.HttpContext;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.entity.ContentType;
import org.apache.http.client.AuthCache;
import org.apache.http.impl.client.BasicAuthCache;
@@ -121,9 +122,16 @@ public class JiraSession {
javax.net.ssl.SSLSocketFactory httpsSocketFactory = KeystoreManagerFactory.getTrustingSecureSocketFactory();
SSLConnectionSocketFactory myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(httpsSocketFactory,connectionTimeout),
- SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ NoopHostnameVerifier.INSTANCE);
- connectionManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ connectionManager = poolingConnectionManager;
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
@@ -138,7 +146,6 @@ public class JiraSession {
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
@@ -167,13 +174,8 @@ public class JiraSession {
httpClient = HttpClients.custom()
.setConnectionManager(connectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(credentialsProvider)
.setSSLSocketFactory(myFactory)
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
Modified: manifoldcf/branches/dev_1x/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java?rev=1672621&r1=1672620&r2=1672621&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java (original)
+++ manifoldcf/branches/dev_1x/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java Fri Apr 10 11:55:39 2015
@@ -49,7 +49,7 @@ import org.apache.http.client.methods.Ht
import org.apache.http.client.methods.HttpPost;
import org.apache.http.NameValuePair;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
@@ -498,8 +498,15 @@ public class LivelinkConnector extends o
int connectionTimeout = 300000;
// Set up connection manager
- connectionManager = new PoolingHttpClientConnectionManager();
-
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ connectionManager = poolingConnectionManager;
+
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
// Set up ingest ssl if indicated
@@ -507,7 +514,7 @@ public class LivelinkConnector extends o
if (ingestKeystoreManager != null)
{
myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(ingestKeystoreManager.getSecureSocketFactory(), connectionTimeout),
- new BrowserCompatHostnameVerifier());
+ NoopHostnameVerifier.INSTANCE);
}
// Set up authentication to use
@@ -519,20 +526,14 @@ public class LivelinkConnector extends o
HttpClientBuilder builder = HttpClients.custom()
.setConnectionManager(connectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout)
.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(credentialsProvider)
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
.setRedirectStrategy(new DefaultRedirectStrategy());
Modified: manifoldcf/branches/dev_1x/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/meridio/meridiowrapper/MeridioWrapper.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/meridio/meridiowrapper/MeridioWrapper.java?rev=1672621&r1=1672620&r2=1672621&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/meridio/meridiowrapper/MeridioWrapper.java (original)
+++ manifoldcf/branches/dev_1x/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/meridio/meridiowrapper/MeridioWrapper.java Fri Apr 10 11:55:39 2015
@@ -257,9 +257,32 @@ public class MeridioWrapper
int socketTimeout = 900000;
int connectionTimeout = 300000;
- dmwsConnectionManager = new PoolingHttpClientConnectionManager();
- rmwsConnectionManager = new PoolingHttpClientConnectionManager();
- mcwsConnectionManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingDmwsConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingDmwsConnectionManager.setDefaultMaxPerRoute(1);
+ poolingDmwsConnectionManager.setValidateAfterInactivity(60000);
+ poolingDmwsConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ dmwsConnectionManager = poolingDmwsConnectionManager;
+
+ PoolingHttpClientConnectionManager poolingRmwsConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingRmwsConnectionManager.setDefaultMaxPerRoute(1);
+ poolingRmwsConnectionManager.setValidateAfterInactivity(60000);
+ poolingRmwsConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ rmwsConnectionManager = poolingRmwsConnectionManager;
+
+ PoolingHttpClientConnectionManager poolingMcwsConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingMcwsConnectionManager.setDefaultMaxPerRoute(1);
+ poolingMcwsConnectionManager.setValidateAfterInactivity(60000);
+ poolingMcwsConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ mcwsConnectionManager = poolingMcwsConnectionManager;
// Initialize the three httpclient objects
@@ -269,14 +292,12 @@ public class MeridioWrapper
RequestConfig.Builder dmwsRequestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(false)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
RequestConfig.Builder rmwsRequestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
@@ -326,13 +347,8 @@ public class MeridioWrapper
dmwsHttpClient = HttpClients.custom()
.setConnectionManager(dmwsConnectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(dmwsRequestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(dmwsCredentialsProvider)
.setSSLSocketFactory(myFactory)
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
@@ -341,13 +357,8 @@ public class MeridioWrapper
rmwsHttpClient = HttpClients.custom()
.setConnectionManager(rmwsConnectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(rmwsRequestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(rmwsCredentialsProvider)
.setSSLSocketFactory(myFactory)
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
@@ -361,7 +372,6 @@ public class MeridioWrapper
RequestConfig.Builder mcwsRequestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
@@ -391,13 +401,8 @@ public class MeridioWrapper
mcwsHttpClient = HttpClients.custom()
.setConnectionManager(mcwsConnectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(mcwsRequestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(mcwsCredentialsProvider)
.setSSLSocketFactory(myFactory)
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
Modified: manifoldcf/branches/dev_1x/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java?rev=1672621&r1=1672620&r2=1672621&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java (original)
+++ manifoldcf/branches/dev_1x/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java Fri Apr 10 11:55:39 2015
@@ -113,28 +113,31 @@ public class OpenSearchServerConnector e
{
if (client == null)
{
- connectionManager = new PoolingHttpClientConnectionManager();
-
final int executorTimeout = 300000;
final int socketTimeout = 60000;
final int connectionTimeout = 60000;
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ connectionManager = poolingConnectionManager;
+
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true).setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true).setExpectContinueEnabled(true)
+ .setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
HttpClientBuilder clientBuilder = HttpClients
.custom()
.setConnectionManager(connectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setRequestExecutor(new HttpRequestExecutor(executorTimeout))
- .setDefaultSocketConfig(
- SocketConfig.custom().setTcpNoDelay(true)
- .setSoTimeout(socketTimeout).build());
+ .setRequestExecutor(new HttpRequestExecutor(executorTimeout));
client = clientBuilder.build();
Propchange: manifoldcf/branches/dev_1x/connectors/rss/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Apr 10 11:55:39 2015
@@ -51,4 +51,4 @@
/manifoldcf/branches/CONNECTORS-912/connectors/rss:1579605-1582723
/manifoldcf/branches/CONNECTORS-962/connectors/rss:1602683-1603198
/manifoldcf/branches/CONNECTORS-990/connectors/rss:1610284-1610707
-/manifoldcf/trunk/connectors/rss:1621613,1621855,1622740,1622850,1624906,1628798,1633727,1633764,1634202,1650911
+/manifoldcf/trunk/connectors/rss:1621613,1621855,1622740,1622850,1624906,1628798,1633727,1633764,1634202,1650911,1672616
Modified: manifoldcf/branches/dev_1x/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java?rev=1672621&r1=1672620&r2=1672621&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java (original)
+++ manifoldcf/branches/dev_1x/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java Fri Apr 10 11:55:39 2015
@@ -39,7 +39,7 @@ import org.apache.http.client.config.Req
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.config.SocketConfig;
-import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
@@ -260,16 +260,22 @@ public class ThrottledFetcher
// Create the https scheme for this connection
javax.net.ssl.SSLSocketFactory httpsSocketFactory = KeystoreManagerFactory.getTrustingSecureSocketFactory();;
SSLConnectionSocketFactory myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(httpsSocketFactory,connectionTimeoutMilliseconds),
- SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ NoopHostnameVerifier.INSTANCE);
- connectionManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(connectionTimeoutMilliseconds)
+ .build());
+ connectionManager = poolingConnectionManager;
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(connectionTimeoutMilliseconds)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeoutMilliseconds)
.setConnectionRequestTimeout(connectionTimeoutMilliseconds);
@@ -301,10 +307,6 @@ public class ThrottledFetcher
.setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(connectionTimeoutMilliseconds)
- .build())
.setDefaultCredentialsProvider(credentialsProvider)
.setSSLSocketFactory(myFactory)
.setRequestExecutor(new HttpRequestExecutor(connectionTimeoutMilliseconds))
Propchange: manifoldcf/branches/dev_1x/connectors/sharepoint/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Apr 10 11:55:39 2015
@@ -59,4 +59,4 @@
/manifoldcf/branches/CONNECTORS-891/connectors/sharepoint:1568778-1568807
/manifoldcf/branches/CONNECTORS-912/connectors/sharepoint:1579605-1582723
/manifoldcf/branches/CONNECTORS-990/connectors/sharepoint:1610284-1610707
-/manifoldcf/trunk/connectors/sharepoint:1621613,1621855,1622740,1623599,1625023,1625108,1626973,1630188,1633668,1640199,1642531,1665848
+/manifoldcf/trunk/connectors/sharepoint:1621613,1621855,1622740,1623599,1625023,1625108,1626973,1630188,1633668,1640199,1642531,1665848,1672616
Modified: manifoldcf/branches/dev_1x/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java?rev=1672621&r1=1672620&r2=1672621&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java (original)
+++ manifoldcf/branches/dev_1x/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java Fri Apr 10 11:55:39 2015
@@ -45,7 +45,7 @@ import org.apache.http.client.Credential
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.conn.ssl.DefaultHostnameVerifier;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
@@ -791,7 +791,14 @@ public class SharePointAuthority extends
// Set up ssl if indicated
- connectionManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ connectionManager = poolingConnectionManager;
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
@@ -799,7 +806,7 @@ public class SharePointAuthority extends
if (keystoreData != null)
{
keystoreManager = KeystoreManagerFactory.make("",keystoreData);
- myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), new BrowserCompatHostnameVerifier());
+ myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), new DefaultHostnameVerifier());
}
if (strippedUserName != null)
@@ -812,7 +819,6 @@ public class SharePointAuthority extends
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(false)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
@@ -841,13 +847,8 @@ public class SharePointAuthority extends
HttpClientBuilder builder = HttpClients.custom()
.setConnectionManager(connectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(credentialsProvider);
if (myFactory != null)
builder.setSSLSocketFactory(myFactory);
Modified: manifoldcf/branches/dev_1x/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java?rev=1672621&r1=1672620&r2=1672621&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java (original)
+++ manifoldcf/branches/dev_1x/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java Fri Apr 10 11:55:39 2015
@@ -55,7 +55,7 @@ import org.apache.http.client.Credential
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.conn.ssl.DefaultHostnameVerifier;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
@@ -254,15 +254,22 @@ public class SharePointRepository extend
int connectionTimeout = 60000;
int socketTimeout = 900000;
- connectionManager = new PoolingHttpClientConnectionManager();
-
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ connectionManager = poolingConnectionManager;
+
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
SSLConnectionSocketFactory myFactory = null;
if (keystoreData != null)
{
keystoreManager = KeystoreManagerFactory.make("",keystoreData);
- myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), new BrowserCompatHostnameVerifier());
+ myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), new DefaultHostnameVerifier());
}
if (strippedUserName != null)
@@ -275,7 +282,6 @@ public class SharePointRepository extend
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(false)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
@@ -304,13 +310,8 @@ public class SharePointRepository extend
HttpClientBuilder builder = HttpClients.custom()
.setConnectionManager(connectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(credentialsProvider);
if (myFactory != null)
builder.setSSLSocketFactory(myFactory);
Modified: manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1672621&r1=1672620&r2=1672621&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java (original)
+++ manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java Fri Apr 10 11:55:39 2015
@@ -23,6 +23,7 @@ import org.apache.http.client.HttpClient
import org.apache.http.client.config.RequestConfig;
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.impl.client.*;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.protocol.HttpRequestExecutor;
@@ -217,40 +218,39 @@ public class HttpPoster
// Initialize standard solr-j.
// First, we need an HttpClient where basic auth is properly set up.
connectionManager = new PoolingHttpClientConnectionManager();
- connectionManager.setMaxTotal(1);
-
+ connectionManager.setDefaultMaxPerRoute(1);
+ connectionManager.setValidateAfterInactivity(60000);
+ connectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+
SSLConnectionSocketFactory myFactory;
if (keystoreManager != null)
{
- myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), NoopHostnameVerifier.INSTANCE);
}
else
{
// Use the "trust everything" one
- myFactory = new SSLConnectionSocketFactory(KeystoreManagerFactory.getTrustingSecureSocketFactory(),SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ myFactory = new SSLConnectionSocketFactory(KeystoreManagerFactory.getTrustingSecureSocketFactory(),NoopHostnameVerifier.INSTANCE);
}
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
- HttpClientBuilder clientBuilder = HttpClients.custom()
- .setConnectionManager(connectionManager)
- .setMaxConnTotal(1)
- .disableAutomaticRetries()
- .setDefaultRequestConfig(requestBuilder.build())
- .setRedirectStrategy(new DefaultRedirectStrategy())
- .setSSLSocketFactory(myFactory)
- .setRequestExecutor(new HttpRequestExecutor(socketTimeout))
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build()
- );
+ HttpClientBuilder clientBuilder = HttpClients.custom()
+ .setConnectionManager(connectionManager)
+ .setMaxConnTotal(1)
+ .disableAutomaticRetries()
+ .setDefaultRequestConfig(requestBuilder.build())
+ .setRedirectStrategy(new DefaultRedirectStrategy())
+ .setSSLSocketFactory(myFactory)
+ .setRequestExecutor(new HttpRequestExecutor(socketTimeout));
if (userID != null && userID.length() > 0 && password != null)
Modified: manifoldcf/branches/dev_1x/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java?rev=1672621&r1=1672620&r2=1672621&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java (original)
+++ manifoldcf/branches/dev_1x/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java Fri Apr 10 11:55:39 2015
@@ -66,14 +66,13 @@ public interface IThrottledConnection
* @param urlPath is the path part of the url, e.g. "/robots.txt"
* @param userAgent is the value of the userAgent header to use.
* @param from is the value of the from header to use.
- * @param connectionTimeoutMilliseconds is the maximum number of milliseconds to wait on socket connect.
* @param redirectOK should be set to true if you want redirects to be automatically followed.
* @param host is the value to use as the "Host" header, or null to use the default.
* @param formData describes additional form arguments and how to fetch the page.
* @param loginCookies describes the cookies that should be in effect for this page fetch.
*/
- public void executeFetch(String urlPath, String userAgent, String from, int connectionTimeoutMilliseconds,
- int socketTimeoutMilliseconds, boolean redirectOK, String host, FormData formData,
+ public void executeFetch(String urlPath, String userAgent, String from,
+ boolean redirectOK, String host, FormData formData,
LoginCookies loginCookies)
throws ManifoldCFException, ServiceInterruption;
Modified: manifoldcf/branches/dev_1x/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java?rev=1672621&r1=1672620&r2=1672621&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java (original)
+++ manifoldcf/branches/dev_1x/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java Fri Apr 10 11:55:39 2015
@@ -48,7 +48,7 @@ import org.apache.http.client.methods.Ht
import org.apache.http.client.methods.HttpPost;
import org.apache.http.NameValuePair;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
@@ -68,9 +68,16 @@ import org.apache.http.protocol.HTTP;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.cookie.CookieOrigin;
import org.apache.http.cookie.ClientCookie;
+import org.apache.http.cookie.CookieSpecProvider;
import org.apache.http.cookie.Cookie;
+import org.apache.http.impl.cookie.DefaultCookieSpec;
+import org.apache.http.impl.cookie.LaxBrowserCompatSpec;
+import org.apache.http.impl.cookie.RFC6265CookieSpecProvider;
import org.apache.http.impl.cookie.BasicPathHandler;
-import org.apache.http.impl.cookie.BrowserCompatSpec;
+import org.apache.http.impl.cookie.LaxMaxAgeHandler;
+import org.apache.http.impl.cookie.BasicDomainHandler;
+import org.apache.http.impl.cookie.BasicSecureHandler;
+import org.apache.http.impl.cookie.LaxExpiresHandler;
import org.apache.http.cookie.CookieSpec;
import org.apache.http.client.CookieStore;
import org.apache.http.protocol.HttpContext;
@@ -78,15 +85,9 @@ import org.apache.http.protocol.BasicHtt
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.cookie.CookieIdentityComparator;
import org.apache.http.client.HttpRequestRetryHandler;
-import org.apache.http.cookie.CookieSpecProvider;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.config.Registry;
import org.apache.http.client.config.CookieSpecs;
-import org.apache.http.impl.cookie.BestMatchSpecFactory;
-import org.apache.http.impl.cookie.BrowserCompatSpecFactory;
-import org.apache.http.impl.cookie.RFC2965SpecFactory;
-import org.apache.http.impl.cookie.NetscapeDraftSpecFactory;
-import org.apache.http.impl.cookie.IgnoreSpecFactory;
import org.apache.http.cookie.MalformedCookieException;
import org.apache.http.conn.ConnectTimeoutException;
@@ -146,11 +147,7 @@ public class ThrottledFetcher
private static final Registry<CookieSpecProvider> cookieSpecRegistry =
RegistryBuilder.<CookieSpecProvider>create()
- .register(CookieSpecs.BEST_MATCH, new BestMatchSpecFactory())
- .register(CookieSpecs.STANDARD, new RFC2965SpecFactory())
- .register(CookieSpecs.BROWSER_COMPATIBILITY, new LaxBrowserCompatSpecFactory())
- .register(CookieSpecs.NETSCAPE, new NetscapeDraftSpecFactory())
- .register(CookieSpecs.IGNORE_COOKIES, new IgnoreSpecFactory())
+ .register(CookieSpecs.STANDARD, new LaxBrowserCompatSpecProvider())
.build();
/** Constructor. Private since we never instantiate.
@@ -182,6 +179,7 @@ public class ThrottledFetcher
IThrottleSpec throttleDescription, String[] binNames,
int connectionLimit,
String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword,
+ int socketTimeoutMilliseconds, int connectionTimeoutMilliseconds,
IAbortActivity activities)
throws ManifoldCFException, ServiceInterruption
{
@@ -207,7 +205,8 @@ public class ThrottledFetcher
// Construct a connection pool key
ConnectionPoolKey poolKey = new ConnectionPoolKey(protocol,server,port,authentication,
- trustStoreString,proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);
+ trustStoreString,proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword,
+ socketTimeoutMilliseconds,connectionTimeoutMilliseconds);
ConnectionPool p;
synchronized (connectionPools)
@@ -219,7 +218,8 @@ public class ThrottledFetcher
IConnectionThrottler connectionThrottler =
throttleGroups.obtainConnectionThrottler(webThrottleGroupType,throttleGroupName,binNames);
p = new ConnectionPool(connectionThrottler,protocol,server,port,authentication,baseFactory,
- proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);
+ proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword,
+ socketTimeoutMilliseconds,connectionTimeoutMilliseconds);
connectionPools.put(poolKey,p);
}
}
@@ -293,6 +293,10 @@ public class ThrottledFetcher
protected final String proxyAuthPassword;
/** Https protocol */
protected final javax.net.ssl.SSLSocketFactory httpsSocketFactory;
+ /** Socket timeout milliseconds */
+ protected final int socketTimeoutMilliseconds;
+ /** Connection timeout milliseconds */
+ protected final int connectionTimeoutMilliseconds;
/** The thread that is actually doing the work */
protected ExecuteMethodThread methodThread = null;
@@ -307,7 +311,8 @@ public class ThrottledFetcher
public ThrottledConnection(ConnectionPool myPool, IFetchThrottler fetchThrottler,
String protocol, String server, int port, PageCredentials authentication,
javax.net.ssl.SSLSocketFactory httpsSocketFactory,
- String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)
+ String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword,
+ int socketTimeoutMilliseconds, int connectionTimeoutMilliseconds)
{
this.myPool = myPool;
this.fetchThrottler = fetchThrottler;
@@ -321,6 +326,8 @@ public class ThrottledFetcher
this.port = port;
this.authentication = authentication;
this.httpsSocketFactory = httpsSocketFactory;
+ this.socketTimeoutMilliseconds = socketTimeoutMilliseconds;
+ this.connectionTimeoutMilliseconds = connectionTimeoutMilliseconds;
}
/** Set the abort checker. This must be done before the connection is actually used.
@@ -401,21 +408,20 @@ public class ThrottledFetcher
* @param urlPath is the path part of the url, e.g. "/robots.txt"
* @param userAgent is the value of the userAgent header to use.
* @param from is the value of the from header to use.
- * @param connectionTimeoutMilliseconds is the maximum number of milliseconds to wait on socket connect.
* @param redirectOK should be set to true if you want redirects to be automatically followed.
* @param host is the value to use as the "Host" header, or null to use the default.
* @param formData describes additional form arguments and how to fetch the page.
* @param loginCookies describes the cookies that should be in effect for this page fetch.
*/
@Override
- public void executeFetch(String urlPath, String userAgent, String from, int connectionTimeoutMilliseconds,
- int socketTimeoutMilliseconds, boolean redirectOK, String host, FormData formData,
+ public void executeFetch(String urlPath, String userAgent, String from,
+ boolean redirectOK, String host, FormData formData,
LoginCookies loginCookies)
throws ManifoldCFException, ServiceInterruption
{
// Set up scheme
SSLConnectionSocketFactory myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(httpsSocketFactory,connectionTimeoutMilliseconds),
- SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ NoopHostnameVerifier.INSTANCE);
int hostPort;
String displayedPort;
@@ -461,7 +467,14 @@ public class ThrottledFetcher
if (connManager == null)
{
- connManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingConnManager = new PoolingHttpClientConnectionManager();
+ poolingConnManager.setDefaultMaxPerRoute(1);
+ poolingConnManager.setValidateAfterInactivity(60000);
+ poolingConnManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeoutMilliseconds)
+ .build());
+ connManager = poolingConnManager;
}
long startTime = 0L;
@@ -485,11 +498,10 @@ public class ThrottledFetcher
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeoutMilliseconds)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeoutMilliseconds)
.setConnectionRequestTimeout(socketTimeoutMilliseconds)
- .setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY)
+ .setCookieSpec(CookieSpecs.STANDARD)
.setRedirectsEnabled(redirectOK);
// If there's a proxy, set that too.
@@ -511,15 +523,8 @@ public class ThrottledFetcher
httpClient = HttpClients.custom()
.setConnectionManager(connManager)
- .setMaxConnTotal(1)
- .setMaxConnPerRoute(1)
.disableAutomaticRetries()
- .setDefaultCookieSpecRegistry(cookieSpecRegistry)
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeoutMilliseconds)
- .build())
.setDefaultCredentialsProvider(credentialsProvider)
.setSSLSocketFactory(myFactory)
.setRequestExecutor(new HttpRequestExecutor(socketTimeoutMilliseconds))
@@ -1311,35 +1316,14 @@ public class ThrottledFetcher
/** Class to create a cookie spec.
*/
- protected static class LaxBrowserCompatSpecFactory extends BrowserCompatSpecFactory
+ protected static class LaxBrowserCompatSpecProvider extends RFC6265CookieSpecProvider
{
+ @Override
public CookieSpec create(HttpContext context)
{
return new LaxBrowserCompatSpec();
}
}
-
- /** Class to override browser compatibility to make it not check cookie paths. See CONNECTORS-97.
- */
- protected static class LaxBrowserCompatSpec extends BrowserCompatSpec
- {
-
- public LaxBrowserCompatSpec()
- {
- super();
- registerAttribHandler(ClientCookie.PATH_ATTR, new BasicPathHandler()
- {
- @Override
- public void validate(Cookie cookie, CookieOrigin origin) throws MalformedCookieException
- {
- // No validation
- }
-
- }
- );
- }
-
- }
/** This thread does the actual socket communication with the server.
* It's set up so that it can be abandoned at shutdown time.
@@ -1858,11 +1842,14 @@ public class ThrottledFetcher
protected final String proxyAuthDomain;
protected final String proxyAuthUsername;
protected final String proxyAuthPassword;
+ protected final int socketTimeoutMilliseconds;
+ protected final int connectionTimeoutMilliseconds;
public ConnectionPoolKey(String protocol,
String server, int port, PageCredentials authentication,
String trustStoreString, String proxyHost, int proxyPort,
- String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)
+ String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword,
+ int socketTimeoutMilliseconds, int connectionTimeoutMilliseconds)
{
this.protocol = protocol;
this.server = server;
@@ -1874,6 +1861,8 @@ public class ThrottledFetcher
this.proxyAuthDomain = proxyAuthDomain;
this.proxyAuthUsername = proxyAuthUsername;
this.proxyAuthPassword = proxyAuthPassword;
+ this.socketTimeoutMilliseconds = socketTimeoutMilliseconds;
+ this.connectionTimeoutMilliseconds = connectionTimeoutMilliseconds;
}
public int hashCode()
@@ -1887,7 +1876,9 @@ public class ThrottledFetcher
(proxyPort * 29) +
((proxyAuthDomain==null)?0:proxyAuthDomain.hashCode()) +
((proxyAuthUsername==null)?0:proxyAuthUsername.hashCode()) +
- ((proxyAuthPassword==null)?0:proxyAuthPassword.hashCode());
+ ((proxyAuthPassword==null)?0:proxyAuthPassword.hashCode()) +
+ new Integer(socketTimeoutMilliseconds).hashCode() +
+ new Integer(connectionTimeoutMilliseconds).hashCode();
}
public boolean equals(Object o)
@@ -1960,7 +1951,8 @@ public class ThrottledFetcher
if (!proxyAuthPassword.equals(other.proxyAuthPassword))
return false;
}
- return true;
+ return socketTimeoutMilliseconds == other.socketTimeoutMilliseconds &&
+ connectionTimeoutMilliseconds == other.connectionTimeoutMilliseconds;
}
}
@@ -1983,6 +1975,8 @@ public class ThrottledFetcher
protected final String proxyAuthDomain;
protected final String proxyAuthUsername;
protected final String proxyAuthPassword;
+ protected final int socketTimeoutMilliseconds;
+ protected final int connectionTimeoutMilliseconds;
/** The actual pool of connections */
protected final List<IThrottledConnection> connections = new ArrayList<IThrottledConnection>();
@@ -1992,7 +1986,8 @@ public class ThrottledFetcher
String server, int port, PageCredentials authentication,
javax.net.ssl.SSLSocketFactory baseFactory,
String proxyHost, int proxyPort,
- String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)
+ String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword,
+ int socketTimeoutMilliseconds, int connectionTimeoutMilliseconds)
{
this.connectionThrottler = connectionThrottler;
@@ -2006,6 +2001,8 @@ public class ThrottledFetcher
this.proxyAuthDomain = proxyAuthDomain;
this.proxyAuthUsername = proxyAuthUsername;
this.proxyAuthPassword = proxyAuthPassword;
+ this.socketTimeoutMilliseconds = socketTimeoutMilliseconds;
+ this.connectionTimeoutMilliseconds = connectionTimeoutMilliseconds;
}
public IThrottledConnection grab(IAbortActivity activities)
@@ -2030,7 +2027,8 @@ public class ThrottledFetcher
connection = new ThrottledConnection(this,connectionThrottler.getNewConnectionFetchThrottler(),
protocol,server,port,authentication,baseFactory,
proxyHost,proxyPort,
- proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);
+ proxyAuthDomain,proxyAuthUsername,proxyAuthPassword,
+ socketTimeoutMilliseconds,connectionTimeoutMilliseconds);
}
else
throw new IllegalStateException("Unexpected return value from waitConnectionAvailable(): "+result);
Modified: manifoldcf/branches/dev_1x/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java?rev=1672621&r1=1672620&r2=1672621&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java (original)
+++ manifoldcf/branches/dev_1x/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java Fri Apr 10 11:55:39 2015
@@ -888,6 +888,7 @@ public class WebcrawlerConnector extends
protocol,ipAddress,port,
credential,trustStore,throttleDescription,binNames,connectionLimit,
proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword,
+ socketTimeoutMilliseconds,connectionTimeoutMilliseconds,
activities);
try
{
@@ -895,8 +896,8 @@ public class WebcrawlerConnector extends
try
{
// Execute the fetch!
- connection.executeFetch(url.getFile(),userAgent,from,connectionTimeoutMilliseconds,
- socketTimeoutMilliseconds,false,hostName,formData,lc);
+ connection.executeFetch(url.getFile(),userAgent,from,
+ false,hostName,formData,lc);
int response = connection.getResponseCode();
if (response == 200 || response == 302 || response == 301)
@@ -5606,13 +5607,14 @@ public class WebcrawlerConnector extends
protocol,hostIPAddress,port,credential,
trustStore,throttleDescription,binNames,connectionLimit,
proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword,
+ socketTimeoutMilliseconds,connectionTimeoutMilliseconds,
versionActivities);
try
{
connection.beginFetch(FETCH_ROBOTS);
try
{
- connection.executeFetch("/robots.txt",userAgent,from,connectionTimeoutMilliseconds,socketTimeoutMilliseconds,true,hostName,null,null);
+ connection.executeFetch("/robots.txt",userAgent,from,true,hostName,null,null);
long expirationTime = currentTime+1000*60*60*24;
int code = connection.getResponseCode();
if (code == 200)
Propchange: manifoldcf/branches/dev_1x/connectors/wiki/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Apr 10 11:55:39 2015
@@ -52,4 +52,4 @@
/manifoldcf/branches/CONNECTORS-774/connectors/wiki:1521768-1523232
/manifoldcf/branches/CONNECTORS-912/connectors/wiki:1579605-1582723
/manifoldcf/branches/CONNECTORS-962/connectors/wiki:1602683-1603198
-/manifoldcf/trunk/connectors/wiki:1621613,1621855,1622740,1625103,1628798,1630188,1633108,1633202
+/manifoldcf/trunk/connectors/wiki:1621613,1621855,1622740,1625103,1628798,1630188,1633108,1633202,1672616
Modified: manifoldcf/branches/dev_1x/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java?rev=1672621&r1=1672620&r2=1672621&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java (original)
+++ manifoldcf/branches/dev_1x/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java Fri Apr 10 11:55:39 2015
@@ -63,6 +63,7 @@ import org.apache.http.message.BasicHead
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.protocol.HttpContext;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.entity.ContentType;
import org.apache.http.conn.ConnectTimeoutException;
@@ -225,10 +226,17 @@ public class WikiConnector extends org.a
javax.net.ssl.SSLSocketFactory httpsSocketFactory = KeystoreManagerFactory.getTrustingSecureSocketFactory();
SSLConnectionSocketFactory myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(httpsSocketFactory,connectionTimeout),
- SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ NoopHostnameVerifier.INSTANCE);
// Set up connection manager
- connectionManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ connectionManager = poolingConnectionManager;
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
@@ -244,7 +252,6 @@ public class WikiConnector extends org.a
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
@@ -287,13 +294,8 @@ public class WikiConnector extends org.a
httpClient = HttpClients.custom()
.setConnectionManager(connectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(credentialsProvider)
.setSSLSocketFactory(myFactory)
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
Propchange: manifoldcf/branches/dev_1x/framework/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Apr 10 11:55:39 2015
@@ -113,4 +113,4 @@
/manifoldcf/branches/CONNECTORS-989/framework:1611600-1612101
/manifoldcf/branches/CONNECTORS-990/framework:1610284-1610707
/manifoldcf/trunk:1629122
-/manifoldcf/trunk/framework:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1632013,1632289,1632844,1633108,1633193,1633202,1633348,1633364,1634145,1634148,1634155,1634264,1634373,1634530,1635438,1635809,1636146,1636180,1636207,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1639593,1639600,1640018,1640101,1640199,1640314,1640319,1640749,1640772,1640925,1640941,1641222,1641557,1641559,1
641724,1641911,1642163,1642255,1642318,1644197,1644399,1646317,1646397,1646403,1646640,1647574,1647585,1647608,1649605,1650351,1650911,1651332,1651539,1651921,1655377,1655411,1657346,1658004,1658036,1660258,1660276,1669487,1670614
+/manifoldcf/trunk/framework:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1632013,1632289,1632844,1633108,1633193,1633202,1633348,1633364,1634145,1634148,1634155,1634264,1634373,1634530,1635438,1635809,1636146,1636180,1636207,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1639593,1639600,1640018,1640101,1640199,1640314,1640319,1640749,1640772,1640925,1640941,1641222,1641557,1641559,1
641724,1641911,1642163,1642255,1642318,1644197,1644399,1646317,1646397,1646403,1646640,1647574,1647585,1647608,1649605,1650351,1650911,1651332,1651539,1651921,1655377,1655411,1657346,1658004,1658036,1660258,1660276,1669487,1670614,1672616
Modified: manifoldcf/branches/dev_1x/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyShutdown.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyShutdown.java?rev=1672621&r1=1672620&r2=1672621&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyShutdown.java (original)
+++ manifoldcf/branches/dev_1x/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyShutdown.java Fri Apr 10 11:55:39 2015
@@ -71,25 +71,26 @@ public class ManifoldCFJettyShutdown
int socketTimeout = 900000;
int connectionTimeout = 300000;
- HttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ HttpClientConnectionManager connectionManager = poolingConnectionManager;
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(false)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
HttpClient httpClient = HttpClients.custom()
.setConnectionManager(connectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
.setRedirectStrategy(new DefaultRedirectStrategy())
.build();
Modified: manifoldcf/branches/dev_1x/framework/script-engine/src/main/java/org/apache/manifoldcf/scriptengine/ScriptParser.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/script-engine/src/main/java/org/apache/manifoldcf/scriptengine/ScriptParser.java?rev=1672621&r1=1672620&r2=1672621&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/script-engine/src/main/java/org/apache/manifoldcf/scriptengine/ScriptParser.java (original)
+++ manifoldcf/branches/dev_1x/framework/script-engine/src/main/java/org/apache/manifoldcf/scriptengine/ScriptParser.java Fri Apr 10 11:55:39 2015
@@ -1221,14 +1221,20 @@ public class ScriptParser
int socketTimeout = 900000;
int connectionTimeout = 300000;
- connectionManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ connectionManager = poolingConnectionManager;
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(false)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
@@ -1238,10 +1244,6 @@ public class ScriptParser
.setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(credentialsProvider)
//.setSSLSocketFactory(myFactory)
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))