You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/03/08 21:03:28 UTC
svn commit: r1454518 - in /manifoldcf/trunk: ./
connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/
connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver...
Author: kwright
Date: Fri Mar 8 20:03:28 2013
New Revision: 1454518
URL: http://svn.apache.org/r1454518
Log:
Add expect-continue to all output connectors. Otherwise auth won't work with httpclient. Fix for CONNECTORS-661.
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1454518&r1=1454517&r2=1454518&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Fri Mar 8 20:03:28 2013
@@ -3,6 +3,10 @@ $Id$
======================= 1.2-dev =====================
+CONNECTORS-661: Turn on expect-continue for solr output. Otherwise
+basic auth won't work.
+(Erlend Garåsen, Karl Wright)
+
CONNECTORS-657: Add a standard way of handling created and modified dates
throughout ManifoldCF, so that these can be dealt with consistently in
an output connector.
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1454518&r1=1454517&r2=1454518&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java Fri Mar 8 20:03:28 2013
@@ -33,6 +33,13 @@ import org.apache.http.client.HttpClient
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.protocol.HttpContext;
+import org.apache.http.params.HttpConnectionParams;
+import org.apache.http.params.HttpParams;
+import org.apache.http.params.BasicHttpParams;
+import org.apache.http.params.CoreConnectionPNames;
+import org.apache.http.params.CoreProtocolPNames;
+import org.apache.http.client.params.ClientPNames;
+import org.apache.http.client.params.HttpClientParams;
import org.apache.commons.io.FilenameUtils;
import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
@@ -116,7 +123,22 @@ public class ElasticSearchConnector exte
PoolingClientConnectionManager localConnectionManager = new PoolingClientConnectionManager();
localConnectionManager.setMaxTotal(1);
connectionManager = localConnectionManager;
- DefaultHttpClient localClient = new DefaultHttpClient(connectionManager);
+
+ int socketTimeout = 900000;
+ int connectionTimeout = 60000;
+
+ BasicHttpParams params = new BasicHttpParams();
+ // This one is essential to prevent us from reading from the content stream before necessary during auth, but
+ // is incompatible with some proxies.
+ params.setBooleanParameter(CoreProtocolPNames.USE_EXPECT_CONTINUE,true);
+ // Enabled for Solr, but probably not necessary for better-behaved ES
+ //params.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
+ params.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,true);
+ params.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
+ params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,socketTimeout);
+ params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,connectionTimeout);
+ params.setBooleanParameter(ClientPNames.HANDLE_REDIRECTS,true);
+ DefaultHttpClient localClient = new DefaultHttpClient(connectionManager,params);
// No retries
localClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
{
Modified: manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java?rev=1454518&r1=1454517&r2=1454518&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java (original)
+++ manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java Fri Mar 8 20:03:28 2013
@@ -34,6 +34,13 @@ import org.apache.http.client.HttpClient
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.protocol.HttpContext;
+import org.apache.http.params.HttpConnectionParams;
+import org.apache.http.params.HttpParams;
+import org.apache.http.params.BasicHttpParams;
+import org.apache.http.params.CoreConnectionPNames;
+import org.apache.http.params.CoreProtocolPNames;
+import org.apache.http.client.params.ClientPNames;
+import org.apache.http.client.params.HttpClientParams;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
@@ -124,7 +131,22 @@ public class OpenSearchServerConnector e
PoolingClientConnectionManager localConnectionManager = new PoolingClientConnectionManager();
localConnectionManager.setMaxTotal(1);
connectionManager = localConnectionManager;
- DefaultHttpClient localClient = new DefaultHttpClient(connectionManager);
+
+ int socketTimeout = 900000;
+ int connectionTimeout = 60000;
+
+ BasicHttpParams params = new BasicHttpParams();
+ // This one is essential to prevent us from reading from the content stream before necessary during auth, but
+ // is incompatible with some proxies.
+ params.setBooleanParameter(CoreProtocolPNames.USE_EXPECT_CONTINUE,true);
+ // Enabled for Solr, but probably not necessary for better-behaved ES
+ //params.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
+ params.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,true);
+ params.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
+ params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,socketTimeout);
+ params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,connectionTimeout);
+ params.setBooleanParameter(ClientPNames.HANDLE_REDIRECTS,true);
+ DefaultHttpClient localClient = new DefaultHttpClient(connectionManager,params);
// No retries
localClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
{
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1454518&r1=1454517&r2=1454518&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java Fri Mar 8 20:03:28 2013
@@ -44,6 +44,7 @@ import org.apache.http.params.HttpConnec
import org.apache.http.params.HttpParams;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.CoreConnectionPNames;
+import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.client.params.ClientPNames;
import org.apache.http.client.params.HttpClientParams;
import org.apache.http.conn.ClientConnectionManager;
@@ -228,6 +229,9 @@ public class HttpPoster
connectionManager = localConnectionManager;
BasicHttpParams params = new BasicHttpParams();
+ // This one is essential to prevent us from reading from the content stream before necessary during auth, but
+ // is incompatible with some proxies.
+ params.setBooleanParameter(CoreProtocolPNames.USE_EXPECT_CONTINUE,true);
params.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
params.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,true);
params.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);