You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2012/12/09 23:54:33 UTC
svn commit: r1419176 - in /manifoldcf/trunk: ./ connectors/elasticsearch/
connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/
connectors/opensearchserver/
connectors/opensearchserver/connector/src/main/ja...
Author: kwright
Date: Sun Dec 9 22:54:32 2012
New Revision: 1419176
URL: http://svn.apache.org/viewvc?rev=1419176&view=rev
Log:
Port OpenSearchServer connector and ElasticSearch connector to httpcomponents.
Modified:
manifoldcf/trunk/ (props changed)
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
manifoldcf/trunk/connectors/elasticsearch/pom.xml
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerAction.java
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerDelete.java
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerIndex.java
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerSchema.java
manifoldcf/trunk/connectors/opensearchserver/pom.xml
Propchange: manifoldcf/trunk/
------------------------------------------------------------------------------
Merged /manifoldcf/branches/CONNECTORS-581:r1419077-1419175
Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1419176&r1=1419175&r2=1419176&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Sun Dec 9 22:54:32 2012
@@ -3,6 +3,10 @@ $Id$
======================= 1.1-dev =====================
+CONNECTORS-581: Port OpenSearchServer connector and ElasticSearch
+connector to httpcomponents.
+(Karl Wright)
+
CONNECTORS-566: Simple JDBC Authority connector.
(Maciej Li¿ewski)
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java?rev=1419176&r1=1419175&r2=1419176&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java Sun Dec 9 22:54:32 2012
@@ -19,8 +19,9 @@
package org.apache.manifoldcf.agents.output.elasticsearch;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.HttpClient;
+
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
public class ElasticSearchAction extends ElasticSearchConnection
@@ -36,7 +37,7 @@ public class ElasticSearchAction extends
{
super(config, client);
StringBuffer url = getApiUrl(cmd.toString(), checkConnection);
- GetMethod method = new GetMethod(url.toString());
+ HttpGet method = new HttpGet(url.toString());
call(method);
if ("true".equals(checkJson(jsonStatus)))
return;
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java?rev=1419176&r1=1419175&r2=1419176&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java Sun Dec 9 22:54:32 2012
@@ -21,11 +21,38 @@ package org.apache.manifoldcf.agents.out
import java.io.IOException;
import java.io.UnsupportedEncodingException;
+import java.io.Writer;
+import java.io.StringWriter;
+import java.io.Reader;
+import java.io.InputStreamReader;
+import java.io.InputStream;
import java.net.URLEncoder;
-import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.HttpClient;
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.impl.conn.PoolingClientConnectionManager;
+import org.apache.http.client.HttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.impl.client.DefaultRedirectStrategy;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.Header;
+import org.apache.http.params.CoreProtocolPNames;
+import org.apache.http.params.HttpProtocolParams;
+import org.apache.http.ProtocolVersion;
+import org.apache.http.util.EntityUtils;
+import org.apache.http.message.BasicHeader;
+
+import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.client.RedirectException;
+import org.apache.http.client.CircularRedirectException;
+import org.apache.http.NoHttpResponseException;
+import org.apache.http.HttpException;
+
import org.apache.commons.io.IOUtils;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
@@ -87,15 +114,14 @@ public class ElasticSearchConnection
return url;
}
- protected void call(HttpMethod method) throws ManifoldCFException
+ protected void call(HttpRequestBase method) throws ManifoldCFException
{
- HttpClient hc = client;
try
{
- hc.executeMethod(method);
- if (!checkResultCode(method.getStatusCode()))
+ HttpResponse resp = client.execute(method);
+ if (!checkResultCode(resp.getStatusLine().getStatusCode()))
throw new ManifoldCFException(getResultDescription());
- response = IOUtils.toString(method.getResponseBodyAsStream());
+ response = getResponseBodyAsString(resp.getEntity());
} catch (HttpException e)
{
setResult(Result.ERROR, e.getMessage());
@@ -107,8 +133,45 @@ public class ElasticSearchConnection
} finally
{
if (method != null)
- method.releaseConnection();
+ method.abort();
+ }
+ }
+
+ private static String getResponseBodyAsString(HttpEntity entity)
+ throws IOException, HttpException {
+ InputStream is = entity.getContent();
+ if (is != null)
+ {
+ try
+ {
+ String charSet = EntityUtils.getContentCharSet(entity);
+ if (charSet == null)
+ charSet = "utf-8";
+ char[] buffer = new char[65536];
+ Reader r = new InputStreamReader(is,charSet);
+ Writer w = new StringWriter();
+ try
+ {
+ while (true)
+ {
+ int amt = r.read(buffer);
+ if (amt == -1)
+ break;
+ w.write(buffer,0,amt);
+ }
+ }
+ finally
+ {
+ w.flush();
+ }
+ return w.toString();
+ }
+ finally
+ {
+ is.close();
+ }
}
+ return "";
}
protected String checkJson(String jsonQuery) throws ManifoldCFException
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1419176&r1=1419175&r2=1419176&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java Sun Dec 9 22:54:32 2012
@@ -26,8 +26,11 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.impl.conn.PoolingClientConnectionManager;
+import org.apache.http.client.HttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
+
import org.apache.commons.io.FilenameUtils;
import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
import org.apache.manifoldcf.agents.interfaces.IOutputNotifyActivity;
@@ -86,7 +89,7 @@ public class ElasticSearchConnector exte
private static final String VIEW_SPEC_FORWARD = "viewSpecification.html";
- private MultiThreadedHttpConnectionManager connectionManager = null;
+ private ClientConnectionManager connectionManager = null;
private HttpClient client = null;
public ElasticSearchConnector()
@@ -97,9 +100,11 @@ public class ElasticSearchConnector exte
public void connect(ConfigParams configParams)
{
super.connect(configParams);
- connectionManager = new MultiThreadedHttpConnectionManager();
- connectionManager.getParams().setMaxTotalConnections(1);
- client = new HttpClient(connectionManager);
+ PoolingClientConnectionManager localConnectionManager = new PoolingClientConnectionManager();
+ localConnectionManager.setMaxTotal(1);
+ connectionManager = localConnectionManager;
+ DefaultHttpClient localClient = new DefaultHttpClient(connectionManager);
+ client = localClient;
}
@Override
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java?rev=1419176&r1=1419175&r2=1419176&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java Sun Dec 9 22:54:32 2012
@@ -19,8 +19,9 @@
package org.apache.manifoldcf.agents.output.elasticsearch;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.methods.DeleteMethod;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpDelete;
+
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
public class ElasticSearchDelete extends ElasticSearchConnection
@@ -33,8 +34,8 @@ public class ElasticSearchDelete extends
try
{
String idField = java.net.URLEncoder.encode(documentURI,"utf-8");
- DeleteMethod method = new DeleteMethod(config.getServerLocation());
- method.setPath("/" + config.getIndexName() + "/" + config.getIndexType()
+ HttpDelete method = new HttpDelete(config.getServerLocation() +
+ "/" + config.getIndexName() + "/" + config.getIndexType()
+ "/" + idField);
call(method);
if ("ok".equals(jsonStatus))
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java?rev=1419176&r1=1419175&r2=1419176&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java Sun Dec 9 22:54:32 2012
@@ -25,18 +25,23 @@ import java.io.OutputStream;
import java.io.PrintWriter;
import java.util.Iterator;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.methods.PutMethod;
-import org.apache.commons.httpclient.methods.RequestEntity;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.HttpEntity;
+import org.apache.http.util.EntityUtils;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.Header;
+
import org.apache.commons.io.IOUtils;
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
import org.apache.manifoldcf.core.common.Base64;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.crawler.system.Logging;
public class ElasticSearchIndex extends ElasticSearchConnection
{
- private class IndexRequestEntity implements RequestEntity
+ private class IndexRequestEntity implements HttpEntity
{
private RepositoryDocument document;
@@ -49,23 +54,36 @@ public class ElasticSearchIndex extends
this.inputStream = inputStream;
}
- public long getContentLength()
- {
- return -1;
+ @Override
+ public boolean isChunked() {
+ return false;
}
-
- public String getContentType()
- {
- return "application/x-www-form-urlencoded";
+
+ @Override
+ public void consumeContent()
+ throws IOException {
+ EntityUtils.consume(this);
}
-
- public boolean isRepeatable()
- {
+
+ @Override
+ public boolean isRepeatable() {
return false;
}
- public void writeRequest(OutputStream out) throws IOException
- {
+ @Override
+ public boolean isStreaming() {
+ return false;
+ }
+
+ @Override
+ public InputStream getContent()
+ throws IOException, IllegalStateException {
+ return inputStream;
+ }
+
+ @Override
+ public void writeTo(OutputStream out)
+ throws IOException {
PrintWriter pw = new PrintWriter(out);
try
{
@@ -114,6 +132,23 @@ public class ElasticSearchIndex extends
IOUtils.closeQuietly(pw);
}
}
+
+ @Override
+ public long getContentLength() {
+ // Unknown (chunked) length
+ return -1L;
+ }
+
+ @Override
+ public Header getContentType() {
+ return new BasicHeader("Content-type","application/x-www-form-urlencoded");
+ }
+
+ @Override
+ public Header getContentEncoding() {
+ return null;
+ }
+
}
public ElasticSearchIndex(HttpClient client, String documentURI, RepositoryDocument document,
@@ -132,15 +167,14 @@ public class ElasticSearchIndex extends
}
StringBuffer url = getApiUrl(config.getIndexType() + "/" + idField, false);
- PutMethod put = new PutMethod(url.toString());
- RequestEntity entity = new IndexRequestEntity(document, inputStream);
- put.setRequestEntity(entity);
+ HttpPut put = new HttpPut(url.toString());
+ put.setEntity(new IndexRequestEntity(document, inputStream));
call(put);
if ("true".equals(checkJson(jsonStatus)))
return;
String error = checkJson(jsonException);
setResult(Result.ERROR, error);
- System.err.println(getResponse());
+ Logging.connectors.error(getResponse());
}
}
Modified: manifoldcf/trunk/connectors/elasticsearch/pom.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/pom.xml?rev=1419176&r1=1419175&r2=1419176&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/pom.xml (original)
+++ manifoldcf/trunk/connectors/elasticsearch/pom.xml Sun Dec 9 22:54:32 2012
@@ -91,9 +91,9 @@
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <version>${commons-httpclient.version}</version>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpcomponent.version}</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
Modified: manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerAction.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerAction.java?rev=1419176&r1=1419175&r2=1419176&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerAction.java (original)
+++ manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerAction.java Sun Dec 9 22:54:32 2012
@@ -17,7 +17,8 @@
package org.apache.manifoldcf.agents.output.opensearchserver;
-import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.http.client.methods.HttpGet;
+
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
public class OpenSearchServerAction extends OpenSearchServerConnection {
@@ -32,7 +33,7 @@ public class OpenSearchServerAction exte
StringBuffer url = getApiUrl("action");
url.append("&action=");
url.append(cmd.name());
- GetMethod method = new GetMethod(url.toString());
+ HttpGet method = new HttpGet(url.toString());
call(method);
if ("OK".equals(checkXPath(xPathStatus)))
return;
Modified: manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java?rev=1419176&r1=1419175&r2=1419176&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java (original)
+++ manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java Sun Dec 9 22:54:32 2012
@@ -20,6 +20,11 @@ package org.apache.manifoldcf.agents.out
import java.io.IOException;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
+import java.io.Writer;
+import java.io.StringWriter;
+import java.io.Reader;
+import java.io.InputStreamReader;
+import java.io.InputStream;
import java.net.URLEncoder;
import javax.xml.parsers.DocumentBuilder;
@@ -30,11 +35,32 @@ import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
-import org.apache.commons.io.IOUtils;
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.impl.conn.PoolingClientConnectionManager;
+import org.apache.http.client.HttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.impl.client.DefaultRedirectStrategy;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.Header;
+import org.apache.http.params.CoreProtocolPNames;
+import org.apache.http.params.HttpProtocolParams;
+import org.apache.http.ProtocolVersion;
+import org.apache.http.util.EntityUtils;
+import org.apache.http.message.BasicHeader;
+
+import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.client.RedirectException;
+import org.apache.http.client.CircularRedirectException;
+import org.apache.http.NoHttpResponseException;
+import org.apache.http.HttpException;
+
+
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
import org.w3c.dom.Document;
import org.xml.sax.InputSource;
@@ -58,7 +84,7 @@ public class OpenSearchServerConnection
private Document xmlResponse;
- private MultiThreadedHttpConnectionManager connectionManager;
+ private ClientConnectionManager connectionManager;
private HttpClient httpClient;
@@ -81,15 +107,11 @@ public class OpenSearchServerConnection
indexName = config.getIndexName();
userName = config.getUserName();
apiKey = config.getApiKey();
- connectionManager = new MultiThreadedHttpConnectionManager();
- // I don't know where CommonsHTTPClientPropertiesFactory.create() gets its parameters, but it was too small
- // by default. Since we control
- // the pool sizes at a higher level, these should be pretty much wide open at this level
- //cm.getParams().setDefaultMaxConnectionsPerHost(clientProperties.getMaximumConnectionsPerHost());
- //cm.getParams().setMaxTotalConnections(clientProperties.getMaximumTotalConnections());
- connectionManager.getParams().setDefaultMaxConnectionsPerHost(1000);
- connectionManager.getParams().setMaxTotalConnections(1000);
- httpClient = new HttpClient(connectionManager);
+ PoolingClientConnectionManager localConnectionManager = new PoolingClientConnectionManager();
+ localConnectionManager.setMaxTotal(1);
+ connectionManager = localConnectionManager;
+ DefaultHttpClient localHttpClient = new DefaultHttpClient(connectionManager);
+ httpClient = localHttpClient;
}
protected final String urlEncode(String t) throws ManifoldCFException {
@@ -118,13 +140,12 @@ public class OpenSearchServerConnection
return url;
}
- protected void call(HttpMethod method) throws ManifoldCFException {
- HttpClient hc = httpClient;
+ protected void call(HttpRequestBase method) throws ManifoldCFException {
try {
- hc.executeMethod(method);
- if (!checkResultCode(method.getStatusCode()))
+ HttpResponse resp = httpClient.execute(method);
+ if (!checkResultCode(resp.getStatusLine().getStatusCode()))
throw new ManifoldCFException(getResultDescription());
- response = IOUtils.toString(method.getResponseBodyAsStream());
+ response = getResponseBodyAsString(resp.getEntity());
} catch (HttpException e) {
setResult(Result.ERROR, e.getMessage());
throw new ManifoldCFException(e);
@@ -133,8 +154,45 @@ public class OpenSearchServerConnection
throw new ManifoldCFException(e);
} finally {
if (method != null)
- method.releaseConnection();
+ method.abort();
+ }
+ }
+
+ private static String getResponseBodyAsString(HttpEntity entity)
+ throws IOException, HttpException {
+ InputStream is = entity.getContent();
+ if (is != null)
+ {
+ try
+ {
+ String charSet = EntityUtils.getContentCharSet(entity);
+ if (charSet == null)
+ charSet = "utf-8";
+ char[] buffer = new char[65536];
+ Reader r = new InputStreamReader(is,charSet);
+ Writer w = new StringWriter();
+ try
+ {
+ while (true)
+ {
+ int amt = r.read(buffer);
+ if (amt == -1)
+ break;
+ w.write(buffer,0,amt);
+ }
+ }
+ finally
+ {
+ w.flush();
+ }
+ return w.toString();
+ }
+ finally
+ {
+ is.close();
+ }
}
+ return "";
}
private void readXmlResponse() throws ManifoldCFException {
@@ -155,8 +213,9 @@ public class OpenSearchServerConnection
} catch (IOException e) {
throw new ManifoldCFException(e);
} finally {
- if (sw != null)
- IOUtils.closeQuietly(sw);
+ if (sw != null) {
+ sw.close();
+ }
}
}
Modified: manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerDelete.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerDelete.java?rev=1419176&r1=1419175&r2=1419176&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerDelete.java (original)
+++ manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerDelete.java Sun Dec 9 22:54:32 2012
@@ -17,7 +17,8 @@
package org.apache.manifoldcf.agents.output.opensearchserver;
-import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.http.client.methods.HttpGet;
+
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
public class OpenSearchServerDelete extends OpenSearchServerConnection {
@@ -28,7 +29,7 @@ public class OpenSearchServerDelete exte
StringBuffer url = getApiUrl("delete");
url.append("&uniq=");
url.append(urlEncode(documentURI));
- GetMethod method = new GetMethod(url.toString());
+ HttpGet method = new HttpGet(url.toString());
call(method);
if ("OK".equals(checkXPath(xPathStatus)))
return;
Modified: manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerIndex.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerIndex.java?rev=1419176&r1=1419175&r2=1419176&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerIndex.java (original)
+++ manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerIndex.java Sun Dec 9 22:54:32 2012
@@ -22,8 +22,12 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
-import org.apache.commons.httpclient.methods.PutMethod;
-import org.apache.commons.httpclient.methods.RequestEntity;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.HttpEntity;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.Header;
+import org.apache.http.util.EntityUtils;
+
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.apache.manifoldcf.core.common.Base64;
@@ -31,7 +35,7 @@ import org.apache.manifoldcf.core.interf
public class OpenSearchServerIndex extends OpenSearchServerConnection {
- private class IndexRequestEntity implements RequestEntity {
+ private static class IndexRequestEntity implements HttpEntity {
private String documentURI;
@@ -45,19 +49,36 @@ public class OpenSearchServerIndex exten
this.fileName = FilenameUtils.getName(documentURI);
}
- public long getContentLength() {
- return -1;
+ @Override
+ public boolean isChunked() {
+ return false;
}
-
- public String getContentType() {
- return "text/xml; charset=utf-8";
+
+ @Override
+ public void consumeContent()
+ throws IOException {
+ EntityUtils.consume(this);
}
-
+
+ @Override
public boolean isRepeatable() {
return false;
}
- public void writeRequest(OutputStream out) throws IOException {
+ @Override
+ public boolean isStreaming() {
+ return false;
+ }
+
+ @Override
+ public InputStream getContent()
+ throws IOException, IllegalStateException {
+ return null;
+ }
+
+ @Override
+ public void writeTo(OutputStream out)
+ throws IOException {
PrintWriter pw = new PrintWriter(out);
try {
pw.println("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>");
@@ -78,15 +99,31 @@ public class OpenSearchServerIndex exten
IOUtils.closeQuietly(pw);
}
}
+
+ @Override
+ public long getContentLength() {
+ // Unknown (chunked) length
+ return -1L;
+ }
+
+ @Override
+ public Header getContentType() {
+ return new BasicHeader("Content-type","text/xml; charset=utf-8");
+ }
+
+ @Override
+ public Header getContentEncoding() {
+ return null;
+ }
+
}
public OpenSearchServerIndex(String documentURI, InputStream inputStream,
OpenSearchServerConfig config) throws ManifoldCFException {
super(config);
StringBuffer url = getApiUrl("update");
- PutMethod put = new PutMethod(url.toString());
- RequestEntity entity = new IndexRequestEntity(documentURI, inputStream);
- put.setRequestEntity(entity);
+ HttpPut put = new HttpPut(url.toString());
+ put.setEntity(new IndexRequestEntity(documentURI, inputStream));
call(put);
if ("OK".equals(checkXPath(xPathStatus)))
return;
Modified: manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerSchema.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerSchema.java?rev=1419176&r1=1419175&r2=1419176&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerSchema.java (original)
+++ manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerSchema.java Sun Dec 9 22:54:32 2012
@@ -17,7 +17,8 @@
package org.apache.manifoldcf.agents.output.opensearchserver;
-import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.http.client.methods.HttpGet;
+
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
public class OpenSearchServerSchema extends OpenSearchServerConnection {
@@ -28,7 +29,7 @@ public class OpenSearchServerSchema exte
String indexName = config.getIndexName();
StringBuffer url = getApiUrl("schema");
url.append("&cmd=indexList");
- GetMethod method = new GetMethod(url.toString());
+ HttpGet method = new HttpGet(url.toString());
String xpath = "count(/response/index[@name='" + indexName + "'])";
call(method);
if ("1".equals(checkXPath(xpath)))
Modified: manifoldcf/trunk/connectors/opensearchserver/pom.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/pom.xml?rev=1419176&r1=1419175&r2=1419176&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/opensearchserver/pom.xml (original)
+++ manifoldcf/trunk/connectors/opensearchserver/pom.xml Sun Dec 9 22:54:32 2012
@@ -91,9 +91,9 @@
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <version>${commons-httpclient.version}</version>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpcomponent.version}</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>