You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2012/12/09 22:48:45 UTC
svn commit: r1419169 - in
/manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch: ./
connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/
Author: kwright
Date: Sun Dec 9 21:48:44 2012
New Revision: 1419169
URL: http://svn.apache.org/viewvc?rev=1419169&view=rev
Log:
Convert to httpcomponents
Modified:
manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/pom.xml
Modified: manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java?rev=1419169&r1=1419168&r2=1419169&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java (original)
+++ manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java Sun Dec 9 21:48:44 2012
@@ -19,8 +19,9 @@
package org.apache.manifoldcf.agents.output.elasticsearch;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.HttpClient;
+
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
public class ElasticSearchAction extends ElasticSearchConnection
@@ -36,7 +37,7 @@ public class ElasticSearchAction extends
{
super(config, client);
StringBuffer url = getApiUrl(cmd.toString(), checkConnection);
- GetMethod method = new GetMethod(url.toString());
+ HttpGet method = new HttpGet(url.toString());
call(method);
if ("true".equals(checkJson(jsonStatus)))
return;
Modified: manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java?rev=1419169&r1=1419168&r2=1419169&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java (original)
+++ manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java Sun Dec 9 21:48:44 2012
@@ -21,11 +21,38 @@ package org.apache.manifoldcf.agents.out
import java.io.IOException;
import java.io.UnsupportedEncodingException;
+import java.io.Writer;
+import java.io.StringWriter;
+import java.io.Reader;
+import java.io.InputStreamReader;
+import java.io.InputStream;
import java.net.URLEncoder;
-import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.HttpClient;
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.impl.conn.PoolingClientConnectionManager;
+import org.apache.http.client.HttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.impl.client.DefaultRedirectStrategy;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.Header;
+import org.apache.http.params.CoreProtocolPNames;
+import org.apache.http.params.HttpProtocolParams;
+import org.apache.http.ProtocolVersion;
+import org.apache.http.util.EntityUtils;
+import org.apache.http.message.BasicHeader;
+
+import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.client.RedirectException;
+import org.apache.http.client.CircularRedirectException;
+import org.apache.http.NoHttpResponseException;
+import org.apache.http.HttpException;
+
import org.apache.commons.io.IOUtils;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
@@ -87,15 +114,14 @@ public class ElasticSearchConnection
return url;
}
- protected void call(HttpMethod method) throws ManifoldCFException
+ protected void call(HttpRequestBase method) throws ManifoldCFException
{
- HttpClient hc = client;
try
{
- hc.executeMethod(method);
- if (!checkResultCode(method.getStatusCode()))
+ HttpResponse resp = client.execute(method);
+ if (!checkResultCode(resp.getStatusLine().getStatusCode()))
throw new ManifoldCFException(getResultDescription());
- response = IOUtils.toString(method.getResponseBodyAsStream());
+ response = getResponseBodyAsString(resp.getEntity());
} catch (HttpException e)
{
setResult(Result.ERROR, e.getMessage());
@@ -107,8 +133,45 @@ public class ElasticSearchConnection
} finally
{
if (method != null)
- method.releaseConnection();
+ method.abort();
+ }
+ }
+
+ private static String getResponseBodyAsString(HttpEntity entity)
+ throws IOException, HttpException {
+ InputStream is = entity.getContent();
+ if (is != null)
+ {
+ try
+ {
+ String charSet = EntityUtils.getContentCharSet(entity);
+ if (charSet == null)
+ charSet = "utf-8";
+ char[] buffer = new char[65536];
+ Reader r = new InputStreamReader(is,charSet);
+ Writer w = new StringWriter();
+ try
+ {
+ while (true)
+ {
+ int amt = r.read(buffer);
+ if (amt == -1)
+ break;
+ w.write(buffer,0,amt);
+ }
+ }
+ finally
+ {
+ w.flush();
+ }
+ return w.toString();
+ }
+ finally
+ {
+ is.close();
+ }
}
+ return "";
}
protected String checkJson(String jsonQuery) throws ManifoldCFException
Modified: manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1419169&r1=1419168&r2=1419169&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java Sun Dec 9 21:48:44 2012
@@ -26,8 +26,11 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.impl.conn.PoolingClientConnectionManager;
+import org.apache.http.client.HttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
+
import org.apache.commons.io.FilenameUtils;
import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
import org.apache.manifoldcf.agents.interfaces.IOutputNotifyActivity;
@@ -86,7 +89,7 @@ public class ElasticSearchConnector exte
private static final String VIEW_SPEC_FORWARD = "viewSpecification.html";
- private MultiThreadedHttpConnectionManager connectionManager = null;
+ private ClientConnectionManager connectionManager = null;
private HttpClient client = null;
public ElasticSearchConnector()
@@ -97,9 +100,11 @@ public class ElasticSearchConnector exte
public void connect(ConfigParams configParams)
{
super.connect(configParams);
- connectionManager = new MultiThreadedHttpConnectionManager();
- connectionManager.getParams().setMaxTotalConnections(1);
- client = new HttpClient(connectionManager);
+ PoolingClientConnectionManager localConnectionManager = new PoolingClientConnectionManager();
+ localConnectionManager.setMaxTotal(1);
+ connectionManager = localConnectionManager;
+ DefaultHttpClient localClient = new DefaultHttpClient(connectionManager);
+ client = localClient;
}
@Override
Modified: manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java?rev=1419169&r1=1419168&r2=1419169&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java (original)
+++ manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java Sun Dec 9 21:48:44 2012
@@ -19,8 +19,9 @@
package org.apache.manifoldcf.agents.output.elasticsearch;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.methods.DeleteMethod;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpDelete;
+
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
public class ElasticSearchDelete extends ElasticSearchConnection
@@ -33,8 +34,8 @@ public class ElasticSearchDelete extends
try
{
String idField = java.net.URLEncoder.encode(documentURI,"utf-8");
- DeleteMethod method = new DeleteMethod(config.getServerLocation());
- method.setPath("/" + config.getIndexName() + "/" + config.getIndexType()
+ HttpDelete method = new HttpDelete(config.getServerLocation() +
+ "/" + config.getIndexName() + "/" + config.getIndexType()
+ "/" + idField);
call(method);
if ("ok".equals(jsonStatus))
Modified: manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java?rev=1419169&r1=1419168&r2=1419169&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java (original)
+++ manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java Sun Dec 9 21:48:44 2012
@@ -25,18 +25,23 @@ import java.io.OutputStream;
import java.io.PrintWriter;
import java.util.Iterator;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.methods.PutMethod;
-import org.apache.commons.httpclient.methods.RequestEntity;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.HttpEntity;
+import org.apache.http.util.EntityUtils;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.Header;
+
import org.apache.commons.io.IOUtils;
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
import org.apache.manifoldcf.core.common.Base64;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.crawler.system.Logging;
public class ElasticSearchIndex extends ElasticSearchConnection
{
- private class IndexRequestEntity implements RequestEntity
+ private class IndexRequestEntity implements HttpEntity
{
private RepositoryDocument document;
@@ -49,23 +54,36 @@ public class ElasticSearchIndex extends
this.inputStream = inputStream;
}
- public long getContentLength()
- {
- return -1;
+ @Override
+ public boolean isChunked() {
+ return false;
}
-
- public String getContentType()
- {
- return "application/x-www-form-urlencoded";
+
+ @Override
+ public void consumeContent()
+ throws IOException {
+ EntityUtils.consume(this);
}
-
- public boolean isRepeatable()
- {
+
+ @Override
+ public boolean isRepeatable() {
return false;
}
- public void writeRequest(OutputStream out) throws IOException
- {
+ @Override
+ public boolean isStreaming() {
+ return false;
+ }
+
+ @Override
+ public InputStream getContent()
+ throws IOException, IllegalStateException {
+ return inputStream;
+ }
+
+ @Override
+ public void writeTo(OutputStream out)
+ throws IOException {
PrintWriter pw = new PrintWriter(out);
try
{
@@ -114,6 +132,23 @@ public class ElasticSearchIndex extends
IOUtils.closeQuietly(pw);
}
}
+
+ @Override
+ public long getContentLength() {
+ // Unknown (chunked) length
+ return -1L;
+ }
+
+ @Override
+ public Header getContentType() {
+ return new BasicHeader("Content-type","application/x-www-form-urlencoded");
+ }
+
+ @Override
+ public Header getContentEncoding() {
+ return null;
+ }
+
}
public ElasticSearchIndex(HttpClient client, String documentURI, RepositoryDocument document,
@@ -132,15 +167,14 @@ public class ElasticSearchIndex extends
}
StringBuffer url = getApiUrl(config.getIndexType() + "/" + idField, false);
- PutMethod put = new PutMethod(url.toString());
- RequestEntity entity = new IndexRequestEntity(document, inputStream);
- put.setRequestEntity(entity);
+ HttpPut put = new HttpPut(url.toString());
+ put.setEntity(new IndexRequestEntity(document, inputStream));
call(put);
if ("true".equals(checkJson(jsonStatus)))
return;
String error = checkJson(jsonException);
setResult(Result.ERROR, error);
- System.err.println(getResponse());
+ Logging.connectors.error(getResponse());
}
}
Modified: manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/pom.xml
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/pom.xml?rev=1419169&r1=1419168&r2=1419169&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/pom.xml (original)
+++ manifoldcf/branches/CONNECTORS-581/connectors/elasticsearch/pom.xml Sun Dec 9 21:48:44 2012
@@ -91,9 +91,9 @@
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <version>${commons-httpclient.version}</version>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpcomponent.version}</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>