You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2012/02/27 23:48:03 UTC
svn commit: r1294377 -
/incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/
Author: kwright
Date: Mon Feb 27 22:48:02 2012
New Revision: 1294377
URL: http://svn.apache.org/viewvc?rev=1294377&view=rev
Log:
Clean up a couple of things; for one thing disable internal caching until we sort out the problems.
Modified:
incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
Modified: incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java?rev=1294377&r1=1294376&r2=1294377&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java (original)
+++ incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java Mon Feb 27 22:48:02 2012
@@ -18,6 +18,7 @@
package org.apache.manifoldcf.agents.output.elasticsearch;
import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.HttpClient;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
public class ElasticSearchAction extends ElasticSearchConnection
@@ -28,13 +29,12 @@ public class ElasticSearchAction extends
_optimize, _refresh, _status;
}
- public ElasticSearchAction(CommandEnum cmd, ElasticSearchConfig config)
+ public ElasticSearchAction(HttpClient client, CommandEnum cmd, ElasticSearchConfig config)
throws ManifoldCFException
{
- super(config);
+ super(config, client);
StringBuffer url = getApiUrl(cmd.toString());
GetMethod method = new GetMethod(url.toString());
- System.out.println("URL is '"+url.toString()+"'");
call(method);
if ("true".equals(checkJson(jsonStatus)))
return;
Modified: incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java?rev=1294377&r1=1294376&r2=1294377&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java (original)
+++ incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java Mon Feb 27 22:48:02 2012
@@ -21,18 +21,16 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
-import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
+import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.io.IOUtils;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
public class ElasticSearchConnection
{
-
- private MultiThreadedHttpConnectionManager connectionManager;
-
+ private HttpClient client;
+
private String serverLocation;
private String indexName;
@@ -57,8 +55,9 @@ public class ElasticSearchConnection
private Result result;
- protected ElasticSearchConnection(ElasticSearchConfig config)
+ protected ElasticSearchConnection(ElasticSearchConfig config, HttpClient client)
{
+ this.client = client;
result = Result.UNKNOWN;
response = null;
resultDescription = "";
@@ -67,8 +66,6 @@ public class ElasticSearchConnection
indexName = config.getIndexName();
userName = config.getUserName();
apiKey = config.getApiKey();
- connectionManager = new MultiThreadedHttpConnectionManager();
- connectionManager.getParams().setMaxTotalConnections(1);
}
protected final String urlEncode(String t) throws ManifoldCFException
@@ -103,7 +100,7 @@ public class ElasticSearchConnection
protected void call(HttpMethod method) throws ManifoldCFException
{
- HttpClient hc = new HttpClient(connectionManager);
+ HttpClient hc = client;
try
{
hc.executeMethod(method);
Modified: incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1294377&r1=1294376&r2=1294377&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java (original)
+++ incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java Mon Feb 27 22:48:02 2012
@@ -42,6 +42,8 @@ import org.apache.manifoldcf.core.interf
import org.apache.manifoldcf.core.interfaces.IThreadContext;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
import org.apache.manifoldcf.core.interfaces.SpecificationNode;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.json.JSONException;
import org.json.JSONObject;
@@ -53,9 +55,9 @@ import org.json.JSONObject;
public class ElasticSearchConnector extends BaseOutputConnector
{
- private final static String ELASTICSEARCH_INDEXATION_ACTIVITY = "Optimize";
+ private final static String ELASTICSEARCH_INDEXATION_ACTIVITY = "Indexation";
private final static String ELASTICSEARCH_DELETION_ACTIVITY = "Deletion";
- private final static String ELASTICSEARCH_OPTIMIZE_ACTIVITY = "Indexation";
+ private final static String ELASTICSEARCH_OPTIMIZE_ACTIVITY = "Optimize";
private final static String[] ELASTICSEARCH_ACTIVITIES =
{ ELASTICSEARCH_INDEXATION_ACTIVITY, ELASTICSEARCH_DELETION_ACTIVITY,
@@ -89,16 +91,44 @@ public class ElasticSearchConnector exte
/** Forward to the javascript to check the specification parameters for the job */
private static final String EDIT_SPEC_HEADER_FORWARD = "editSpecification.js";
- private String specsCacheOutpuDescription;
- private ElasticSearchSpecs specsCache;
+ private MultiThreadedHttpConnectionManager connectionManager = null;
+ private HttpClient client = null;
+ private String specsCacheOutpuDescription = null;
+ private ElasticSearchSpecs specsCache = null;
public ElasticSearchConnector()
{
- specsCacheOutpuDescription = null;
- specsCache = null;
}
@Override
+ public void connect(ConfigParams configParams)
+ {
+ super.connect(configParams);
+ connectionManager = new MultiThreadedHttpConnectionManager();
+ connectionManager.getParams().setMaxTotalConnections(1);
+ client = new HttpClient(connectionManager);
+ }
+
+ @Override
+ public void disconnect()
+ throws ManifoldCFException
+ {
+ super.disconnect();
+ connectionManager.shutdown();
+ connectionManager = null;
+ client = null;
+ }
+
+ @Override
+ public void poll()
+ throws ManifoldCFException
+ {
+ super.poll();
+ // Free idle connections in the pool.
+ // MHL
+ }
+
+ @Override
public String[] getActivitiesList()
{
return ELASTICSEARCH_ACTIVITIES;
@@ -213,10 +243,7 @@ public class ElasticSearchConnector exte
{
if (configParams == null)
configParams = getConfiguration();
- synchronized (this)
- {
- return new ElasticSearchConfig(configParams);
- }
+ return new ElasticSearchConfig(configParams);
}
final private ElasticSearchSpecs getSpecParameters(OutputSpecification os)
@@ -230,6 +257,7 @@ public class ElasticSearchConnector exte
{
try
{
+ /*
synchronized (this)
{
if (!outputDescription.equals(specsCacheOutpuDescription))
@@ -238,6 +266,8 @@ public class ElasticSearchConnector exte
specsCache = new ElasticSearchSpecs(new JSONObject(outputDescription));
return specsCache;
}
+ */
+ return new ElasticSearchSpecs(new JSONObject(outputDescription));
} catch (JSONException e)
{
throw new ManifoldCFException(e);
@@ -305,41 +335,6 @@ public class ElasticSearchConnector exte
return null;
}
- private static Map<String, Integer> ossInstances = null;
-
- private synchronized final Integer addInstance(ElasticSearchConfig config)
- {
- if (ossInstances == null)
- ossInstances = new TreeMap<String, Integer>();
- synchronized (ossInstances)
- {
- String uii = config.getUniqueIndexIdentifier();
- Integer count = ossInstances.get(uii);
- if (count == null)
- {
- count = new Integer(1);
- ossInstances.put(uii, count);
- } else
- count++;
- return count;
- }
- }
-
- private synchronized final void removeInstance(ElasticSearchConfig config)
- {
- if (ossInstances == null)
- return;
- synchronized (ossInstances)
- {
- String uii = config.getUniqueIndexIdentifier();
- Integer count = ossInstances.get(uii);
- if (count == null)
- return;
- if (--count == 0)
- ossInstances.remove(uii);
- }
- }
-
@Override
public int addOrReplaceDocument(String documentURI, String outputDescription,
RepositoryDocument document, String authorityNameString,
@@ -347,26 +342,16 @@ public class ElasticSearchConnector exte
ServiceInterruption
{
ElasticSearchConfig config = getConfigParameters(null);
- Integer count = addInstance(config);
- synchronized (count)
- {
- InputStream inputStream = document.getBinaryStream();
- try
- {
- long startTime = System.currentTimeMillis();
- ElasticSearchIndex oi = new ElasticSearchIndex(documentURI,
- inputStream, config);
- activities.recordActivity(startTime, ELASTICSEARCH_INDEXATION_ACTIVITY,
- document.getBinaryLength(), documentURI, oi.getResult().name(),
- oi.getResultDescription());
- if (oi.getResult() != Result.OK)
- return DOCUMENTSTATUS_REJECTED;
- } finally
- {
- removeInstance(config);
- }
- return DOCUMENTSTATUS_ACCEPTED;
- }
+ InputStream inputStream = document.getBinaryStream();
+ long startTime = System.currentTimeMillis();
+ ElasticSearchIndex oi = new ElasticSearchIndex(client, documentURI,
+ inputStream, config);
+ activities.recordActivity(startTime, ELASTICSEARCH_INDEXATION_ACTIVITY,
+ document.getBinaryLength(), documentURI, oi.getResult().name(),
+ oi.getResultDescription());
+ if (oi.getResult() != Result.OK)
+ return DOCUMENTSTATUS_REJECTED;
+ return DOCUMENTSTATUS_ACCEPTED;
}
@Override
@@ -375,7 +360,7 @@ public class ElasticSearchConnector exte
ServiceInterruption
{
long startTime = System.currentTimeMillis();
- ElasticSearchDelete od = new ElasticSearchDelete(documentURI,
+ ElasticSearchDelete od = new ElasticSearchDelete(client, documentURI,
getConfigParameters(null));
activities.recordActivity(startTime, ELASTICSEARCH_DELETION_ACTIVITY, null,
documentURI, od.getResult().name(), od.getResultDescription());
@@ -384,20 +369,12 @@ public class ElasticSearchConnector exte
@Override
public String check() throws ManifoldCFException
{
- try
- {
- ElasticSearchAction oss = new ElasticSearchAction(CommandEnum._status,
- getConfigParameters(null));
- String resultName = oss.getResult().name();
- if (resultName.equals("OK"))
- return super.check();
- return resultName + " " + oss.getResultDescription();
- }
- catch (ManifoldCFException e)
- {
- e.printStackTrace();
- throw e;
- }
+ ElasticSearchAction oss = new ElasticSearchAction(client, CommandEnum._status,
+ getConfigParameters(null));
+ String resultName = oss.getResult().name();
+ if (resultName.equals("OK"))
+ return super.check();
+ return resultName + " " + oss.getResultDescription();
}
@Override
@@ -405,7 +382,7 @@ public class ElasticSearchConnector exte
throws ManifoldCFException, ServiceInterruption
{
long startTime = System.currentTimeMillis();
- ElasticSearchAction oo = new ElasticSearchAction(CommandEnum._optimize,
+ ElasticSearchAction oo = new ElasticSearchAction(client, CommandEnum._optimize,
getConfigParameters(null));
activities.recordActivity(startTime, ELASTICSEARCH_OPTIMIZE_ACTIVITY, null,
oo.getCallUrlSnippet(), oo.getResult().name(),
Modified: incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java?rev=1294377&r1=1294376&r2=1294377&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java (original)
+++ incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java Mon Feb 27 22:48:02 2012
@@ -18,22 +18,26 @@
package org.apache.manifoldcf.agents.output.elasticsearch;
import org.apache.commons.httpclient.methods.DeleteMethod;
+import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.io.FilenameUtils;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
public class ElasticSearchDelete extends ElasticSearchConnection
{
- public ElasticSearchDelete(String documentURI, ElasticSearchConfig config)
+ public ElasticSearchDelete(HttpClient client, String documentURI, ElasticSearchConfig config)
throws ManifoldCFException
{
- super(config);
+ super(config, client);
String fileName = FilenameUtils.getName(documentURI);
DeleteMethod method = new DeleteMethod(config.getServerLocation());
method.setPath("/" + config.getIndexName() + "/" + config.getIndexType()
+ "/" + fileName);
+ System.out.println("Deleting '"+documentURI+"'...");
call(method);
- if ("true".equals(checkJson(jsonStatus)))
+ System.out.println("... completed");
+ System.out.println(jsonStatus);
+ if ("ok".equals(jsonStatus))
return;
setResult(Result.ERROR, checkJson(jsonException));
}
Modified: incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java?rev=1294377&r1=1294376&r2=1294377&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java (original)
+++ incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java Mon Feb 27 22:48:02 2012
@@ -23,6 +23,7 @@ import java.io.OutputStream;
import java.io.PrintWriter;
import org.apache.commons.httpclient.methods.PutMethod;
+import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.methods.RequestEntity;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
@@ -88,10 +89,10 @@ public class ElasticSearchIndex extends
}
}
- public ElasticSearchIndex(String documentURI, InputStream inputStream,
+ public ElasticSearchIndex(HttpClient client, String documentURI, InputStream inputStream,
ElasticSearchConfig config) throws ManifoldCFException
{
- super(config);
+ super(config, client);
String fileName = FilenameUtils.getName(documentURI);
StringBuffer url = getApiUrl(config.getIndexType() + "/" + fileName);
PutMethod put = new PutMethod(url.toString());