You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by mo...@apache.org on 2014/10/21 13:43:17 UTC
svn commit: r1633348 - in /manifoldcf/trunk:
connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/
framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/
Author: molgun
Date: Tue Oct 21 11:43:17 2014
New Revision: 1633348
URL: http://svn.apache.org/r1633348
Log:
CONNECTORS-1077: For ElasticSearch
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java?rev=1633348&r1=1633347&r2=1633348&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java Tue Oct 21 11:43:17 2014
@@ -28,6 +28,7 @@ import java.io.InputStream;
import java.io.InterruptedIOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
+import java.util.Locale;
import org.apache.http.client.HttpClient;
import org.apache.http.impl.client.DefaultRedirectStrategy;
@@ -51,6 +52,7 @@ import org.apache.http.NoHttpResponseExc
import org.apache.http.HttpException;
import org.apache.http.ParseException;
+import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
import org.apache.manifoldcf.core.util.URLEncoder;
@@ -71,6 +73,8 @@ public class ElasticSearchConnection
private String response;
+ private String resultCode;
+
protected final static String jsonException = "\"error\"";
public enum Result
@@ -225,22 +229,22 @@ public class ElasticSearchConnection
{
if (code == 200 || code == 201)
{
- setResult(Result.OK, null);
+ setResult("OK",Result.OK, null);
return true;
}
else if (code == 404)
{
- setResult(Result.ERROR, "Page not found: " + response);
+ setResult(IOutputHistoryActivity.HTTP_ERROR,Result.ERROR, "Page not found: " + response);
throw new ManifoldCFException("Server/page not found");
}
else if (code >= 400 && code < 500)
{
- setResult(Result.ERROR, "HTTP code = "+code+", Response = "+response);
+ setResult(IOutputHistoryActivity.HTTP_ERROR,Result.ERROR, "HTTP code = "+code+", Response = "+response);
return false;
}
else if (code >= 500 && code < 600)
{
- setResult(Result.ERROR, "Server exception: "+response);
+ setResult(IOutputHistoryActivity.HTTP_ERROR,Result.ERROR, "Server exception: "+response);
long currentTime = System.currentTimeMillis();
throw new ServiceInterruption("Server exception: "+response,
new ManifoldCFException(response),
@@ -249,13 +253,13 @@ public class ElasticSearchConnection
-1,
false);
}
- setResult(Result.UNKNOWN, "HTTP code = "+code+", Response = "+response);
+ setResult(IOutputHistoryActivity.HTTP_ERROR,Result.UNKNOWN, "HTTP code = "+code+", Response = "+response);
throw new ManifoldCFException("Unexpected HTTP result code: "+code+": "+response);
}
protected void handleHttpException(HttpException e)
throws ManifoldCFException, ServiceInterruption {
- setResult(Result.ERROR, e.getMessage());
+ setResult(e.getClass().getSimpleName().toUpperCase(Locale.ROOT),Result.ERROR, e.getMessage());
throw new ManifoldCFException(e);
}
@@ -263,7 +267,7 @@ public class ElasticSearchConnection
throws ManifoldCFException, ServiceInterruption {
if (e instanceof java.io.InterruptedIOException && !(e instanceof java.net.SocketTimeoutException))
throw new ManifoldCFException(e.getMessage(),ManifoldCFException.INTERRUPTED);
- setResult(Result.ERROR, e.getMessage());
+ setResult(e.getClass().getSimpleName().toUpperCase(Locale.ROOT),Result.ERROR, e.getMessage());
long currentTime = System.currentTimeMillis();
// All IO exceptions are treated as service interruptions, retried for an hour
throw new ServiceInterruption("IO exception: "+e.getMessage(),e,
@@ -334,13 +338,14 @@ public class ElasticSearchConnection
return result;
}
- protected void setResult(Result res, String desc)
+ protected void setResult(String resultCode, Result res, String desc)
{
if (res != null)
result = res;
if (desc != null)
if (desc.length() > 0)
resultDescription = desc;
+ setResultCode(resultCode);
}
public String getResultDescription()
@@ -363,4 +368,8 @@ public class ElasticSearchConnection
{
return callUrlSnippet;
}
+
+ public String getResultCode(){ return resultCode; }
+
+ public void setResultCode(String resultCode){ this.resultCode = resultCode; }
}
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1633348&r1=1633347&r2=1633348&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java Tue Oct 21 11:43:17 2014
@@ -365,6 +365,7 @@ public class ElasticSearchConnector exte
else
{
// Don't know how to deal with it
+ activities.recordActivity(null,ELASTICSEARCH_INDEXATION_ACTIVITY,document.getBinaryLength(),documentURI,activities.UNKNOWN_SECURITY,"Rejected document that has security info which ElasticSearch does not recognize: '"+ securityType + "'");
return DOCUMENTSTATUS_REJECTED;
}
}
@@ -381,7 +382,7 @@ public class ElasticSearchConnector exte
finally
{
activities.recordActivity(startTime, ELASTICSEARCH_INDEXATION_ACTIVITY,
- document.getBinaryLength(), documentURI, oi.getResult().name(), oi.getResultDescription());
+ document.getBinaryLength(), documentURI, oi.getResultCode(), oi.getResultDescription());
}
}
@@ -400,7 +401,7 @@ public class ElasticSearchConnector exte
finally
{
activities.recordActivity(startTime, ELASTICSEARCH_DELETION_ACTIVITY, null,
- documentURI, od.getResult().name(), od.getResultDescription());
+ documentURI, od.getResultCode(), od.getResultDescription());
}
}
@@ -437,7 +438,7 @@ public class ElasticSearchConnector exte
finally
{
activities.recordActivity(startTime, ELASTICSEARCH_OPTIMIZE_ACTIVITY, null,
- oo.getCallUrlSnippet(), oo.getResult().name(),
+ oo.getCallUrlSnippet(), oo.getResultCode(),
oo.getResultDescription());
}
}
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java?rev=1633348&r1=1633347&r2=1633348&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java Tue Oct 21 11:43:17 2014
@@ -22,6 +22,7 @@ package org.apache.manifoldcf.agents.out
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpDelete;
+import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
import org.apache.manifoldcf.core.util.URLEncoder;
@@ -48,7 +49,7 @@ public class ElasticSearchDelete extends
return;
// We thought we needed to delete, but ElasticSearch disagreed.
// Log the result as an error, but proceed anyway.
- setResult(Result.ERROR, error);
+ setResult(IOutputHistoryActivity.JSON_ERROR,Result.ERROR, error);
Logging.connectors.warn("ES: Delete failed: "+getResponse());
}
}
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java?rev=1633348&r1=1633347&r2=1633348&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java Tue Oct 21 11:43:17 2014
@@ -35,6 +35,7 @@ import org.apache.http.message.BasicHead
import org.apache.http.Header;
import org.apache.commons.io.IOUtils;
+import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
import org.apache.manifoldcf.agents.output.elasticsearch.ElasticSearchConnection.Result;
@@ -280,7 +281,7 @@ public class ElasticSearchIndex extends
String error = checkJson(jsonException);
if (getResult() == Result.OK && error == null)
return true;
- setResult(Result.ERROR, error);
+ setResult(IOutputHistoryActivity.JSON_ERROR,Result.ERROR, error);
Logging.connectors.warn("ES: Index failed: "+getResponse());
return true;
}
Modified: manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java?rev=1633348&r1=1633347&r2=1633348&view=diff
==============================================================================
--- manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java (original)
+++ manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java Tue Oct 21 11:43:17 2014
@@ -35,7 +35,9 @@ public interface IOutputHistoryActivity
public static final String EXCLUDED_LENGTH = "EXCLUDEDLENGTH";
public static final String EXCLUDED_MIMETYPE = "EXCLUDEDMIMETYPE";
public static final String EXCLUDED_DATE = "EXCLUDEDDATE";
-
+ public static final String UNKNOWN_SECURITY = "UNKNOWNSECURITY";
+ public static final String HTTP_ERROR = "HTTPERROR";
+ public static final String JSON_ERROR = "JSONERROR";
/** Record time-stamped information about the activity of the output connector.
*@param startTime is either null or the time since the start of epoch in milliseconds (Jan 1, 1970). Every
* activity has an associated time; the startTime field records when the activity began. A null value