You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/10/21 11:32:07 UTC
svn commit: r1633326 - in /manifoldcf/branches/dev_1x: ./
connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/
connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/
connectors/solr/co...
Author: kwright
Date: Tue Oct 21 09:32:07 2014
New Revision: 1633326
URL: http://svn.apache.org/r1633326
Log:
Pull up more CONNECTORS-1077 changes from trunk
Modified:
manifoldcf/branches/dev_1x/ (props changed)
manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk:r1633282,1633284,1633295
Modified: manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java?rev=1633326&r1=1633325&r2=1633326&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java (original)
+++ manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java Tue Oct 21 09:32:07 2014
@@ -169,6 +169,9 @@ public class FileOutputConnector extends
FileOutputSpecs specs = new FileOutputSpecs(getSpecNode(outputDescription.getSpecification()));;
StringBuffer path = new StringBuffer();
+ String errorCode = "OK";
+ String errorDesc = null;
+
try {
/*
* make file path
@@ -215,8 +218,9 @@ public class FileOutputConnector extends
}
else
{
- activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.CREATED_DIRECTORY,"Couldn't create directory. '"+newPath+"'");
- throw new ManifoldCFException("Could not create directory '"+newPath+"'. Permission issue?");
+ errorCode = activities.CREATED_DIRECTORY;
+ errorDesc = "Could not create directory '\"+newPath+\"'. Permission issue?";
+ throw new ManifoldCFException(errorDesc);
}
}
// Directory successfully created!
@@ -247,8 +251,9 @@ public class FileOutputConnector extends
continue;
}
// Probably some other error
- activities.recordActivity(null, INGEST_ACTIVITY, new Long(document.getBinaryLength()), documentURI, activities.EXCEPTION, "Unexpected exception while creating the file.'" + outputPath + "'");
- throw new ManifoldCFException("Could not create file '"+outputPath+"': "+e.getMessage(),e);
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Could not create file '" + outputPath + "': " +e.getMessage();
+ throw new ManifoldCFException(errorDesc,e);
}
}
@@ -259,8 +264,9 @@ public class FileOutputConnector extends
FileChannel channel = output.getChannel();
FileLock lock = channel.tryLock();
if (lock == null){
- activities.recordActivity(null,INGEST_ACTIVITY, new Long(document.getBinaryLength()), documentURI,activities.EXCEPTION,"Could not lock file: '" + outputPath + "'");
- throw new ServiceInterruption("Could not lock file: '"+outputPath+"'",null,1000L,-1L,10,false);
+ errorCode = ServiceInterruption.class.getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Could not lock file: '"+outputPath+"'";
+ throw new ServiceInterruption(errorDesc,null,1000L,-1L,10,false);
}
@@ -292,24 +298,29 @@ public class FileOutputConnector extends
}
}
} catch (URISyntaxException e) {
- activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()), documentURI,activities.EXCEPTION,"Failed to write document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
handleURISyntaxException(e);
return DOCUMENTSTATUS_REJECTED;
} catch (FileNotFoundException e) {
- activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()), documentURI,activities.EXCEPTION,"Failed to write document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
handleFileNotFoundException(e);
return DOCUMENTSTATUS_REJECTED;
} catch (SecurityException e) {
- activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()), documentURI,activities.EXCEPTION,"Failed to write document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
handleSecurityException(e);
return DOCUMENTSTATUS_REJECTED;
} catch (IOException e) {
- activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()), documentURI ,activities.EXCEPTION,"Failed to write document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
handleIOException(e);
return DOCUMENTSTATUS_REJECTED;
+ } finally {
+ activities.recordActivity(null, INGEST_ACTIVITY, new Long(document.getBinaryLength()), documentURI, errorCode, errorDesc);
}
- activities.recordActivity(null, INGEST_ACTIVITY, new Long(document.getBinaryLength()), documentURI, "OK", null);
return DOCUMENTSTATUS_ACCEPTED;
}
@@ -361,6 +372,9 @@ public class FileOutputConnector extends
// Establish a session
getSession();
+ String errorCode = "OK";
+ String errorDesc = null;
+
FileOutputConfig config = getConfigParameters(null);
StringBuffer path = new StringBuffer();
@@ -432,27 +446,32 @@ public class FileOutputConnector extends
catch (FileNotFoundException e)
{
// Probably some other error
- activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Couldn't delete the file due to FileNotFoundException '"+outputPath+"':" + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Couldn't delete the file due to:" + e.getMessage();
throw new ManifoldCFException("Could not zero out file '"+outputPath+"': "+e.getMessage(),e);
}
}
// Just close it, to make a zero-length grave marker.
output.close();
} catch (URISyntaxException e) {
- activities.recordActivity(null,REMOVE_ACTIVITY,null, documentURI,activities.EXCEPTION,"Failed to delete document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to delete document due to: " + e.getMessage();
handleURISyntaxException(e);
} catch (FileNotFoundException e) {
- activities.recordActivity(null,REMOVE_ACTIVITY,null, documentURI,activities.EXCEPTION,"Failed to delete document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to delete document due to: " + e.getMessage();
handleFileNotFoundException(e);
} catch (SecurityException e) {
- activities.recordActivity(null,REMOVE_ACTIVITY,null, documentURI,activities.EXCEPTION,"Failed to delete document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to delete document due to: " + e.getMessage();
handleSecurityException(e);
} catch (IOException e) {
- activities.recordActivity(null,REMOVE_ACTIVITY,null, documentURI ,activities.EXCEPTION,"Failed to delete document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to delete document due to: " + e.getMessage();
handleIOException(e);
+ } finally {
+ activities.recordActivity(null, REMOVE_ACTIVITY, null, documentURI, errorCode, errorDesc);
}
-
- activities.recordActivity(null, REMOVE_ACTIVITY, null, documentURI, "OK", null);
}
/** Obtain the name of the form check javascript method to call.
Modified: manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java?rev=1633326&r1=1633325&r2=1633326&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java (original)
+++ manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java Tue Oct 21 09:32:07 2014
@@ -290,7 +290,7 @@ public class HDFSOutputConnector extends
activities.recordActivity(startTime, INGEST_ACTIVITY, new Long(document.getBinaryLength()), documentURI, "OK", null);
return DOCUMENTSTATUS_ACCEPTED;
} catch (URISyntaxException e) {
- activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()),documentURI,activities.EXCEPTION,"Failed to write document due to: " + e.getMessage());
+ activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()),documentURI,e.getClass().getSimpleName().toUpperCase(Locale.ROOT),"Failed to write document due to: " + e.getMessage());
handleURISyntaxException(e);
return DOCUMENTSTATUS_REJECTED;
}
@@ -324,10 +324,10 @@ public class HDFSOutputConnector extends
deleteFile(path,activities,documentURI);
activities.recordActivity(startTime, REMOVE_ACTIVITY, null, documentURI, "OK", null);
} catch (JSONException e) {
- activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed to delete document due to: " + e.getMessage());
+ activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,e.getClass().getSimpleName().toUpperCase(Locale.ROOT),"Failed to delete document due to: " + e.getMessage());
handleJSONException(e);
} catch (URISyntaxException e) {
- activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed to delete document due to: " + e.getMessage());
+ activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,e.getClass().getSimpleName().toUpperCase(Locale.ROOT),"Failed to delete document due to: " + e.getMessage());
handleURISyntaxException(e);
}
}
@@ -675,23 +675,33 @@ public class HDFSOutputConnector extends
protected void createFile(Path path, InputStream input,IOutputAddActivity activities, String documentURI)
throws ManifoldCFException, ServiceInterruption {
CreateFileThread t = new CreateFileThread(getSession(), path, input);
+ String errorCode = null;
+ String errorDesc = null;
try {
t.start();
t.finishUp();
} catch (InterruptedException e) {
t.interrupt();
- activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed to write document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
} catch (java.net.SocketTimeoutException e) {
- activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed to write document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
handleIOException(e);
} catch (InterruptedIOException e) {
t.interrupt();
- activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed to write document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
handleIOException(e);
} catch (IOException e) {
- activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed to write document due to: " + e.getMessage());
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
handleIOException(e);
+ } finally {
+ if(errorCode != null & errorDesc != null){
+ activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,errorCode,errorDesc);
+ }
}
}
@@ -734,23 +744,33 @@ public class HDFSOutputConnector extends
throws ManifoldCFException, ServiceInterruption {
// Establish a session
DeleteFileThread t = new DeleteFileThread(getSession(),path);
+ String errorCode = null;
+ String errorDesc = null;
try {
t.start();
t.finishUp();
} catch (InterruptedException e) {
- t.interrupt();
- activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed to delete document due to: " + e.getMessage());
- throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+ t.interrupt();
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
+ throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
} catch (java.net.SocketTimeoutException e) {
- activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed to delete document due to: " + e.getMessage());
- handleIOException(e);
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
+ handleIOException(e);
} catch (InterruptedIOException e) {
- t.interrupt();
- activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed to delete document due to: " + e.getMessage());
- handleIOException(e);
+ t.interrupt();
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
+ handleIOException(e);
} catch (IOException e) {
- activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed to delete document due to: " + e.getMessage());
- handleIOException(e);
+ errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Failed to write document due to: " + e.getMessage();
+ handleIOException(e);
+ } finally {
+ if(errorCode != null & errorDesc != null){
+ activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,errorCode,errorDesc);
+ }
}
}
Modified: manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1633326&r1=1633325&r2=1633326&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java (original)
+++ manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java Tue Oct 21 09:32:07 2014
@@ -536,8 +536,11 @@ public class HttpPoster
Logging.ingest.debug("indexPost(): '" + documentURI + "'");
// If the document is too long, reject it.
- if (maxDocumentLength != null && document.getBinaryLength() > maxDocumentLength.longValue())
+ if (maxDocumentLength != null && document.getBinaryLength() > maxDocumentLength.longValue()){
+ activities.recordActivity(null,SolrConnector.INGEST_ACTIVITY,null,documentURI,activities.EXCLUDED_LENGTH,"Solr connector rejected document due to its big size. ('"+document.getBinaryLength()+"')");
return false;
+ }
+
// Convert the incoming acls that we know about to qualified forms, and reject the document if
// we don't know how to deal with its acls
@@ -554,8 +557,11 @@ public class HttpPoster
// Reject documents that have security we don't know how to deal with in the Solr plugin!! Only safe thing to do.
if (!aclType.equals(RepositoryDocument.SECURITY_TYPE_DOCUMENT) &&
!aclType.equals(RepositoryDocument.SECURITY_TYPE_SHARE) &&
- !aclType.startsWith(RepositoryDocument.SECURITY_TYPE_PARENT))
- return false;
+ !aclType.startsWith(RepositoryDocument.SECURITY_TYPE_PARENT)){
+ activities.recordActivity(null,SolrConnector.INGEST_ACTIVITY,null,documentURI,activities.UNKNOWN_SECURITY,"Solr connector rejected document that has security info which is unknown.");
+ return false;
+ }
+
}
try
@@ -940,9 +946,9 @@ public class HttpPoster
(activityDetails.toLowerCase(Locale.ROOT).indexOf("broken pipe") != -1 ||
activityDetails.toLowerCase(Locale.ROOT).indexOf("connection reset") != -1 ||
activityDetails.toLowerCase(Locale.ROOT).indexOf("target server failed to respond") != -1))
- activityCode = "SOLR REJECT";
+ activityCode = "SOLRREJECT";
else
- activityCode = "FAILED";
+ activityCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
// Rethrow; will interpret at a higher level
throw e;
@@ -966,7 +972,7 @@ public class HttpPoster
return;
activityStart = new Long(fullStartTime);
- activityCode = "IO ERROR";
+ activityCode = ioe.getClass().getSimpleName().toUpperCase(Locale.ROOT);
activityDetails = ioe.getMessage();
// Log the error
@@ -1331,7 +1337,7 @@ public class HttpPoster
catch (SolrServerException e)
{
activityStart = new Long(fullStartTime);
- activityCode = "FAILED";
+ activityCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
activityDetails = e.getMessage() +
((e.getCause() != null)?": "+e.getCause().getMessage():"");
@@ -1340,7 +1346,7 @@ public class HttpPoster
catch (SolrException e)
{
activityStart = new Long(fullStartTime);
- activityCode = "FAILED";
+ activityCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
activityDetails = e.getMessage() +
((e.getCause() != null)?": "+e.getCause().getMessage():"");
@@ -1352,7 +1358,7 @@ public class HttpPoster
Logging.ingest.warn("Error deleting document: "+ioe.getMessage(),ioe);
activityStart = new Long(fullStartTime);
- activityCode = "IO ERROR";
+ activityCode = ioe.getClass().getSimpleName().toUpperCase(Locale.ROOT);
activityDetails = ioe.getMessage();
throw ioe;