You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/10/19 19:51:04 UTC
svn commit: r1632961 - in /manifoldcf/branches/dev_1x: ./
connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/
connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/
framework/ framewo...
Author: kwright
Date: Sun Oct 19 17:51:04 2014
New Revision: 1632961
URL: http://svn.apache.org/r1632961
Log:
Pull up changes related to CONNECTORS-1077 from trunk
Modified:
manifoldcf/branches/dev_1x/ (props changed)
manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
manifoldcf/branches/dev_1x/framework/ (props changed)
manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java
Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk:r1632844,1632847,1632854
Modified: manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java?rev=1632961&r1=1632960&r2=1632961&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java (original)
+++ manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java Sun Oct 19 17:51:04 2014
@@ -19,7 +19,6 @@
package org.apache.manifoldcf.agents.output.filesystem;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -37,10 +36,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.HashMap;
-import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
-import org.apache.manifoldcf.agents.interfaces.IOutputRemoveActivity;
-import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
-import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
+import org.apache.manifoldcf.agents.interfaces.*;
import org.apache.manifoldcf.agents.output.BaseOutputConnector;
import org.apache.manifoldcf.agents.system.Logging;
import org.apache.manifoldcf.core.interfaces.Specification;
@@ -48,7 +44,6 @@ import org.apache.manifoldcf.core.interf
import org.apache.manifoldcf.core.interfaces.ConfigurationNode;
import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
import org.apache.manifoldcf.core.interfaces.IPostParameters;
-import org.apache.manifoldcf.core.interfaces.IThreadContext;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
import org.apache.manifoldcf.core.interfaces.SpecificationNode;
import org.apache.manifoldcf.core.interfaces.VersionContext;
@@ -219,7 +214,10 @@ public class FileOutputConnector extends
suffix++;
}
else
- throw new ManifoldCFException("Could not create directory '"+newPath+"'. Permission issue?");
+ {
+ activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.CREATED_DIRECTORY,"Couldn't create directory ("+newPath+").");
+ throw new ManifoldCFException("Could not create directory '"+newPath+"'. Permission issue?");
+ }
}
// Directory successfully created!
currentPath = newPath;
@@ -249,6 +247,7 @@ public class FileOutputConnector extends
continue;
}
// Probably some other error
+ activities.recordActivity(null, INGEST_ACTIVITY, new Long(document.getBinaryLength()), documentURI, activities.EXCEPTION, "Rejected due to FileNotFoundException.");
throw new ManifoldCFException("Could not create file '"+outputPath+"': "+e.getMessage(),e);
}
}
@@ -259,8 +258,11 @@ public class FileOutputConnector extends
*/
FileChannel channel = output.getChannel();
FileLock lock = channel.tryLock();
- if (lock == null)
+ if (lock == null){
+ activities.recordActivity(null,INGEST_ACTIVITY, new Long(document.getBinaryLength()), documentURI,activities.EXCEPTION,"Rejected due to ServiceInterruptionException.");
throw new ServiceInterruption("Could not lock file: '"+outputPath+"'",null,1000L,-1L,10,false);
+ }
+
try {
@@ -290,15 +292,19 @@ public class FileOutputConnector extends
}
}
} catch (URISyntaxException e) {
+ activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()), documentURI,activities.EXCEPTION,"Rejected due to URISyntaxException");
handleURISyntaxException(e);
return DOCUMENTSTATUS_REJECTED;
} catch (FileNotFoundException e) {
+ activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()), documentURI,activities.EXCEPTION,"Rejected due to FileNotFoundException");
handleFileNotFoundException(e);
return DOCUMENTSTATUS_REJECTED;
} catch (SecurityException e) {
+ activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()), documentURI,activities.EXCEPTION,"Rejected due to SecurityException");
handleSecurityException(e);
return DOCUMENTSTATUS_REJECTED;
} catch (IOException e) {
+ activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()), documentURI ,activities.EXCEPTION,"Rejected due to IOException.");
handleIOException(e);
return DOCUMENTSTATUS_REJECTED;
}
@@ -426,18 +432,23 @@ public class FileOutputConnector extends
catch (FileNotFoundException e)
{
// Probably some other error
+ activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Couldn't deleted due to FileNotFoundException");
throw new ManifoldCFException("Could not zero out file '"+outputPath+"': "+e.getMessage(),e);
}
}
// Just close it, to make a zero-length grave marker.
output.close();
} catch (URISyntaxException e) {
+ activities.recordActivity(null,REMOVE_ACTIVITY,null, documentURI,activities.EXCEPTION,"Rejected due to URISyntaxException");
handleURISyntaxException(e);
} catch (FileNotFoundException e) {
+ activities.recordActivity(null,REMOVE_ACTIVITY,null, documentURI,activities.EXCEPTION,"Rejected due to FileNotFoundException");
handleFileNotFoundException(e);
} catch (SecurityException e) {
+ activities.recordActivity(null,REMOVE_ACTIVITY,null, documentURI,activities.EXCEPTION,"Rejected due to SecurityException");
handleSecurityException(e);
} catch (IOException e) {
+ activities.recordActivity(null,INGEST_ACTIVITY,null, documentURI ,activities.EXCEPTION,"Rejected due to IOException.");
handleIOException(e);
}
Modified: manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java?rev=1632961&r1=1632960&r2=1632961&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java (original)
+++ manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java Sun Oct 19 17:51:04 2014
@@ -286,10 +286,11 @@ public class HDFSOutputConnector extends
Path path = new Path(strBuff.toString());
Long startTime = new Long(System.currentTimeMillis());
- createFile(path, document.getBinaryStream());
+ createFile(path, document.getBinaryStream(),activities,documentURI);
activities.recordActivity(startTime, INGEST_ACTIVITY, new Long(document.getBinaryLength()), documentURI, "OK", null);
return DOCUMENTSTATUS_ACCEPTED;
} catch (URISyntaxException e) {
+ activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()),documentURI,activities.EXCEPTION,"Rejected due to URISyntaxException");
handleURISyntaxException(e);
return DOCUMENTSTATUS_REJECTED;
}
@@ -320,11 +321,13 @@ public class HDFSOutputConnector extends
strBuff.append(documentURItoFilePath(documentURI));
Path path = new Path(strBuff.toString());
Long startTime = new Long(System.currentTimeMillis());
- deleteFile(path);
+ deleteFile(path,activities,documentURI);
activities.recordActivity(startTime, REMOVE_ACTIVITY, null, documentURI, "OK", null);
} catch (JSONException e) {
+ activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected due to JSONException.");
handleJSONException(e);
} catch (URISyntaxException e) {
+ activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected due to URISyntaxException.");
handleURISyntaxException(e);
}
}
@@ -669,7 +672,7 @@ public class HDFSOutputConnector extends
}
}
- protected void createFile(Path path, InputStream input)
+ protected void createFile(Path path, InputStream input,IOutputAddActivity activities, String documentURI)
throws ManifoldCFException, ServiceInterruption {
CreateFileThread t = new CreateFileThread(getSession(), path, input);
try {
@@ -677,13 +680,17 @@ public class HDFSOutputConnector extends
t.finishUp();
} catch (InterruptedException e) {
t.interrupt();
+ activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected due to InterruptedException.");
throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
} catch (java.net.SocketTimeoutException e) {
+ activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected due to SocketTimeoutException.");
handleIOException(e);
} catch (InterruptedIOException e) {
t.interrupt();
+ activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected due to InterruptedIOException.");
handleIOException(e);
} catch (IOException e) {
+ activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected due to IOException.");
handleIOException(e);
}
}
@@ -723,7 +730,7 @@ public class HDFSOutputConnector extends
}
}
- protected void deleteFile(Path path)
+ protected void deleteFile(Path path,IOutputRemoveActivity activities,String documentURI)
throws ManifoldCFException, ServiceInterruption {
// Establish a session
DeleteFileThread t = new DeleteFileThread(getSession(),path);
@@ -732,13 +739,17 @@ public class HDFSOutputConnector extends
t.finishUp();
} catch (InterruptedException e) {
t.interrupt();
+ activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected due to InterruptedException.");
throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
} catch (java.net.SocketTimeoutException e) {
+ activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected due to SocketTimeoutException.");
handleIOException(e);
} catch (InterruptedIOException e) {
t.interrupt();
+ activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected due to InterruptedIOException.");
handleIOException(e);
} catch (IOException e) {
+ activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected due to IOException.");
handleIOException(e);
}
}
Propchange: manifoldcf/branches/dev_1x/framework/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk/framework:r1632844
Modified: manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java?rev=1632961&r1=1632960&r2=1632961&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java (original)
+++ manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java Sun Oct 19 17:51:04 2014
@@ -27,6 +27,9 @@ public interface IOutputHistoryActivity
{
public static final String _rcsid = "@(#)$Id: IOutputHistoryActivity.java 988245 2010-08-23 18:39:35Z kwright $";
+ public static final String EXCEPTION = "EXCEPTION";
+ public static final String CREATED_DIRECTORY = "CREATEDDIRECTORY";
+
/** Record time-stamped information about the activity of the output connector.
*@param startTime is either null or the time since the start of epoch in milliseconds (Jan 1, 1970). Every
* activity has an associated time; the startTime field records when the activity began. A null value