You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/06/05 10:54:00 UTC
svn commit: r1489745 - in /manifoldcf/trunk/connectors:
dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/
googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/
Author: kwright
Date: Wed Jun 5 08:53:59 2013
New Revision: 1489745
URL: http://svn.apache.org/r1489745
Log:
One more improvement: If interrupted, don't wait for background thread.
Modified:
manifoldcf/trunk/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java
manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java
Modified: manifoldcf/trunk/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java?rev=1489745&r1=1489744&r2=1489745&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java (original)
+++ manifoldcf/trunk/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java Wed Jun 5 08:53:59 2013
@@ -676,6 +676,7 @@ public class DropboxRepositoryConnector
GetSeedsThread t = new GetSeedsThread(dropboxPath);
try {
t.start();
+ boolean wasInterrupted = false;
try {
XThreadStringBuffer seedBuffer = t.getBuffer();
@@ -688,8 +689,16 @@ public class DropboxRepositoryConnector
// Add the pageID to the queue
activities.addSeedDocument(docPath);
}
+ } catch (InterruptedException e) {
+ wasInterrupted = true;
+ throw e;
+ } catch (ManifoldCFException e) {
+ if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+ wasInterrupted = true;
+ throw e;
} finally {
- t.finishUp();
+ if (!wasInterrupted)
+ t.finishUp();
}
} catch (InterruptedException e) {
t.interrupt();
@@ -852,6 +861,7 @@ public class DropboxRepositoryConnector
BackgroundStreamThread t = new BackgroundStreamThread(nodeId);
try {
t.start();
+ boolean wasInterrupted = false;
try {
InputStream is = t.getSafeInputStream();
try {
@@ -860,9 +870,19 @@ public class DropboxRepositoryConnector
} finally {
is.close();
}
+ } catch (java.net.SocketTimeoutException e) {
+ throw e;
+ } catch (InterruptedIOException e) {
+ wasInterrupted = true;
+ throw e;
+ } catch (ManifoldCFException e) {
+ if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+ wasInterrupted = true;
+ throw e;
} finally {
- // This does a join
- t.finishUp();
+ if (!wasInterrupted)
+ // This does a join
+ t.finishUp();
}
// No errors. Record the fact that we made it.
@@ -874,6 +894,10 @@ public class DropboxRepositoryConnector
t.interrupt();
throw new ManifoldCFException("Interrupted: " + e.getMessage(), e,
ManifoldCFException.INTERRUPTED);
+ } catch (java.net.SocketTimeoutException e) {
+ errorCode = "IO ERROR";
+ errorDesc = e.getMessage();
+ handleIOException(e);
} catch (InterruptedIOException e) {
t.interrupt();
throw new ManifoldCFException("Interrupted: " + e.getMessage(), e,
@@ -1111,7 +1135,7 @@ public class DropboxRepositoryConnector
/** Handle an IO exception. */
protected static void handleIOException(IOException e)
throws ManifoldCFException, ServiceInterruption {
- if (e instanceof InterruptedIOException) {
+ if (!(e instanceof java.net.SocketTimeoutException) && (e instanceof InterruptedIOException)) {
throw new ManifoldCFException("Interrupted: " + e.getMessage(), e,
ManifoldCFException.INTERRUPTED);
}
Modified: manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java?rev=1489745&r1=1489744&r2=1489745&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java (original)
+++ manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java Wed Jun 5 08:53:59 2013
@@ -720,6 +720,7 @@ public class GoogleDriveRepositoryConnec
GetSeedsThread t = new GetSeedsThread(googleDriveQuery);
try {
t.start();
+ boolean wasInterrupted = false;
try {
XThreadStringBuffer seedBuffer = t.getBuffer();
// Pick up the paths, and add them to the activities, before we join with the child thread.
@@ -731,8 +732,16 @@ public class GoogleDriveRepositoryConnec
// Add the pageID to the queue
activities.addSeedDocument(docPath);
}
+ } catch (InterruptedException e) {
+ wasInterrupted = true;
+ throw e;
+ } catch (ManifoldCFException e) {
+ if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+ wasInterrupted = true;
+ throw e;
} finally {
- t.finishUp();
+ if (!wasInterrupted)
+ t.finishUp();
}
} catch (InterruptedException e) {
t.interrupt();
@@ -927,6 +936,7 @@ public class GoogleDriveRepositoryConnec
GetChildrenThread t = new GetChildrenThread(nodeId);
try {
t.start();
+ boolean wasInterrupted = false;
try {
XThreadStringBuffer childBuffer = t.getBuffer();
// Pick up the paths, and add them to the activities, before we join with the child thread.
@@ -938,8 +948,16 @@ public class GoogleDriveRepositoryConnec
// Add the pageID to the queue
activities.addDocumentReference(child, nodeId, RELATIONSHIP_CHILD);
}
+ } catch (InterruptedException e) {
+ wasInterrupted = true;
+ throw e;
+ } catch (ManifoldCFException e) {
+ if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+ wasInterrupted = true;
+ throw e;
} finally {
- t.finishUp();
+ if (!wasInterrupted)
+ t.finishUp();
}
} catch (InterruptedException e) {
t.interrupt();
@@ -1002,6 +1020,7 @@ public class GoogleDriveRepositoryConnec
DocumentReadingThread t = new DocumentReadingThread(documentURI);
try {
t.start();
+ boolean wasInterrupted = false;
try {
InputStream is = t.getSafeInputStream();
try {
@@ -1011,8 +1030,18 @@ public class GoogleDriveRepositoryConnec
} finally {
is.close();
}
+ } catch (ManifoldCFException e) {
+ if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+ wasInterrupted = true;
+ throw e;
+ } catch (java.net.SocketTimeoutException e) {
+ throw e;
+ } catch (InterruptedIOException e) {
+ wasInterrupted = true;
+ throw e;
} finally {
- t.finishUp();
+ if (!wasInterrupted)
+ t.finishUp();
}
// No errors. Record the fact that we made it.
@@ -1196,9 +1225,12 @@ public class GoogleDriveRepositoryConnec
private static void handleIOException(IOException e)
throws ManifoldCFException, ServiceInterruption {
- // MHL to deal with various kinds of IOException
+ if (!(e instanceof java.net.SocketTimeoutException) && (e instanceof InterruptedIOException)) {
+ throw new ManifoldCFException("Interrupted: " + e.getMessage(), e,
+ ManifoldCFException.INTERRUPTED);
+ }
long currentTime = System.currentTimeMillis();
- throw new ServiceInterruption("GoogleDrive exception: "+e.getMessage(), e, currentTime + 300000L,
+ throw new ServiceInterruption("IO exception: "+e.getMessage(), e, currentTime + 300000L,
currentTime + 3 * 60 * 60000L,-1,false);
}