You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2011/07/08 17:21:22 UTC
svn commit: r1144356 - in /incubator/lcf/trunk: ./
framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/
Author: kwright
Date: Fri Jul 8 15:21:21 2011
New Revision: 1144356
URL: http://svn.apache.org/viewvc?rev=1144356&view=rev
Log:
Fix for CONNECTORS-220. Move reset activity into noteEvent method, so ALL queues can be reset, not just one.
Modified:
incubator/lcf/trunk/CHANGES.txt
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocCleanupResetManager.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocDeleteResetManager.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupStufferThread.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupThread.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentDeleteStufferThread.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentDeleteThread.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireStufferThread.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireThread.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobNotificationThread.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ResetManager.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartDeleteThread.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerResetManager.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
Modified: incubator/lcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/CHANGES.txt?rev=1144356&r1=1144355&r2=1144356&view=diff
==============================================================================
--- incubator/lcf/trunk/CHANGES.txt (original)
+++ incubator/lcf/trunk/CHANGES.txt Fri Jul 8 15:21:21 2011
@@ -3,6 +3,10 @@ $Id$
======================= 0.3-dev =========================
+CONNECTORS-220: Fix long-standing issue with database connection
+problems. Reset logic was insufficiently robust.
+(Farzad Valad, Karl Wright)
+
CONNECTORS-217: Check running more than one of instance of agent.
(Shinichiro Abe)
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocCleanupResetManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocCleanupResetManager.java?rev=1144356&r1=1144355&r2=1144356&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocCleanupResetManager.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocCleanupResetManager.java Fri Jul 8 15:21:21 2011
@@ -48,4 +48,12 @@ public class DocCleanupResetManager exte
jobManager.resetDocCleanupWorkerStatus();
ddq.clear();
}
+
+ /** Do the wakeup logic.
+ */
+ protected void performWakeupLogic()
+ {
+ ddq.reset();
+ }
+
}
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocDeleteResetManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocDeleteResetManager.java?rev=1144356&r1=1144355&r2=1144356&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocDeleteResetManager.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocDeleteResetManager.java Fri Jul 8 15:21:21 2011
@@ -48,4 +48,12 @@ public class DocDeleteResetManager exten
jobManager.resetDocDeleteWorkerStatus();
ddq.clear();
}
+
+ /** Do the wakeup logic.
+ */
+ protected void performWakeupLogic()
+ {
+ ddq.reset();
+ }
+
}
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupStufferThread.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupStufferThread.java?rev=1144356&r1=1144355&r2=1144356&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupStufferThread.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupStufferThread.java Fri Jul 8 15:21:21 2011
@@ -163,7 +163,6 @@ public class DocumentCleanupStufferThrea
if (e.getErrorCode() == ManifoldCFException.DATABASE_CONNECTION_ERROR)
{
resetManager.noteEvent();
- documentCleanupQueue.reset();
Logging.threads.error("Cleanup stuffer thread aborting and restarting due to database connection reset",e);
try
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupThread.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupThread.java?rev=1144356&r1=1144355&r2=1144356&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupThread.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupThread.java Fri Jul 8 15:21:21 2011
@@ -309,8 +309,6 @@ public class DocumentCleanupThread exten
{
// Note the failure, which will cause a reset to occur
resetManager.noteEvent();
- // Wake up all sleeping worker threads
- documentCleanupQueue.reset();
Logging.threads.error("Document cleanup thread aborting and restarting due to database connection reset: "+e.getMessage(),e);
try
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentDeleteStufferThread.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentDeleteStufferThread.java?rev=1144356&r1=1144355&r2=1144356&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentDeleteStufferThread.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentDeleteStufferThread.java Fri Jul 8 15:21:21 2011
@@ -161,7 +161,6 @@ public class DocumentDeleteStufferThread
if (e.getErrorCode() == ManifoldCFException.DATABASE_CONNECTION_ERROR)
{
resetManager.noteEvent();
- documentDeleteQueue.reset();
Logging.threads.error("Delete stuffer thread aborting and restarting due to database connection reset",e);
try
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentDeleteThread.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentDeleteThread.java?rev=1144356&r1=1144355&r2=1144356&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentDeleteThread.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentDeleteThread.java Fri Jul 8 15:21:21 2011
@@ -214,7 +214,6 @@ public class DocumentDeleteThread extend
if (e.getErrorCode() == ManifoldCFException.DATABASE_CONNECTION_ERROR)
{
resetManager.noteEvent();
- documentDeleteQueue.reset();
Logging.threads.error("Document delete thread aborting and restarting due to database connection reset: "+e.getMessage(),e);
try
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireStufferThread.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireStufferThread.java?rev=1144356&r1=1144355&r2=1144356&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireStufferThread.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireStufferThread.java Fri Jul 8 15:21:21 2011
@@ -179,7 +179,6 @@ public class ExpireStufferThread extends
if (e.getErrorCode() == ManifoldCFException.DATABASE_CONNECTION_ERROR)
{
resetManager.noteEvent();
- documentQueue.reset();
Logging.threads.error("Expiration stuffer thread aborting and restarting due to database connection reset: "+e.getMessage(),e);
try
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireThread.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireThread.java?rev=1144356&r1=1144355&r2=1144356&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireThread.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireThread.java Fri Jul 8 15:21:21 2011
@@ -312,8 +312,6 @@ public class ExpireThread extends Thread
{
// Note the failure, which will cause a reset to occur
resetManager.noteEvent();
- // Wake up all sleeping worker threads
- documentQueue.reset();
Logging.threads.error("Expiration thread aborting and restarting due to database connection reset: "+e.getMessage(),e);
try
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobNotificationThread.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobNotificationThread.java?rev=1144356&r1=1144355&r2=1144356&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobNotificationThread.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobNotificationThread.java Fri Jul 8 15:21:21 2011
@@ -350,6 +350,13 @@ public class JobNotificationThread exten
IJobManager jobManager = JobManagerFactory.make(tc);
jobManager.resetNotificationWorkerStatus();
}
+
+ /** Do the wakeup logic.
+ */
+ protected void performWakeupLogic()
+ {
+ }
+
}
}
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ResetManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ResetManager.java?rev=1144356&r1=1144355&r2=1144356&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ResetManager.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ResetManager.java Fri Jul 8 15:21:21 2011
@@ -62,6 +62,7 @@ public abstract class ResetManager
public synchronized void noteEvent()
{
resetRequired = true;
+ performWakeupLogic();
}
/** Enter "wait" state for current thread.
@@ -114,5 +115,8 @@ public abstract class ResetManager
protected abstract void performResetLogic(IThreadContext tc)
throws ManifoldCFException;
-
+ /** Do the wakeup logic.
+ */
+ protected abstract void performWakeupLogic();
+
}
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java?rev=1144356&r1=1144355&r2=1144356&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java Fri Jul 8 15:21:21 2011
@@ -312,6 +312,13 @@ public class SeedingThread extends Threa
IJobManager jobManager = JobManagerFactory.make(tc);
jobManager.resetSeedingWorkerStatus();
}
+
+ /** Do the wakeup logic.
+ */
+ protected void performWakeupLogic()
+ {
+ }
+
}
}
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartDeleteThread.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartDeleteThread.java?rev=1144356&r1=1144355&r2=1144356&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartDeleteThread.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartDeleteThread.java Fri Jul 8 15:21:21 2011
@@ -228,6 +228,13 @@ public class StartDeleteThread extends T
IJobManager jobManager = JobManagerFactory.make(tc);
jobManager.resetDeleteStartupWorkerStatus();
}
+
+ /** Do the wakeup logic.
+ */
+ protected void performWakeupLogic()
+ {
+ }
+
}
}
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java?rev=1144356&r1=1144355&r2=1144356&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java Fri Jul 8 15:21:21 2011
@@ -327,6 +327,13 @@ public class StartupThread extends Threa
IJobManager jobManager = JobManagerFactory.make(tc);
jobManager.resetStartupWorkerStatus();
}
+
+ /** Do the wakeup logic.
+ */
+ protected void performWakeupLogic()
+ {
+ }
+
}
}
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java?rev=1144356&r1=1144355&r2=1144356&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java Fri Jul 8 15:21:21 2011
@@ -353,7 +353,6 @@ public class StufferThread extends Threa
if (e.getErrorCode() == ManifoldCFException.DATABASE_CONNECTION_ERROR)
{
resetManager.noteEvent();
- documentQueue.reset();
Logging.threads.error("Stuffer thread aborting and restarting due to database connection reset: "+e.getMessage(),e);
try
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerResetManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerResetManager.java?rev=1144356&r1=1144355&r2=1144356&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerResetManager.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerResetManager.java Fri Jul 8 15:21:21 2011
@@ -52,5 +52,14 @@ public class WorkerResetManager extends
dq.clear();
eq.clear();
}
+
+ /** Do the wakeup logic.
+ */
+ protected void performWakeupLogic()
+ {
+ // Wake up all sleeping worker threads
+ dq.reset();
+ eq.reset();
+ }
}
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java?rev=1144356&r1=1144355&r2=1144356&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java Fri Jul 8 15:21:21 2011
@@ -917,8 +917,6 @@ public class WorkerThread extends Thread
{
// Note the failure, which will cause a reset to occur
resetManager.noteEvent();
- // Wake up all sleeping worker threads
- documentQueue.reset();
Logging.threads.error("Worker thread aborting and restarting due to database connection reset: "+e.getMessage(),e);
try