You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/12/26 18:54:07 UTC
svn commit: r1553520 - in /manifoldcf/trunk: ./
framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/
framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/
tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/
Author: kwright
Date: Thu Dec 26 17:54:07 2013
New Revision: 1553520
URL: http://svn.apache.org/r1553520
Log:
Fix for CONNECTORS-733.
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupQueue.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentQueue.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireStufferThread.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ResetManager.java
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java
manifoldcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/FlakyDerbyInstance.java
manifoldcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/RSSFlakyDerbyIT.java
manifoldcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/RSSSimpleCrawlTester.java
Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1553520&r1=1553519&r2=1553520&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Thu Dec 26 17:54:07 2013
@@ -3,6 +3,10 @@ $Id$
======================= 1.5-dev =====================
+CONNECTORS-733: Write a test which exercises database connection
+interruption logic, and debug the same.
+(Ahmet Arslan, Karl Wright)
+
CONNECTORS-816: Document revisions to SharePoint connector,
and new SharePoint authorities.
(Karl Wright)
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupQueue.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupQueue.java?rev=1553520&r1=1553519&r2=1553520&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupQueue.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupQueue.java Thu Dec 26 17:54:07 2013
@@ -32,7 +32,9 @@ public class DocumentCleanupQueue
public static final String _rcsid = "@(#)$Id$";
// Since the queue has a maximum size, an ArrayList is a fine way to keep it
- protected ArrayList queue = new ArrayList();
+ protected final List<DocumentCleanupSet> queue = new ArrayList<DocumentCleanupSet>();
+ // This flag gets set to 'true' if the queue is being cleared due to a reset
+ protected boolean resetFlag = false;
/** Constructor.
*/
@@ -46,6 +48,7 @@ public class DocumentCleanupQueue
{
synchronized (queue)
{
+ resetFlag = true;
queue.notifyAll();
}
}
@@ -57,6 +60,7 @@ public class DocumentCleanupQueue
synchronized (queue)
{
queue.clear();
+ resetFlag = false;
}
}
@@ -96,14 +100,19 @@ public class DocumentCleanupQueue
synchronized (queue)
{
// If queue is empty, go to sleep
- while (queue.size() == 0)
+ if (resetFlag)
+ return null;
+
+ while (queue.size() == 0 && resetFlag == false)
queue.wait();
+
// If we've been awakened, there's either an entry to grab, or we've been
// awakened because it's time to reset.
- if (queue.size() == 0)
- return null;
+ if (resetFlag)
+ return null;
+
// If we've been awakened, there's an entry to grab
- DocumentCleanupSet dd = (DocumentCleanupSet)queue.remove(queue.size()-1);
+ DocumentCleanupSet dd = queue.remove(queue.size()-1);
return dd;
}
}
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentQueue.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentQueue.java?rev=1553520&r1=1553519&r2=1553520&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentQueue.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentQueue.java Thu Dec 26 17:54:07 2013
@@ -32,7 +32,7 @@ public class DocumentQueue
public static final String _rcsid = "@(#)$Id: DocumentQueue.java 988245 2010-08-23 18:39:35Z kwright $";
// Since the queue has a maximum size, an ArrayList is a fine way to keep it
- protected ArrayList queue = new ArrayList();
+ protected final List<QueuedDocumentSet> queue = new ArrayList<QueuedDocumentSet>();
// This flag gets set to 'true' if the queue is being cleared due to a reset
protected boolean resetFlag = false;
@@ -111,7 +111,7 @@ public class DocumentQueue
// If we've been awakened, there's either an entry to grab, or we've been
// awakened because it's time to reset.
- if (queue.size() == 0)
+ if (resetFlag)
return null;
// Go through all the documents and pick the one with the best rating
@@ -120,7 +120,7 @@ public class DocumentQueue
double bestRating = Double.NEGATIVE_INFINITY;
while (i < queue.size())
{
- QueuedDocumentSet dd = (QueuedDocumentSet)queue.get(i);
+ QueuedDocumentSet dd = queue.get(i);
// Evaluate each document's bins. These will be saved in the QueuedDocumentSet.
double rating = dd.calculateAssignmentRating(overlapCalculator);
if (bestIndex == -1 || rating > bestRating)
@@ -131,7 +131,7 @@ public class DocumentQueue
i++;
}
// Pull off the best one. DON'T REORDER!!
- QueuedDocumentSet rval = (QueuedDocumentSet)queue.remove(bestIndex);
+ QueuedDocumentSet rval = queue.remove(bestIndex);
return rval;
}
}
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireStufferThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireStufferThread.java?rev=1553520&r1=1553519&r2=1553520&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireStufferThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireStufferThread.java Thu Dec 26 17:54:07 2013
@@ -134,7 +134,7 @@ public class ExpireStufferThread extends
// The theory is that we need to allow stuff to accumulate.
if (descs.length == 0)
{
- ManifoldCF.sleep(60000L); // 1 minute
+ ManifoldCF.sleep(5000L); // 5 seconds
continue;
}
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ResetManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ResetManager.java?rev=1553520&r1=1553519&r2=1553520&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ResetManager.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ResetManager.java Thu Dec 26 17:54:07 2013
@@ -41,11 +41,11 @@ public abstract class ResetManager
protected final String processID;
/** Boolean which describes whether an event requiring reset has occurred. */
- protected boolean resetRequired = false;
+ protected volatile boolean resetRequired = false;
/** This is the count of the threads that care about this resource. */
protected int involvedThreadCount = 0;
/** This is the number of threads that are waiting for the reset. */
- protected int waitingThreads = 0;
+ protected volatile int waitingThreads = 0;
/** Constructor.
*/
@@ -63,9 +63,13 @@ public abstract class ResetManager
/** Note a resettable event.
*/
- public synchronized void noteEvent()
+ public void noteEvent()
{
- resetRequired = true;
+ //System.out.println(this + " Event noted; involvedThreadCount = "+involvedThreadCount);
+ synchronized (this)
+ {
+ resetRequired = true;
+ }
performWakeupLogic();
}
@@ -81,12 +85,14 @@ public abstract class ResetManager
{
if (resetRequired == false)
return false;
+
waitingThreads++;
// Check if this is the "Prince Charming" thread, who will wake up
// all the others.
if (waitingThreads == involvedThreadCount)
{
+ //System.out.println(this + " Prince Charming thread found!");
// Kick off reset, and wake everyone up
// There's a question of what to do if the reset fails.
// Right now, my notion is that we throw the exception
@@ -108,6 +114,7 @@ public abstract class ResetManager
return true;
}
+ //System.out.println(this + " Waiting threads = "+waitingThreads+"; going to sleep");
// Just go to sleep until kicked.
wait();
// If we were awakened, it's because reset was fired.
Modified: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java?rev=1553520&r1=1553519&r2=1553520&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java Thu Dec 26 17:54:07 2013
@@ -67,7 +67,7 @@ public class ManifoldCFInstance
public ManifoldCFInstance()
{
- this(ManifoldCF.getProcessID(), 8346, false, true);
+ this("", 8346, false, true);
}
public ManifoldCFInstance(String processID)
@@ -77,7 +77,7 @@ public class ManifoldCFInstance
public ManifoldCFInstance(boolean singleWar)
{
- this(ManifoldCF.getProcessID(), 8346, singleWar, true);
+ this("", 8346, singleWar, true);
}
public ManifoldCFInstance(String processID, boolean singleWar)
@@ -87,7 +87,7 @@ public class ManifoldCFInstance
public ManifoldCFInstance(boolean singleWar, boolean webapps)
{
- this(ManifoldCF.getProcessID(), 8346, singleWar, webapps);
+ this("", 8346, singleWar, webapps);
}
public ManifoldCFInstance(String processID, boolean singleWar, boolean webapps)
Modified: manifoldcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/FlakyDerbyInstance.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/FlakyDerbyInstance.java?rev=1553520&r1=1553519&r2=1553520&view=diff
==============================================================================
--- manifoldcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/FlakyDerbyInstance.java (original)
+++ manifoldcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/FlakyDerbyInstance.java Thu Dec 26 17:54:07 2013
@@ -26,11 +26,14 @@ import org.junit.*;
import java.sql.*;
import javax.naming.*;
import javax.sql.*;
+import java.util.concurrent.atomic.*;
/** This is a very basic sanity check */
public class FlakyDerbyInstance extends org.apache.manifoldcf.core.database.DBInterfaceDerby
{
+ protected final static AtomicBoolean lostConnection = new AtomicBoolean(false);
+
public FlakyDerbyInstance(IThreadContext tc, String databaseName, String userName, String password)
throws ManifoldCFException
{
@@ -48,8 +51,25 @@ public class FlakyDerbyInstance extends
ResultSpecification spec, ILimitChecker returnLimit)
throws ManifoldCFException
{
- // MHL
- return super.execute(connection,query,params,bResults,maxResults,spec,returnLimit);
+ if (!lostConnection.get())
+ return super.execute(connection,query,params,bResults,maxResults,spec,returnLimit);
+ // Simulate a dead connection by throwing a database error
+ try
+ {
+ // Sleep, to limit log noise.
+ Thread.sleep(1000L);
+ }
+ catch (InterruptedException e)
+ {
+ throw new ManifoldCFException(e.getMessage(),ManifoldCFException.INTERRUPTED);
+ }
+
+ throw new ManifoldCFException("Database error", new Exception("Manufactured db error"), ManifoldCFException.DATABASE_CONNECTION_ERROR);
}
+ public static void setConnectionWorking(boolean value)
+ {
+ lostConnection.set(!value);
+ }
+
}
Modified: manifoldcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/RSSFlakyDerbyIT.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/RSSFlakyDerbyIT.java?rev=1553520&r1=1553519&r2=1553520&view=diff
==============================================================================
--- manifoldcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/RSSFlakyDerbyIT.java (original)
+++ manifoldcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/RSSFlakyDerbyIT.java Thu Dec 26 17:54:07 2013
@@ -23,11 +23,39 @@ import java.util.*;
import org.junit.*;
/** This is a very basic sanity check */
-public class RSSFlakyDerbyIT extends RSSSimpleCrawlDerbyIT
+public class RSSFlakyDerbyIT extends BaseDerby
{
+ protected RSSSimpleCrawlTester tester;
+ protected MockRSSService rssService = null;
+
public RSSFlakyDerbyIT()
{
- super();
+ tester = new RSSSimpleCrawlTester(mcfInstance);
+ }
+
+ // Setup and teardown the mock wiki service
+
+ @Before
+ public void createRSSService()
+ throws Exception
+ {
+ rssService = new MockRSSService(10);
+ rssService.start();
+ }
+
+ @After
+ public void shutdownRSSService()
+ throws Exception
+ {
+ if (rssService != null)
+ rssService.stop();
+ }
+
+ @Test
+ public void simpleCrawl()
+ throws Exception
+ {
+ tester.executeTest(new DBInterruptionNotification());
}
/** Method to get database implementation class */
@@ -38,4 +66,24 @@ public class RSSFlakyDerbyIT extends RSS
return FlakyDerbyInstance.class.getName();
}
+ protected static class DBInterruptionNotification implements RSSSimpleCrawlTester.TestNotification
+ {
+ public void notifyMe()
+ throws Exception
+ {
+ // Wait 5 seconds, then turn of database access for 10 seconds. Then, do it again.
+ Thread.sleep(5000L);
+ FlakyDerbyInstance.setConnectionWorking(false);
+ System.out.println("Database connectivity is OFF");
+ Thread.sleep(10000L);
+ FlakyDerbyInstance.setConnectionWorking(true);
+ System.out.println("Database connectivity restored");
+ Thread.sleep(5000L);
+ FlakyDerbyInstance.setConnectionWorking(false);
+ System.out.println("Database connectivity is OFF");
+ Thread.sleep(10000L);
+ FlakyDerbyInstance.setConnectionWorking(true);
+ System.out.println("Database connectivity restored");
+ }
+ }
}
Modified: manifoldcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/RSSSimpleCrawlTester.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/RSSSimpleCrawlTester.java?rev=1553520&r1=1553519&r2=1553520&view=diff
==============================================================================
--- manifoldcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/RSSSimpleCrawlTester.java (original)
+++ manifoldcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/RSSSimpleCrawlTester.java Thu Dec 26 17:54:07 2013
@@ -41,6 +41,12 @@ public class RSSSimpleCrawlTester
public void executeTest()
throws Exception
{
+ executeTest(null);
+ }
+
+ public void executeTest(TestNotification tn)
+ throws Exception
+ {
// Hey, we were able to install the file system connector etc.
// Now, create a local test job and run it.
IThreadContext tc = ThreadContextFactory.make();
@@ -60,7 +66,7 @@ public class RSSSimpleCrawlTester
cp.setParameter(RSSConfig.PARAMETER_ROBOTSUSAGE,"none");
// Now, save
mgr.save(conn);
-
+
// Create a basic null output connection, and save it.
IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
IOutputConnection outputConn = outputMgr.create();
@@ -101,6 +107,11 @@ public class RSSSimpleCrawlTester
// Now, start the job, and wait until it completes.
long startTime = System.currentTimeMillis();
jobManager.manualStart(job.getID());
+ // Wait 15 seconds, then do a notification
+ if (tn != null)
+ {
+ tn.notifyMe();
+ }
instance.waitJobInactiveNative(jobManager,job.getID(),600000L);
System.err.println("Crawl required "+new Long(System.currentTimeMillis()-startTime).toString()+" milliseconds");
@@ -117,4 +128,9 @@ public class RSSSimpleCrawlTester
// Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
}
+ public static interface TestNotification
+ {
+ public void notifyMe()
+ throws Exception;
+ }
}