You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2011/07/25 00:19:53 UTC
svn commit: r1150502 - in /incubator/lcf/trunk: ./
framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/
framework/core/src/main/java/org/apache/manifoldcf/core/database/
framework/pull-agent/src/main/java/org/apache/manifoldcf...
Author: kwright
Date: Sun Jul 24 22:19:52 2011
New Revision: 1150502
URL: http://svn.apache.org/viewvc?rev=1150502&view=rev
Log:
Fix for CONNECTORS-225. Put a deadlock wait and retry.
Modified:
incubator/lcf/trunk/CHANGES.txt
incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Modified: incubator/lcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/CHANGES.txt?rev=1150502&r1=1150501&r2=1150502&view=diff
==============================================================================
--- incubator/lcf/trunk/CHANGES.txt (original)
+++ incubator/lcf/trunk/CHANGES.txt Sun Jul 24 22:19:52 2011
@@ -3,6 +3,10 @@ $Id$
======================= 0.3-dev =========================
+CONNECTORS-225: Derby throws deadlock exceptions when indexing
+rapidly.
+(Karl Wright)
+
CONNECTORS-223: Move test classes to be compatible with maven
conventions.
(Karl Wright)
Modified: incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1150502&r1=1150501&r2=1150502&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java (original)
+++ incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java Sun Jul 24 22:19:52 2011
@@ -78,7 +78,6 @@ public class IncrementalIngester extends
// Output connection manager
protected IOutputConnectionManager connectionManager;
-
/** Constructor.
*/
public IncrementalIngester(IThreadContext threadContext, IDBInterface database)
@@ -1362,47 +1361,62 @@ public class IncrementalIngester extends
else
map.put(authorityNameField,"");
- beginTransaction();
- try
+ // Transaction abort due to deadlock should be retried here.
+ while (true)
{
- // Look for existing row.
- ArrayList list = new ArrayList();
- list.add(docKey);
- list.add(outputConnectionName);
- IResultSet set = performQuery("SELECT "+idField+","+changeCountField+" FROM "+getTableName()+" WHERE "+
- docKeyField+"=? AND "+outputConnNameField+"=? FOR UPDATE",list,null,null);
- IResultRow row = null;
- if (set.getRowCount() > 0)
- row = set.getRow(0);
+ long sleepAmt = 0L;
- if (row != null)
+ beginTransaction();
+ try
{
- // Update the record
- list.clear();
- list.add(row.getValue(idField));
- long changeCount = ((Long)row.getValue(changeCountField)).longValue();
- changeCount++;
- map.put(changeCountField,new Long(changeCount));
- performUpdate(map,"WHERE "+idField+"=?",list,null);
- // Update successful!
- return;
+ // Look for existing row.
+ ArrayList list = new ArrayList();
+ list.add(docKey);
+ list.add(outputConnectionName);
+ IResultSet set = performQuery("SELECT "+idField+","+changeCountField+" FROM "+getTableName()+" WHERE "+
+ docKeyField+"=? AND "+outputConnNameField+"=? FOR UPDATE",list,null,null);
+ IResultRow row = null;
+ if (set.getRowCount() > 0)
+ row = set.getRow(0);
+
+ if (row != null)
+ {
+ // Update the record
+ list.clear();
+ list.add(row.getValue(idField));
+ long changeCount = ((Long)row.getValue(changeCountField)).longValue();
+ changeCount++;
+ map.put(changeCountField,new Long(changeCount));
+ performUpdate(map,"WHERE "+idField+"=?",list,null);
+ // Update successful!
+ return;
+ }
+
+ // Update failed to find a matching record, so cycle back to retry the insert
}
+ catch (ManifoldCFException e)
+ {
+ signalRollback();
+ if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
+ {
+ if (Logging.perf.isDebugEnabled())
+ Logging.perf.debug("Aborted transaction noting ingestion: "+e.getMessage());
+ sleepAmt = getSleepAmt();
+ continue;
+ }
- // Update failed to find a matching record, so cycle back to retry the insert
- }
- catch (ManifoldCFException e)
- {
- signalRollback();
- throw e;
- }
- catch (Error e)
- {
- signalRollback();
- throw e;
- }
- finally
- {
- endTransaction();
+ throw e;
+ }
+ catch (Error e)
+ {
+ signalRollback();
+ throw e;
+ }
+ finally
+ {
+ endTransaction();
+ sleepFor(sleepAmt);
+ }
}
}
}
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java?rev=1150502&r1=1150501&r2=1150502&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java Sun Jul 24 22:19:52 2011
@@ -274,6 +274,19 @@ public class BaseTable
dbInterface.endTransaction();
}
+ /** Get a random amount to sleep for (to resolve a deadlock) */
+ protected long getSleepAmt()
+ {
+ return dbInterface.getSleepAmt();
+ }
+
+ /** Sleep for a specified amount, to resolve a deadlock */
+ protected void sleepFor(long amt)
+ throws ManifoldCFException
+ {
+ dbInterface.sleepFor(amt);
+ }
+
/** Note a number of inserts, modifications, or deletions to a specific table. This is so we can decide when to do appropriate maintenance.
*@param tableName is the name of the table being modified.
*@param insertCount is the number of inserts.
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1150502&r1=1150501&r2=1150502&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java Sun Jul 24 22:19:52 2011
@@ -4230,24 +4230,13 @@ public class JobManager implements IJobM
*/
protected long getRandomAmount()
{
- // Amount should be between .5 and 1 minute, approx, to give things time to unwind
- return (long)(random.nextDouble() * 60000.0 + 500.0);
+ return database.getSleepAmt();
}
protected void sleepFor(long amt)
throws ManifoldCFException
{
- if (amt == 0L)
- return;
-
- try
- {
- ManifoldCF.sleep(amt);
- }
- catch (InterruptedException e)
- {
- throw new ManifoldCFException("Interrupted",e,ManifoldCFException.INTERRUPTED);
- }
+ database.sleepFor(amt);
}
/** Retrieve specific parent data for a given document.