You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2011/10/28 14:31:29 UTC
svn commit: r1190274 - in /incubator/lcf/trunk: ./
framework/core/src/main/java/org/apache/manifoldcf/core/database/ lib/
Author: kwright
Date: Fri Oct 28 12:31:29 2011
New Revision: 1190274
URL: http://svn.apache.org/viewvc?rev=1190274&view=rev
Log:
Fix for CONNECTORS-279. Performing REINDEX inside a transaction could lock up the system.
Modified:
incubator/lcf/trunk/ (props changed)
incubator/lcf/trunk/CHANGES.txt
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
incubator/lcf/trunk/lib/postgresql.jar
Propchange: incubator/lcf/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 28 12:31:29 2011
@@ -9,4 +9,5 @@
/incubator/lcf/branches/CONNECTORS-240:1163953-1166425
/incubator/lcf/branches/CONNECTORS-256:1172846-1182809
/incubator/lcf/branches/CONNECTORS-277:1185949-1187036
+/incubator/lcf/branches/CONNECTORS-284:1190271
/incubator/lcf/branches/CONNECTORS-32:1092556-1094216
Modified: incubator/lcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/CHANGES.txt?rev=1190274&r1=1190273&r2=1190274&view=diff
==============================================================================
--- incubator/lcf/trunk/CHANGES.txt (original)
+++ incubator/lcf/trunk/CHANGES.txt Fri Oct 28 12:31:29 2011
@@ -3,6 +3,10 @@ $Id$
======================= 0.4-dev =====================
+CONNECTORS-279: Prevent reindex or analyze from taking place within
+a transaction. This was causing hangs on our load tests.
+(Karl Wright)
+
CONNECTORS-283: Upgrade to jcifs.jar 1.3.17 in the ant build (download-dependencies target).
(Shinichiro Abe)
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java?rev=1190274&r1=1190273&r2=1190274&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java Fri Oct 28 12:31:29 2011
@@ -1197,7 +1197,8 @@ public class DBInterfaceDerby extends Da
*@param modifyCount is the number of updates.
*@param deleteCount is the number of deletions.
*/
- public void noteModifications(String tableName, int insertCount, int modifyCount, int deleteCount)
+ @Override
+ protected void noteModificationsNoTransactions(String tableName, int insertCount, int modifyCount, int deleteCount)
throws ManifoldCFException
{
String tableStatisticsLock;
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java?rev=1190274&r1=1190273&r2=1190274&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java Fri Oct 28 12:31:29 2011
@@ -1309,7 +1309,8 @@ public class DBInterfacePostgreSQL exten
*@param modifyCount is the number of updates.
*@param deleteCount is the number of deletions.
*/
- public void noteModifications(String tableName, int insertCount, int modifyCount, int deleteCount)
+ @Override
+ protected void noteModificationsNoTransactions(String tableName, int insertCount, int modifyCount, int deleteCount)
throws ManifoldCFException
{
String tableStatisticsLock;
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java?rev=1190274&r1=1190273&r2=1190274&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java Fri Oct 28 12:31:29 2011
@@ -48,6 +48,9 @@ public class Database
protected Connection connection = null;
protected boolean doRollback = false;
protected int delayedTransactionDepth = 0;
+ protected Map<String,Modifications> modificationsSet = new HashMap<String,Modifications>();
+
+ protected static Random random = new Random();
protected final static String _TRANSACTION_ = "_TRANSACTION_";
@@ -340,10 +343,31 @@ public class Database
else
cacheManager.commitTransaction(th.getTransactionID());
th = parentTransaction;
+ if (th == null)
+ {
+ if (doRollback)
+ modificationsSet.clear();
+ else
+ playbackModifications();
+ }
}
}
+ /** Playback modifications */
+ private void playbackModifications()
+ throws ManifoldCFException
+ {
+ Iterator<String> modIterator = modificationsSet.keySet().iterator();
+ while (modIterator.hasNext())
+ {
+ String tableName = modIterator.next();
+ Modifications c = modificationsSet.get(tableName);
+ noteModificationsNoTransactions(tableName,c.getInsertCount(),c.getModifyCount(),c.getDeleteCount());
+ }
+ modificationsSet.clear();
+ }
+
/** Note a number of inserts, modifications, or deletions to a specific table. This is so we can decide when to do appropriate maintenance.
*@param tableName is the name of the table being modified.
*@param insertCount is the number of inserts.
@@ -353,10 +377,29 @@ public class Database
public void noteModifications(String tableName, int insertCount, int modifyCount, int deleteCount)
throws ManifoldCFException
{
+ if (th != null)
+ {
+ // In a transaction; record for later
+ Modifications c = modificationsSet.get(tableName);
+ if (c == null)
+ {
+ c = new Modifications();
+ modificationsSet.put(tableName,c);
+ }
+ c.update(insertCount,modifyCount,deleteCount);
+ }
+ else
+ noteModificationsNoTransactions(tableName,insertCount,modifyCount,deleteCount);
}
- protected static Random random = new Random();
-
+ /** Protected method for receiving information about inserts, modifications, or deletions OUTSIDE of all transactions.
+ */
+ protected void noteModificationsNoTransactions(String tableName, int insertCount, int modifyCount, int deleteCount)
+ throws ManifoldCFException
+ {
+ }
+
+
/** Sleep a random amount of time after a transaction abort.
*/
public long getSleepAmt()
@@ -383,6 +426,41 @@ public class Database
}
}
+ /** Class to keep track of modifications while we're in a transaction.
+ */
+ protected static class Modifications
+ {
+ protected int insertCount = 0;
+ protected int modifyCount = 0;
+ protected int deleteCount = 0;
+
+ public Modifications()
+ {
+ }
+
+ public void update(int insertCount, int modifyCount, int deleteCount)
+ {
+ this.insertCount += insertCount;
+ this.modifyCount += modifyCount;
+ this.deleteCount += deleteCount;
+ }
+
+ public int getInsertCount()
+ {
+ return insertCount;
+ }
+
+ public int getModifyCount()
+ {
+ return modifyCount;
+ }
+
+ public int getDeleteCount()
+ {
+ return deleteCount;
+ }
+ }
+
/** Thread used to execute queries. An instance of this thread is spun up every time a query is executed. This is necessary because JDBC does not
* guarantee interruptability, and the Postgresql JDBC driver unfortunately eats all thread interrupts. So, we fire up a thread to do each interaction with
* the database server, thus insuring that the owning thread remains interruptable and will therefore not block shutdown.
Modified: incubator/lcf/trunk/lib/postgresql.jar
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/lib/postgresql.jar?rev=1190274&r1=1190273&r2=1190274&view=diff
==============================================================================
Binary files - no diff available.