You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2011/12/21 15:42:00 UTC
svn commit: r1221745 - in /incubator/lcf/trunk/framework:
agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/
agents/src/main/java/org/apache/manifoldcf/agents/outputconnmgr/
core/src/main/java/org/apache/manifoldcf/core/database/ core/...
Author: kwright
Date: Wed Dec 21 14:41:59 2011
New Revision: 1221745
URL: http://svn.apache.org/viewvc?rev=1221745&view=rev
Log:
Get rid of 'lock table' abstraction, since it is not necessary anymore and causes nothing but trouble with MySQL.
Modified:
incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java
incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnmgr/OutputConnectorManager.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authconnmgr/AuthorityConnectorManager.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connmgr/ConnectorManager.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
Modified: incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java (original)
+++ incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java Wed Dec 21 14:41:59 2011
@@ -290,7 +290,7 @@ public class OutputConnectionManager ext
beginTransaction();
try
{
- performLock();
+ //performLock();
// Notify of a change to the configuration
ManifoldCF.noteConfigurationChange();
boolean isNew = object.getIsNew();
Modified: incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnmgr/OutputConnectorManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnmgr/OutputConnectorManager.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnmgr/OutputConnectorManager.java (original)
+++ incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnmgr/OutputConnectorManager.java Wed Dec 21 14:41:59 2011
@@ -158,7 +158,7 @@ public class OutputConnectorManager exte
beginTransaction();
try
{
- performLock();
+ //performLock();
// See if already there.
ArrayList params = new ArrayList();
params.add(className);
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java Wed Dec 21 14:41:59 2011
@@ -57,14 +57,6 @@ public class BaseTable
return dbInterface.getTransactionID();
}
- /** Perform a table lock operation.
- */
- protected void performLock()
- throws ManifoldCFException
- {
- dbInterface.performLock(tableName);
- }
-
/** Perform an insert operation.
*@param invalidateKeys are the cache keys that should be
* invalidated.
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java Wed Dec 21 14:41:59 2011
@@ -171,15 +171,6 @@ public class DBInterfaceDerby extends Da
return cacheKey;
}
- /** Perform a table lock operation.
- *@param tableName is the name of the table.
- */
- public void performLock(String tableName)
- throws ManifoldCFException
- {
- performModification("LOCK TABLE "+tableName+" IN EXCLUSIVE MODE",null,null);
- }
-
/** Perform an insert operation.
*@param tableName is the name of the table.
*@param invalidateKeys are the cache keys that should be
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java Wed Dec 21 14:41:59 2011
@@ -192,15 +192,6 @@ public class DBInterfaceHSQLDB extends D
return cacheKey;
}
- /** Perform a table lock operation.
- *@param tableName is the name of the table.
- */
- public void performLock(String tableName)
- throws ManifoldCFException
- {
- performModification("LOCK TABLE "+tableName+" WRITE",null,null);
- }
-
/** Perform an insert operation.
*@param tableName is the name of the table.
*@param invalidateKeys are the cache keys that should be
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java Wed Dec 21 14:41:59 2011
@@ -74,15 +74,6 @@ public class DBInterfaceMySQL extends Da
return cacheKey;
}
- /** Perform a table lock operation.
- *@param tableName is the name of the table.
- */
- public void performLock(String tableName)
- throws ManifoldCFException
- {
- //performModification("LOCK TABLE "+tableName+" IN EXCLUSIVE MODE",null,null);
- }
-
/** Perform an insert operation.
*@param tableName is the name of the table.
*@param invalidateKeys are the cache keys that should be
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java Wed Dec 21 14:41:59 2011
@@ -131,15 +131,6 @@ public class DBInterfacePostgreSQL exten
return cacheKey;
}
- /** Perform a table lock operation.
- *@param tableName is the name of the table.
- */
- public void performLock(String tableName)
- throws ManifoldCFException
- {
- performModification("LOCK TABLE "+tableName+" IN EXCLUSIVE MODE",null,null);
- }
-
/** Perform an insert operation.
*@param tableName is the name of the table.
*@param invalidateKeys are the cache keys that should be
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java Wed Dec 21 14:41:59 2011
@@ -66,12 +66,6 @@ public interface IDBInterface
*/
public String getDatabaseCacheKey();
- /** Perform a table lock operation.
- *@param tableName is the name of the table.
- */
- public void performLock(String tableName)
- throws ManifoldCFException;
-
/** Perform an insert operation.
*@param tableName is the name of the table.
*@param invalidateKeys are the cache keys that should be
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authconnmgr/AuthorityConnectorManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authconnmgr/AuthorityConnectorManager.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authconnmgr/AuthorityConnectorManager.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authconnmgr/AuthorityConnectorManager.java Wed Dec 21 14:41:59 2011
@@ -158,7 +158,7 @@ public class AuthorityConnectorManager e
beginTransaction();
try
{
- performLock();
+ //performLock();
// See if already there.
ArrayList params = new ArrayList();
params.add(className);
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java Wed Dec 21 14:41:59 2011
@@ -267,7 +267,7 @@ public class AuthorityConnectionManager
beginTransaction();
try
{
- performLock();
+ //performLock();
ManifoldCF.noteConfigurationChange();
boolean isNew = object.getIsNew();
// See whether the instance exists
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connmgr/ConnectorManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connmgr/ConnectorManager.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connmgr/ConnectorManager.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connmgr/ConnectorManager.java Wed Dec 21 14:41:59 2011
@@ -154,7 +154,7 @@ public class ConnectorManager extends or
beginTransaction();
try
{
- performLock();
+ //performLock();
// See if already there.
ArrayList params = new ArrayList();
params.add(className);
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java Wed Dec 21 14:41:59 2011
@@ -672,111 +672,130 @@ public class Jobs extends org.apache.man
ssb.add(getJobIDKey(id));
StringSet invKeys = new StringSet(ssb);
- ICacheHandle ch = cacheManager.enterCache(null,invKeys,getTransactionID());
- try
+ while (true)
{
- beginTransaction();
+ long sleepAmt = 0L;
try
{
- performLock();
- // See whether the instance exists
- ArrayList params = new ArrayList();
- String query = buildConjunctionClause(params,new ClauseDescription[]{
- new UnitaryClause(idField,id)});
- IResultSet set = performQuery("SELECT * FROM "+getTableName()+" WHERE "+
- query+" FOR UPDATE",params,null,null);
- HashMap values = new HashMap();
- values.put(descriptionField,jobDescription.getDescription());
- values.put(outputNameField,jobDescription.getOutputConnectionName());
- values.put(connectionNameField,jobDescription.getConnectionName());
- String newOutputXML = jobDescription.getOutputSpecification().toXML();
- values.put(outputSpecField,newOutputXML);
- String newXML = jobDescription.getSpecification().toXML();
- values.put(documentSpecField,newXML);
- values.put(typeField,typeToString(jobDescription.getType()));
- values.put(startMethodField,startMethodToString(jobDescription.getStartMethod()));
- values.put(intervalField,jobDescription.getInterval());
- values.put(reseedIntervalField,jobDescription.getReseedInterval());
- values.put(expirationField,jobDescription.getExpiration());
- values.put(priorityField,new Integer(jobDescription.getPriority()));
- values.put(hopcountModeField,hopcountModeToString(jobDescription.getHopcountMode()));
-
- if (set.getRowCount() > 0)
+ ICacheHandle ch = cacheManager.enterCache(null,invKeys,getTransactionID());
+ try
{
- // Update
- // We need to reset the lastCheckTimeField if there are any changes that
- // could affect what set of documents we allow!!!
-
- IResultRow row = set.getRow(0);
-
- boolean isSame = true;
-
- // Determine whether we need to reset the scan time for documents.
- // Basically, any change to job parameters that could affect ingestion should clear isSame so that we
- // relook at all the documents, not just the recent ones.
+ beginTransaction();
+ try
+ {
+ //performLock();
+ // See whether the instance exists
+ ArrayList params = new ArrayList();
+ String query = buildConjunctionClause(params,new ClauseDescription[]{
+ new UnitaryClause(idField,id)});
+ IResultSet set = performQuery("SELECT * FROM "+getTableName()+" WHERE "+
+ query+" FOR UPDATE",params,null,null);
+ HashMap values = new HashMap();
+ values.put(descriptionField,jobDescription.getDescription());
+ values.put(outputNameField,jobDescription.getOutputConnectionName());
+ values.put(connectionNameField,jobDescription.getConnectionName());
+ String newOutputXML = jobDescription.getOutputSpecification().toXML();
+ values.put(outputSpecField,newOutputXML);
+ String newXML = jobDescription.getSpecification().toXML();
+ values.put(documentSpecField,newXML);
+ values.put(typeField,typeToString(jobDescription.getType()));
+ values.put(startMethodField,startMethodToString(jobDescription.getStartMethod()));
+ values.put(intervalField,jobDescription.getInterval());
+ values.put(reseedIntervalField,jobDescription.getReseedInterval());
+ values.put(expirationField,jobDescription.getExpiration());
+ values.put(priorityField,new Integer(jobDescription.getPriority()));
+ values.put(hopcountModeField,hopcountModeToString(jobDescription.getHopcountMode()));
+
+ if (set.getRowCount() > 0)
+ {
+ // Update
+ // We need to reset the lastCheckTimeField if there are any changes that
+ // could affect what set of documents we allow!!!
+
+ IResultRow row = set.getRow(0);
+
+ boolean isSame = true;
+
+ // Determine whether we need to reset the scan time for documents.
+ // Basically, any change to job parameters that could affect ingestion should clear isSame so that we
+ // relook at all the documents, not just the recent ones.
+
+ if (isSame)
+ {
+ String oldOutputSpecXML = (String)row.getValue(outputSpecField);
+ if (!oldOutputSpecXML.equals(newOutputXML))
+ isSame = false;
+ }
+
+ if (isSame)
+ {
+ String oldDocSpecXML = (String)row.getValue(documentSpecField);
+ if (!oldDocSpecXML.equals(newXML))
+ isSame = false;
+ }
+
+ if (!isSame)
+ values.put(lastCheckTimeField,null);
+
+ params.clear();
+ query = buildConjunctionClause(params,new ClauseDescription[]{
+ new UnitaryClause(idField,id)});
+ performUpdate(values," WHERE "+query,params,null);
+ scheduleManager.deleteRows(id);
+ hopFilterManager.deleteRows(id);
+ }
+ else
+ {
+ // Insert
+ values.put(startTimeField,null);
+ values.put(lastCheckTimeField,null);
+ values.put(endTimeField,null);
+ values.put(statusField,statusToString(STATUS_INACTIVE));
+ values.put(lastTimeField,new Long(System.currentTimeMillis()));
+ values.put(idField,id);
+ performInsert(values,null);
+ }
+
+ // Write schedule records
+ scheduleManager.writeRows(id,jobDescription);
+ // Write hop filter rows
+ hopFilterManager.writeRows(id,jobDescription);
- if (isSame)
+ cacheManager.invalidateKeys(ch);
+ break;
+ }
+ catch (ManifoldCFException e)
{
- String oldOutputSpecXML = (String)row.getValue(outputSpecField);
- if (!oldOutputSpecXML.equals(newOutputXML))
- isSame = false;
+ signalRollback();
+ throw e;
}
-
- if (isSame)
+ catch (Error e)
{
- String oldDocSpecXML = (String)row.getValue(documentSpecField);
- if (!oldDocSpecXML.equals(newXML))
- isSame = false;
+ signalRollback();
+ throw e;
+ }
+ finally
+ {
+ endTransaction();
}
-
- if (!isSame)
- values.put(lastCheckTimeField,null);
-
- params.clear();
- query = buildConjunctionClause(params,new ClauseDescription[]{
- new UnitaryClause(idField,id)});
- performUpdate(values," WHERE "+query,params,null);
- scheduleManager.deleteRows(id);
- hopFilterManager.deleteRows(id);
}
- else
+ finally
{
- // Insert
- values.put(startTimeField,null);
- values.put(lastCheckTimeField,null);
- values.put(endTimeField,null);
- values.put(statusField,statusToString(STATUS_INACTIVE));
- values.put(lastTimeField,new Long(System.currentTimeMillis()));
- values.put(idField,id);
- performInsert(values,null);
+ cacheManager.leaveCache(ch);
}
-
- // Write schedule records
- scheduleManager.writeRows(id,jobDescription);
- // Write hop filter rows
- hopFilterManager.writeRows(id,jobDescription);
-
- cacheManager.invalidateKeys(ch);
}
catch (ManifoldCFException e)
{
- signalRollback();
- throw e;
- }
- catch (Error e)
- {
- signalRollback();
- throw e;
+ if (e.getErrorCode() != ManifoldCFException.DATABASE_TRANSACTION_ABORT)
+ throw e;
+ sleepAmt = getSleepAmt();
+ continue;
}
finally
{
- endTransaction();
+ sleepFor(sleepAmt);
}
}
- finally
- {
- cacheManager.leaveCache(ch);
- }
}
/** This method is called on a restart.
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java Wed Dec 21 14:41:59 2011
@@ -355,7 +355,7 @@ public class RepositoryConnectionManager
beginTransaction();
try
{
- performLock();
+ //performLock();
// Notify of a change to the configuration
ManifoldCF.noteConfigurationChange();
boolean isNew = object.getIsNew();