You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2011/03/17 14:30:27 UTC
svn commit: r1082484 - in /incubator/lcf/trunk: ./
framework/core/src/main/java/org/apache/manifoldcf/core/database/
framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/
framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/
Author: kwright
Date: Thu Mar 17 13:30:26 2011
New Revision: 1082484
URL: http://svn.apache.org/viewvc?rev=1082484&view=rev
Log:
Fix for CONNECTORS-169. Add a method to database abstraction to handle maximum OR clauses.
Modified:
incubator/lcf/trunk/CHANGES.txt
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Modified: incubator/lcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/CHANGES.txt?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/CHANGES.txt (original)
+++ incubator/lcf/trunk/CHANGES.txt Thu Mar 17 13:30:26 2011
@@ -3,6 +3,11 @@ $Id$
================== 0.2-dev ==================
+CONNECTORS-169: Add a method to the database abstraction to
+return the maximum number of OR clauses in a query. This is to make
+Derby efficient, since it can't seem to use indexes in this situation
+(Karl Wright)
+
CONNECTORS-166: Pull a Derby fix down from trunk into Derby 10.7.1.1,
to fix deadlock problems
(Oleg Tikhonov, Karl Wright)
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java Thu Mar 17 13:30:26 2011
@@ -357,6 +357,17 @@ public class BaseTable
return dbInterface.getMaxInClause();
}
+ /** Obtain the maximum number of individual clauses that should be
+ * present in a sequence of OR clauses. Exceeding this amount will potentially cause the query performance
+ * to drop.
+ *@return the maximum number of OR clause members.
+ */
+ protected int getMaxOrClause()
+ {
+ return dbInterface.getMaxOrClause();
+ }
+
+
/** Set up a base object from a database row.
*@param object is the object to read into.
*@param resultRow is the row to use to initialize the object.
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java Thu Mar 17 13:30:26 2011
@@ -964,6 +964,17 @@ public class DBInterfaceDerby extends Da
return 100;
}
+ /** Obtain the maximum number of individual clauses that should be
+ * present in a sequence of OR clauses. Exceeding this amount will potentially cause the query performance
+ * to drop.
+ *@return the maximum number of OR clause members.
+ */
+ public int getMaxOrClause()
+ {
+ return 1;
+ }
+
+
/** Begin a database transaction. This method call MUST be paired with an endTransaction() call,
* or database handles will be lost. If the transaction should be rolled back, then signalRollback() should
* be called before the transaction is ended.
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java Thu Mar 17 13:30:26 2011
@@ -936,6 +936,17 @@ public class DBInterfaceHSQLDB extends D
return 100;
}
+ /** Obtain the maximum number of individual clauses that should be
+ * present in a sequence of OR clauses. Exceeding this amount will potentially cause the query performance
+ * to drop.
+ *@return the maximum number of OR clause members.
+ */
+ public int getMaxOrClause()
+ {
+ return 25;
+ }
+
+
/** Begin a database transaction. This method call MUST be paired with an endTransaction() call,
* or database handles will be lost. If the transaction should be rolled back, then signalRollback() should
* be called before the transaction is ended.
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java Thu Mar 17 13:30:26 2011
@@ -633,6 +633,17 @@ public class DBInterfaceMySQL extends Da
return 100;
}
+ /** Obtain the maximum number of individual clauses that should be
+ * present in a sequence of OR clauses. Exceeding this amount will potentially cause the query performance
+ * to drop.
+ *@return the maximum number of OR clause members.
+ */
+ public int getMaxOrClause()
+ {
+ return 25;
+ }
+
+
/** Begin a database transaction. This method call MUST be paired with an endTransaction() call,
* or database handles will be lost. If the transaction should be rolled back, then signalRollback() should
* be called before the transaction is ended.
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java Thu Mar 17 13:30:26 2011
@@ -988,6 +988,17 @@ public class DBInterfacePostgreSQL exten
return 100;
}
+ /** Obtain the maximum number of individual clauses that should be
+ * present in a sequence of OR clauses. Exceeding this amount will potentially cause the query performance
+ * to drop.
+ *@return the maximum number of OR clause members.
+ */
+ public int getMaxOrClause()
+ {
+ return 25;
+ }
+
+
/** Begin a database transaction. This method call MUST be paired with an endTransaction() call,
* or database handles will be lost. If the transaction should be rolled back, then signalRollback() should
* be called before the transaction is ended.
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java Thu Mar 17 13:30:26 2011
@@ -304,6 +304,13 @@ public interface IDBInterface
*/
public int getMaxInClause();
+ /** Obtain the maximum number of individual clauses that should be
+ * present in a sequence of OR clauses. Exceeding this amount will potentially cause the query performance
+ * to drop.
+ *@return the maximum number of OR clause members.
+ */
+ public int getMaxOrClause();
+
/** Begin a database transaction. This method call MUST be paired with an endTransaction() call,
* or database handles will be lost. If the transaction should be rolled back, then signalRollback() should
* be called before the transaction is ended.
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java Thu Mar 17 13:30:26 2011
@@ -227,7 +227,7 @@ public class Carrydown extends org.apach
HashMap duplicateRemoval = new HashMap();
HashMap presentMap = new HashMap();
- int maxClause = 25;
+ int maxClause = getMaxOrClause();
StringBuffer sb = new StringBuffer();
ArrayList list = new ArrayList();
int i = 0;
@@ -409,7 +409,7 @@ public class Carrydown extends org.apach
beginTransaction();
try
{
- int maxClause = 25;
+ int maxClause = getMaxOrClause();
StringBuffer sb = new StringBuffer();
ArrayList list = new ArrayList();
int i = 0;
@@ -481,7 +481,7 @@ public class Carrydown extends org.apach
beginTransaction();
try
{
- int maxClause = 25;
+ int maxClause = getMaxOrClause();
StringBuffer sb = new StringBuffer();
StringBuffer sb2 = new StringBuffer();
ArrayList list = new ArrayList();
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java Thu Mar 17 13:30:26 2011
@@ -607,7 +607,7 @@ public class HopCount extends org.apache
i++;
}
- int maxClause = 25;
+ int maxClause = getMaxOrClause();
i = 0;
int k = 0;
while (i < parentIdentifierHashes.length)
@@ -809,7 +809,7 @@ public class HopCount extends org.apache
// an IN clause for one index field, and = clauses for the others. So I have to generate this
// as ORed together match tuples. I do 25 at a pop, which is arbitrary.
- int maxClause = 25;
+ int maxClause = getMaxOrClause();
StringBuffer sb = new StringBuffer();
ArrayList list = new ArrayList();
int i = 0;
@@ -1111,7 +1111,7 @@ public class HopCount extends org.apache
// ...
// OR (t0.jobid=? AND t0.childidhash=? AND t0.childid=?))
- int maxClause = 25;
+ int maxClause = getMaxOrClause();
ArrayList list = new ArrayList();
StringBuffer sb = new StringBuffer();
int i = 0;
@@ -1305,7 +1305,7 @@ public class HopCount extends org.apache
// Accumulate the ids of rows where I need deps too. This is keyed by id and has the right answer object as a value.
Map depsMap = new HashMap();
- int maxClause = 25;
+ int maxClause = getMaxOrClause();
i = 0;
int k = 0;
while (i < unansweredQuestions.length)
@@ -2854,7 +2854,7 @@ public class HopCount extends org.apache
HashMap referenceMap = new HashMap();
- int maxClause = 25;
+ int maxClause = getMaxOrClause();
StringBuffer sb = new StringBuffer();
ArrayList list = new ArrayList();
k = 0;
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java Thu Mar 17 13:30:26 2011
@@ -190,7 +190,7 @@ public class IntrinsicLink extends org.a
// one thread can make it into here at any given time for any given parent.
//performLock();
HashMap duplicateRemoval = new HashMap();
- int maxClause = 25;
+ int maxClause = getMaxOrClause();
StringBuffer sb = new StringBuffer();
ArrayList list = new ArrayList();
int i = 0;
@@ -319,7 +319,7 @@ public class IntrinsicLink extends org.a
{
if (sourceDocumentIDHashes != null)
{
- int maxClause = 25;
+ int maxClause = getMaxOrClause();
StringBuffer sb = new StringBuffer();
ArrayList list = new ArrayList();
int i = 0;
@@ -401,7 +401,7 @@ public class IntrinsicLink extends org.a
beginTransaction();
try
{
- int maxClause = 25;
+ int maxClause = getMaxOrClause();
StringBuffer sb = new StringBuffer();
ArrayList list = new ArrayList();
int i = 0;
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java Thu Mar 17 13:30:26 2011
@@ -2541,7 +2541,7 @@ public class JobManager implements IJobM
HashMap resultHash = new HashMap();
StringBuffer sb = new StringBuffer();
ArrayList list = new ArrayList();
- int maxCount = 25;
+ int maxCount = database.getMaxOrClause();
int i = 0;
int z = 0;
while (i < docIDHashes.length)
@@ -4003,7 +4003,7 @@ public class JobManager implements IJobM
HashMap resultHash = new HashMap();
StringBuffer sb = new StringBuffer();
ArrayList list = new ArrayList();
- int maxCount = 25;
+ int maxCount = database.getMaxOrClause();
int i = 0;
int z = 0;
while (i < parentIDHashes.length)